lib: Move rbuff creation/destruction to IRMd

This stabilises flow allocation now that the rbuffs are created upon
flow allocation. Only the IRMd can sync this process sufficiently.

3 files changed, 37 insertions, 61 deletions

diff --git a/src/lib/dev.c b/src/lib/dev.c
index f735e72b..146070b7 100644
--- a/src/lib/dev.c
+++ b/src/lib/dev.c
@@ -288,7 +288,7 @@ void ap_fini()
                         int idx;
                         while ((idx = shm_rbuff_read(ai.flows[i].rx_rb)) >= 0)
                                 shm_rdrbuff_remove(ai.rdrb, idx);
-                        shm_rbuff_destroy(ai.flows[i].rx_rb);
+                        shm_rbuff_close(ai.flows[i].rx_rb);
                         shm_rbuff_close(ai.flows[i].tx_rb);
                         shm_flow_set_close(ai.flows[i].set);
                 }
@@ -349,7 +349,7 @@ int flow_accept(char ** ae_name)
                 return -1;
         }
 
-        ai.flows[fd].rx_rb = shm_rbuff_create(recv_msg->port_id);
+        ai.flows[fd].rx_rb = shm_rbuff_open(ai.api, recv_msg->port_id);
         if (ai.flows[fd].rx_rb == NULL) {
                 bmp_release(ai.fds, fd);
                 pthread_rwlock_unlock(&ai.flows_lock);
@@ -361,7 +361,7 @@ int flow_accept(char ** ae_name)
         ai.flows[fd].set = shm_flow_set_open(recv_msg->api);
         if (ai.flows[fd].set == NULL) {
                 bmp_release(ai.fds, fd);
-                shm_rbuff_destroy(ai.flows[fd].rx_rb);
+                shm_rbuff_close(ai.flows[fd].rx_rb);
                 shm_rbuff_close(ai.flows[fd].tx_rb);
                 pthread_rwlock_unlock(&ai.flows_lock);
                 pthread_rwlock_unlock(&ai.data_lock);
@@ -373,7 +373,7 @@ int flow_accept(char ** ae_name)
         if (ae_name != NULL) {
                 *ae_name = strdup(recv_msg->ae_name);
                 if (*ae_name == NULL) {
-                        shm_rbuff_destroy(ai.flows[fd].tx_rb);
+                        shm_rbuff_close(ai.flows[fd].tx_rb);
                         shm_rbuff_close(ai.flows[fd].tx_rb);
                         shm_flow_set_close(ai.flows[fd].set);
                         bmp_release(ai.fds, fd);
@@ -508,7 +508,7 @@ int flow_alloc(char * dst_name, char * src_ae_name, struct qos_spec * qos)
         ai.flows[fd].port_id = recv_msg->port_id;
         ai.flows[fd].oflags  = FLOW_O_DEFAULT;
         ai.flows[fd].api     = recv_msg->api;
-        ai.flows[fd].rx_rb   = shm_rbuff_create(recv_msg->port_id);
+        ai.flows[fd].rx_rb   = shm_rbuff_open(ai.api, recv_msg->port_id);
         if (ai.flows[fd].rx_rb == NULL) {
                 bmp_release(ai.fds, fd);
                 pthread_rwlock_unlock(&ai.flows_lock);
@@ -517,27 +517,6 @@ int flow_alloc(char * dst_name, char * src_ae_name, struct qos_spec * qos)
                 return -1;
         }
 
-        ai.flows[fd].tx_rb   = shm_rbuff_open(recv_msg->api, recv_msg->port_id);
-        if (ai.flows[fd].tx_rb == NULL) {
-                shm_rbuff_destroy(ai.flows[fd].rx_rb);
-                bmp_release(ai.fds, fd);
-                pthread_rwlock_unlock(&ai.flows_lock);
-                pthread_rwlock_unlock(&ai.data_lock);
-                irm_msg__free_unpacked(recv_msg, NULL);
-                return -1;
-        }
-
-        ai.flows[fd].set = shm_flow_set_open(recv_msg->api);
-        if (ai.flows[fd].set == NULL) {
-                shm_rbuff_close(ai.flows[fd].tx_rb);
-                shm_rbuff_destroy(ai.flows[fd].rx_rb);
-                bmp_release(ai.fds, fd);
-                pthread_rwlock_unlock(&ai.flows_lock);
-                pthread_rwlock_unlock(&ai.data_lock);
-                irm_msg__free_unpacked(recv_msg, NULL);
-                return -1;
-        }
-
         ai.ports[recv_msg->port_id].fd    = fd;
         ai.ports[recv_msg->port_id].state = PORT_ID_ASSIGNED;
 
@@ -572,6 +551,23 @@ int flow_alloc_res(int fd)
 
         msg.port_id = ai.flows[fd].port_id;
 
+        ai.flows[fd].tx_rb = shm_rbuff_open(ai.flows[fd].api,
+                                            ai.flows[fd].port_id);
+        if (ai.flows[fd].tx_rb == NULL) {
+                pthread_rwlock_unlock(&ai.flows_lock);
+                pthread_rwlock_unlock(&ai.data_lock);
+                irm_msg__free_unpacked(recv_msg, NULL);
+                return -1;
+        }
+
+        ai.flows[fd].set = shm_flow_set_open(ai.flows[fd].api);
+        if (ai.flows[fd].set == NULL) {
+                pthread_rwlock_unlock(&ai.flows_lock);
+                pthread_rwlock_unlock(&ai.data_lock);
+                irm_msg__free_unpacked(recv_msg, NULL);
+                return -1;
+        }
+
         pthread_rwlock_unlock(&ai.flows_lock);
         pthread_rwlock_unlock(&ai.data_lock);
 
@@ -599,7 +595,7 @@ int flow_dealloc(int fd)
         msg.code         = IRM_MSG_CODE__IRM_FLOW_DEALLOC;
         msg.has_port_id  = true;
         msg.has_api      = true;
-        msg.api          = getpid();
+        msg.api          = ai.api;
 
         pthread_rwlock_rdlock(&ai.data_lock);
         pthread_rwlock_wrlock(&ai.flows_lock);
@@ -621,7 +617,7 @@ int flow_dealloc(int fd)
         port_destroy(&ai.ports[msg.port_id]);
 
         ai.flows[fd].port_id = -1;
-        shm_rbuff_destroy(ai.flows[fd].rx_rb);
+        shm_rbuff_close(ai.flows[fd].rx_rb);
         ai.flows[fd].rx_rb = NULL;
         shm_rbuff_close(ai.flows[fd].tx_rb);
         ai.flows[fd].tx_rb = NULL;
@@ -990,7 +986,7 @@ int np1_flow_alloc(pid_t n_api, int port_id)
                 return -1;
         }
 
-        ai.flows[fd].rx_rb = shm_rbuff_create(port_id);
+        ai.flows[fd].rx_rb = shm_rbuff_open(ai.api, port_id);
         if (ai.flows[fd].rx_rb == NULL) {
                 bmp_release(ai.fds, fd);
                 pthread_rwlock_unlock(&ai.flows_lock);
@@ -1046,7 +1042,7 @@ int np1_flow_resp(pid_t n_api, int port_id)
         ai.flows[fd].tx_rb = shm_rbuff_open(n_api, port_id);
         if (ai.flows[fd].tx_rb == NULL) {
                 ai.flows[fd].port_id = -1;
-                shm_rbuff_destroy(ai.flows[fd].rx_rb);
+                shm_rbuff_close(ai.flows[fd].rx_rb);
                 port_destroy(&ai.ports[port_id]);
                 pthread_rwlock_unlock(&ai.flows_lock);
                 pthread_rwlock_unlock(&ai.data_lock);
@@ -1057,7 +1053,7 @@ int np1_flow_resp(pid_t n_api, int port_id)
         if (ai.flows[fd].set == NULL) {
                 shm_rbuff_close(ai.flows[fd].tx_rb);
                 ai.flows[fd].port_id = -1;
-                shm_rbuff_destroy(ai.flows[fd].rx_rb);
+                shm_rbuff_close(ai.flows[fd].rx_rb);
                 port_destroy(&ai.ports[port_id]);
                 pthread_rwlock_unlock(&ai.flows_lock);
                 pthread_rwlock_unlock(&ai.data_lock);
@@ -1143,7 +1139,7 @@ int ipcp_flow_req_arr(pid_t  api, char * dst_name, char * src_ae_name)
         pthread_rwlock_rdlock(&ai.data_lock);
         pthread_rwlock_wrlock(&ai.flows_lock);
 
-        ai.flows[fd].rx_rb = shm_rbuff_create(port_id);
+        ai.flows[fd].rx_rb = shm_rbuff_open(ai.api, port_id);
         if (ai.flows[fd].rx_rb == NULL) {
                 ai.flows[fd].port_id = -1;
                 port_destroy(&ai.ports[port_id]);
diff --git a/src/lib/shm_flow_set.c b/src/lib/shm_flow_set.c
index c960bd25..04de9fc5 100644
--- a/src/lib/shm_flow_set.c
+++ b/src/lib/shm_flow_set.c
@@ -315,7 +315,6 @@ int shm_flow_set_has(struct shm_flow_set * shm_set,
         assert(!(port_id < 0) && port_id < IRMD_MAX_FLOWS);
         assert(!(idx < 0) && idx < AP_MAX_FQUEUES);
 
-
         pthread_mutex_lock(shm_set->lock);
 
         if (shm_set->mtable[port_id] == idx)
diff --git a/src/lib/shm_rbuff.c b/src/lib/shm_rbuff.c
index cf094488..a933fbff 100644
--- a/src/lib/shm_rbuff.c
+++ b/src/lib/shm_rbuff.c
@@ -68,7 +68,7 @@ struct shm_rbuff {
         int               port_id;  /* port_id of the flow           */
 };
 
-struct shm_rbuff * shm_rbuff_create(int port_id)
+struct shm_rbuff * shm_rbuff_create(pid_t api, int port_id)
 {
         struct shm_rbuff *  rb;
         int                 shm_fd;
@@ -78,7 +78,7 @@ struct shm_rbuff * shm_rbuff_create(int port_id)
         char                fn[FN_MAX_CHARS];
         mode_t              mask;
 
-        sprintf(fn, SHM_RBUFF_PREFIX "%d.%d", getpid(), port_id);
+        sprintf(fn, SHM_RBUFF_PREFIX "%d.%d", api, port_id);
 
         rb = malloc(sizeof(*rb));
         if (rb == NULL) {
@@ -148,9 +148,12 @@ struct shm_rbuff * shm_rbuff_create(int port_id)
         *rb->head = 0;
         *rb->tail = 0;
 
-        rb->api = getpid();
+        rb->api = api;
         rb->port_id = port_id;
 
+        if (munmap(rb->shm_base, SHM_RBUFF_FILE_SIZE) == -1)
+                LOG_DBG("Couldn't unmap shared memory.");
+
         return rb;
 }
 
@@ -221,36 +224,14 @@ void shm_rbuff_close(struct shm_rbuff * rb)
 void shm_rbuff_destroy(struct shm_rbuff * rb)
 {
         char fn[25];
-        struct lockfile * lf = NULL;
-
-        assert(rb);
 
-        if (rb->api != getpid()) {
-                lf = lockfile_open();
-                if (lf == NULL) {
-                        LOG_ERR("Failed to open lockfile.");
-                        return;
-                }
-
-                if (lockfile_owner(lf) == getpid()) {
-                        LOG_DBG("Ringbuffer %d destroyed by IRMd %d.",
-                                 rb->api, getpid());
-                        lockfile_close(lf);
-                } else {
-                        LOG_ERR("AP-I %d tried to destroy rbuff owned by %d.",
-                                getpid(), rb->api);
-                        lockfile_close(lf);
-                        return;
-                }
-        }
+        if (rb == NULL)
+                return;
 
         sprintf(fn, SHM_RBUFF_PREFIX "%d.%d", rb->api, rb->port_id);
 
-        if (munmap(rb->shm_base, SHM_RBUFF_FILE_SIZE) == -1)
-                LOG_DBG("Couldn't unmap shared memory.");
-
         if (shm_unlink(fn) == -1)
-                LOG_DBG("Failed to unlink shm.");
+                LOG_DBG("Failed to unlink shm %s.", fn);
 
         free(rb);
 }