ipcp, irmd, lib: Notify IRMd upon IPCP initialization

This will notify the IRMd when the IPCP is initialized and ready to
receive messages. Previously a bootstrap could fail since the IPCP was
not listening to the socket yet.

8 files changed, 122 insertions, 19 deletions

diff --git a/include/ouroboros/ipcp.h b/include/ouroboros/ipcp.h
index 0ce95b1e..a83d8a77 100644
--- a/include/ouroboros/ipcp.h
+++ b/include/ouroboros/ipcp.h
@@ -34,6 +34,9 @@ struct ipcp;
 /* Returns the process id */
 pid_t ipcp_create(enum ipcp_type ipcp_type);
 
+/* IPCP calls this when it is initialized */
+int   ipcp_create_r(pid_t api);
+
 int   ipcp_destroy(pid_t api);
 
 int   ipcp_enroll(pid_t  api,
diff --git a/src/ipcpd/local/main.c b/src/ipcpd/local/main.c
index 99580a45..62f9e0aa 100644
--- a/src/ipcpd/local/main.c
+++ b/src/ipcpd/local/main.c
@@ -621,6 +621,12 @@ int main(int argc, char * argv[])
 
         pthread_rwlock_unlock(&_ipcp->state_lock);
 
+        if (ipcp_create_r(getpid())) {
+                LOG_ERR("Failed to notify IRMd we are initialized.");
+                close_logfile();
+                exit(EXIT_FAILURE);
+        }
+
         pthread_join(_ap_instance->mainloop, NULL);
 
         shim_ap_fini();
diff --git a/src/ipcpd/normal/main.c b/src/ipcpd/normal/main.c
index 2d416942..38789ddd 100644
--- a/src/ipcpd/normal/main.c
+++ b/src/ipcpd/normal/main.c
@@ -27,6 +27,7 @@
 #include <ouroboros/shm_du_map.h>
 #include <ouroboros/shm_ap_rbuff.h>
 #include <ouroboros/dev.h>
+#include <ouroboros/ipcp.h>
 
 #include <stdbool.h>
 #include <signal.h>
@@ -326,6 +327,15 @@ int main(int argc, char * argv[])
 
         pthread_rwlock_unlock(&_ipcp->state_lock);
 
+        if (ipcp_create_r(getpid())) {
+                LOG_ERR("Failed to notify IRMd we are initialized.");
+                normal_ipcp_data_destroy();
+                fmgr_fini();
+                free(_ipcp);
+                close_logfile();
+                exit(EXIT_FAILURE);
+        }
+
         pthread_join(normal_data(_ipcp)->mainloop, NULL);
 
         normal_ipcp_data_destroy();
diff --git a/src/ipcpd/shim-eth-llc/main.c b/src/ipcpd/shim-eth-llc/main.c
index 7471e319..b8689ac7 100644
--- a/src/ipcpd/shim-eth-llc/main.c
+++ b/src/ipcpd/shim-eth-llc/main.c
@@ -1379,6 +1379,12 @@ int main(int argc, char * argv[])
 
         pthread_rwlock_unlock(&_ipcp->state_lock);
 
+        if (ipcp_create_r(getpid())) {
+                LOG_ERR("Failed to notify IRMd we are initialized.");
+                close_logfile();
+                exit(EXIT_FAILURE);
+        }
+
         pthread_join(shim_data(_ipcp)->mainloop, NULL);
 
         eth_llc_ipcp_data_destroy();
diff --git a/src/ipcpd/shim-udp/main.c b/src/ipcpd/shim-udp/main.c
index 9d6d15a7..8d9ac25c 100644
--- a/src/ipcpd/shim-udp/main.c
+++ b/src/ipcpd/shim-udp/main.c
@@ -1609,6 +1609,12 @@ int main(int argc, char * argv[])
 
         pthread_rwlock_unlock(&_ipcp->state_lock);
 
+        if (ipcp_create_r(getpid())) {
+                LOG_ERR("Failed to notify IRMd we are initialized.");
+                close_logfile();
+                exit(EXIT_FAILURE);
+        }
+
         pthread_join(_ap_instance->mainloop, NULL);
 
         shim_ap_fini();
diff --git a/src/irmd/main.c b/src/irmd/main.c
index 8503fcfa..27f57077 100644
--- a/src/irmd/main.c
+++ b/src/irmd/main.c
@@ -63,6 +63,9 @@ struct ipcp_entry {
         pid_t            api;
         enum ipcp_type   type;
         char *           dif_name;
+        pthread_cond_t   init_cond;
+        pthread_mutex_t  init_lock;
+        bool             init;
 };
 
 enum irm_state {
@@ -244,8 +247,11 @@ static pid_t create_ipcp(char *         name,
                 return -1;
         }
 
+        pthread_rwlock_wrlock(&irmd->reg_lock);
+
         api->api = ipcp_create(ipcp_type);
         if (api->api == -1) {
+                pthread_rwlock_unlock(&irmd->reg_lock);
                 pthread_rwlock_unlock(&irmd->state_lock);
                 LOG_ERR("Failed to create IPCP.");
                 return -1;
@@ -253,6 +259,7 @@ static pid_t create_ipcp(char *         name,
 
         tmp = ipcp_entry_create();
         if (tmp == NULL) {
+                pthread_rwlock_unlock(&irmd->reg_lock);
                 pthread_rwlock_unlock(&irmd->state_lock);
                 return -1;
         }
@@ -263,14 +270,17 @@ static pid_t create_ipcp(char *         name,
         tmp->name = strdup(name);
         if (tmp->name  == NULL) {
                 ipcp_entry_destroy(tmp);
+                pthread_rwlock_unlock(&irmd->reg_lock);
                 pthread_rwlock_unlock(&irmd->state_lock);
                 return -1;
         }
 
+        pthread_cond_init(&tmp->init_cond, NULL);
+        pthread_mutex_init(&tmp->init_lock, NULL);
+
         tmp->dif_name = NULL;
         tmp->type = ipcp_type;
-
-        pthread_rwlock_wrlock(&irmd->reg_lock);
+        tmp->init = false;
 
         list_for_each(pos, &irmd->ipcps) {
                 struct ipcp_entry * e =
@@ -283,14 +293,46 @@ static pid_t create_ipcp(char *         name,
 
         list_add(&api->next, &irmd->spawned_apis);
 
+        pthread_mutex_lock(&tmp->init_lock);
+
         pthread_rwlock_unlock(&irmd->reg_lock);
         pthread_rwlock_unlock(&irmd->state_lock);
 
+        while (tmp->init == false)
+                pthread_cond_wait(&tmp->init_cond, &tmp->init_lock);
+
+        pthread_mutex_unlock(&tmp->init_lock);
+
         LOG_INFO("Created IPCP %d.", api->api);
 
         return api->api;
 }
 
+static int create_ipcp_r(pid_t api)
+{
+        struct list_head * pos = NULL;
+
+        pthread_rwlock_rdlock(&irmd->state_lock);
+        pthread_rwlock_rdlock(&irmd->reg_lock);
+
+        list_for_each(pos, &irmd->ipcps) {
+                struct ipcp_entry * e =
+                        list_entry(pos, struct ipcp_entry, next);
+
+                if (e->api == api) {
+                        pthread_mutex_lock(&e->init_lock);
+                        e->init = true;
+                        pthread_cond_broadcast(&e->init_cond);
+                        pthread_mutex_unlock(&e->init_lock);
+                }
+        }
+
+        pthread_rwlock_unlock(&irmd->reg_lock);
+        pthread_rwlock_unlock(&irmd->state_lock);
+
+        return 0;
+}
+
 static void clear_spawned_api(pid_t api)
 {
         struct list_head * pos = NULL;
@@ -1480,6 +1522,10 @@ void * mainloop()
                         ret_msg.result = create_ipcp(msg->dst_name,
                                                      msg->ipcp_type);
                         break;
+                case IRM_MSG_CODE__IPCP_CREATE_R:
+                        ret_msg.has_result = true;
+                        ret_msg.result = create_ipcp_r(msg->api);
+                        break;
                 case IRM_MSG_CODE__IRM_DESTROY_IPCP:
                         ret_msg.has_result = true;
                         ret_msg.result = destroy_ipcp(msg->api);
diff --git a/src/lib/ipcp.c b/src/lib/ipcp.c
index 141db43d..9ff05e7b 100644
--- a/src/lib/ipcp.c
+++ b/src/lib/ipcp.c
@@ -168,6 +168,31 @@ pid_t ipcp_create(enum ipcp_type ipcp_type)
         exit(EXIT_FAILURE);
 }
 
+int ipcp_create_r(pid_t api)
+{
+        irm_msg_t msg = IRM_MSG__INIT;
+        irm_msg_t * recv_msg = NULL;
+        int ret = -1;
+
+        msg.code         = IRM_MSG_CODE__IPCP_CREATE_R;
+        msg.has_api      = true;
+        msg.api          = api;
+
+        recv_msg = send_recv_irm_msg(&msg);
+        if (recv_msg == NULL)
+                return -1;
+
+        if (recv_msg->has_result == false) {
+                irm_msg__free_unpacked(recv_msg, NULL);
+                return -1;
+        }
+
+        ret = recv_msg->result;
+        irm_msg__free_unpacked(recv_msg, NULL);
+
+        return ret;
+}
+
 int ipcp_destroy(pid_t api)
 {
         int status;
diff --git a/src/lib/irmd_messages.proto b/src/lib/irmd_messages.proto
index 315d6092..730f842c 100644
--- a/src/lib/irmd_messages.proto
+++ b/src/lib/irmd_messages.proto
@@ -24,23 +24,24 @@ import "dif_config.proto";
 
 enum irm_msg_code {
         IRM_CREATE_IPCP       =  1;
-        IRM_DESTROY_IPCP      =  2;
-        IRM_LIST_IPCPS        =  3;
-        IRM_BOOTSTRAP_IPCP    =  4;
-        IRM_ENROLL_IPCP       =  5;
-        IRM_BIND              =  6;
-        IRM_UNBIND            =  7;
-        IRM_REG               =  8;
-        IRM_UNREG             =  9;
-        IRM_FLOW_ACCEPT       = 10;
-        IRM_FLOW_ALLOC_RESP   = 11;
-        IRM_FLOW_ALLOC        = 12;
-        IRM_FLOW_ALLOC_RES    = 13;
-        IRM_FLOW_DEALLOC      = 14;
-        IPCP_FLOW_REQ_ARR     = 15;
-        IPCP_FLOW_ALLOC_REPLY = 16;
-        IPCP_FLOW_DEALLOC     = 17;
-        IRM_REPLY             = 18;
+        IPCP_CREATE_R         =  2;
+        IRM_DESTROY_IPCP      =  3;
+        IRM_LIST_IPCPS        =  4;
+        IRM_BOOTSTRAP_IPCP    =  5;
+        IRM_ENROLL_IPCP       =  6;
+        IRM_BIND              =  7;
+        IRM_UNBIND            =  8;
+        IRM_REG               =  9;
+        IRM_UNREG             = 10;
+        IRM_FLOW_ACCEPT       = 11;
+        IRM_FLOW_ALLOC_RESP   = 12;
+        IRM_FLOW_ALLOC        = 13;
+        IRM_FLOW_ALLOC_RES    = 14;
+        IRM_FLOW_DEALLOC      = 15;
+        IPCP_FLOW_REQ_ARR     = 16;
+        IPCP_FLOW_ALLOC_REPLY = 17;
+        IPCP_FLOW_DEALLOC     = 18;
+        IRM_REPLY             = 19;
 };
 
 message irm_msg {