irmd: Revise IRMd internals

This is a full revision of the IRMd internal implementation.

The registry is now a proper subcomponent managing its own internal
lock (a single mutex). Some tests are added for the registry and its
data structures. Some macros for tests are added in <ouroboros/test.h>.

Flow allocation is now more symmetric between the client side (alloc)
and server size (accept). Each will create a flow in pending state
(ALLOC_PENDING/ACCEPT_PENDING) that is potentially fulfilled by an
IPCP using respond_alloc and respond_accept primitives. Deallocation
is split in flow_dealloc (application side) and ipcp_flow_dealloc
(IPCP side) to get the flow in DEALLOC_PENDING and DEALLOCATED state.

Cleanup of failed flow allocation is now properly handled instead of
relying on the sanitizer thread. The new sanitizer only needs to
monitor crashed processes.

On shutdown, the IRMd will now detect hanging processes and SIGKILL
them and clean up their fuse mountpoints if needed.

A lot of other things have been cleaned up and shuffled around a bit.

Signed-off-by: Dimitri Staessens <[email protected]>
Signed-off-by: Sander Vrijders <[email protected]>

11 files changed, 68 insertions, 71 deletions

diff --git a/src/ipcpd/broadcast/main.c b/src/ipcpd/broadcast/main.c
index 55d14b31..d2cbed6f 100644
--- a/src/ipcpd/broadcast/main.c
+++ b/src/ipcpd/broadcast/main.c
@@ -31,14 +31,14 @@
 #define OUROBOROS_PREFIX "broadcast-ipcp"
 #define THIS_TYPE IPCP_BROADCAST
 
-#include <ouroboros/errno.h>
 #include <ouroboros/dev.h>
+#include <ouroboros/errno.h>
 #include <ouroboros/ipcp-dev.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/notifier.h>
 #include <ouroboros/random.h>
 #include <ouroboros/rib.h>
-#include <ouroboros/time_utils.h>
+#include <ouroboros/time.h>
 
 #include "common/connmgr.h"
 #include "common/enroll.h"
@@ -241,7 +241,7 @@ int broadcast_ipcp_dealloc(int fd)
 
         notifier_event(NOTIFY_DT_CONN_DEL, &conn);
 
-        flow_dealloc(fd);
+        ipcp_flow_dealloc(fd);
 
         return 0;
 }
diff --git a/src/ipcpd/common/enroll.c b/src/ipcpd/common/enroll.c
index 94dbf7f4..5e35ce37 100644
--- a/src/ipcpd/common/enroll.c
+++ b/src/ipcpd/common/enroll.c
@@ -28,12 +28,11 @@
 
 #define OUROBOROS_PREFIX "enrollment"
 
-#include <ouroboros/errno.h>
-#include <ouroboros/time_utils.h>
 #include <ouroboros/dev.h>
-#include <ouroboros/logs.h>
 #include <ouroboros/errno.h>
+#include <ouroboros/logs.h>
 #include <ouroboros/serdes-oep.h>
+#include <ouroboros/time.h>
 
 #include "common/connmgr.h"
 #include "common/enroll.h"
diff --git a/src/ipcpd/eth/eth.c b/src/ipcpd/eth/eth.c
index 6bac6c76..c0aaf711 100644
--- a/src/ipcpd/eth/eth.c
+++ b/src/ipcpd/eth/eth.c
@@ -47,7 +47,7 @@
 #include <ouroboros/ipcp-dev.h>
 #include <ouroboros/fqueue.h>
 #include <ouroboros/logs.h>
-#include <ouroboros/time_utils.h>
+#include <ouroboros/time.h>
 #include <ouroboros/fccntl.h>
 #include <ouroboros/pthread.h>
 
@@ -786,8 +786,7 @@ static void * eth_ipcp_mgmt_handler(void * o)
 
         while (true) {
                 int                 ret = 0;
-                struct timespec     timeout = {(MGMT_TIMEO / 1000),
-                                               (MGMT_TIMEO % 1000) * MILLION};
+                struct timespec     timeout = TIMESPEC_INIT_MS(MGMT_TIMEO);
                 struct timespec     abstime;
                 struct mgmt_frame * frame = NULL;
 
@@ -1415,7 +1414,7 @@ static int eth_ipcp_bootstrap(const struct ipcp_config * conf)
 
         if (eth_data.s_fd < 0) {
                 log_err("Failed to create socket.");
-                return -1;
+                goto fail_socket;
         }
 
         flags = fcntl(eth_data.s_fd, F_GETFL, 0);
@@ -1437,38 +1436,30 @@ static int eth_ipcp_bootstrap(const struct ipcp_config * conf)
     #endif
 
         if (bind(eth_data.s_fd, (struct sockaddr *) &eth_data.device,
-                 sizeof(eth_data.device))) {
+                 sizeof(eth_data.device)) < 0) {
                 log_err("Failed to bind socket to interface.");
                 goto fail_device;
         }
-
 #endif /* HAVE_NETMAP */
-
 #if defined(__linux__)
-        if (pthread_create(&eth_data.if_monitor,
-                           NULL,
-                           eth_ipcp_if_monitor,
-                           NULL)) {
+        if (pthread_create(&eth_data.if_monitor, NULL,
+                           eth_ipcp_if_monitor, NULL)) {
                 log_err("Failed to create monitor thread: %s.",
                         strerror(errno));
                 goto fail_device;
         }
 #endif
 
-        if (pthread_create(&eth_data.mgmt_handler,
-                           NULL,
-                           eth_ipcp_mgmt_handler,
-                           NULL)) {
+        if (pthread_create(&eth_data.mgmt_handler, NULL,
+                           eth_ipcp_mgmt_handler, NULL)) {
                 log_err("Failed to create mgmt handler thread: %s.",
                         strerror(errno));
                 goto fail_mgmt_handler;
         }
 
         for (idx = 0; idx < IPCP_ETH_RD_THR; ++idx) {
-                if (pthread_create(&eth_data.packet_reader[idx],
-                                   NULL,
-                                   eth_ipcp_packet_reader,
-                                   NULL)) {
+                if (pthread_create(&eth_data.packet_reader[idx], NULL,
+                                   eth_ipcp_packet_reader, NULL)) {
                         log_err("Failed to create packet reader thread: %s",
                                 strerror(errno));
                         goto fail_packet_reader;
@@ -1476,10 +1467,8 @@ static int eth_ipcp_bootstrap(const struct ipcp_config * conf)
         }
 
         for (idx = 0; idx < IPCP_ETH_WR_THR; ++idx) {
-                if (pthread_create(&eth_data.packet_writer[idx],
-                                   NULL,
-                                   eth_ipcp_packet_writer,
-                                   NULL)) {
+                if (pthread_create(&eth_data.packet_writer[idx], NULL,
+                                   eth_ipcp_packet_writer, NULL)) {
                         log_err("Failed to create packet writer thread: %s",
                                 strerror(errno));
                         goto fail_packet_writer;
@@ -1493,7 +1482,6 @@ static int eth_ipcp_bootstrap(const struct ipcp_config * conf)
         log_dbg("Bootstrapped IPCP over Ethernet with LLC with pid %d.",
                 getpid());
 #endif
-
         return 0;
 
  fail_packet_writer:
@@ -1524,6 +1512,7 @@ static int eth_ipcp_bootstrap(const struct ipcp_config * conf)
 #elif defined(HAVE_RAW_SOCKETS)
         close(eth_data.s_fd);
 #endif
+ fail_socket:
         return -1;
 }
 
@@ -1548,8 +1537,7 @@ static int eth_ipcp_unreg(const uint8_t * hash)
 static int eth_ipcp_query(const uint8_t * hash)
 {
         uint8_t            r_addr[MAC_SIZE];
-        struct timespec    timeout = {(NAME_QUERY_TIMEO / 1000),
-                                      (NAME_QUERY_TIMEO % 1000) * MILLION};
+        struct timespec    timeout = TIMESPEC_INIT_MS(NAME_QUERY_TIMEO);
         struct dir_query * query;
         int                ret;
         uint8_t *          buf;
@@ -1752,7 +1740,7 @@ static int eth_ipcp_flow_dealloc(int fd)
 
         pthread_rwlock_unlock(&eth_data.flows_lock);
 
-        flow_dealloc(fd);
+        ipcp_flow_dealloc(fd);
 
         return 0;
 }
diff --git a/src/ipcpd/ipcp.c b/src/ipcpd/ipcp.c
index 312c4a79..0215cdaa 100644
--- a/src/ipcpd/ipcp.c
+++ b/src/ipcpd/ipcp.c
@@ -35,21 +35,21 @@
 
 #define OUROBOROS_PREFIX  "ipcpd/ipcp"
 #define IPCP_INFO         "info"
-#define ALLOC_TIMEOUT     10 * MILLION /* 10 ms */
+#define ALLOC_TIMEOUT     50 /* ms */
 
-#include <ouroboros/hash.h>
-#include <ouroboros/logs.h>
-#include <ouroboros/time_utils.h>
-#include <ouroboros/utils.h>
-#include <ouroboros/sockets.h>
-#include <ouroboros/errno.h>
+#include <ouroboros/bitmap.h>
 #include <ouroboros/dev.h>
+#include <ouroboros/errno.h>
+#include <ouroboros/hash.h>
 #include <ouroboros/ipcp-dev.h>
-#include <ouroboros/bitmap.h>
+#include <ouroboros/logs.h>
 #include <ouroboros/np1_flow.h>
-#include <ouroboros/rib.h>
 #include <ouroboros/protobuf.h>
 #include <ouroboros/pthread.h>
+#include <ouroboros/rib.h>
+#include <ouroboros/sockets.h>
+#include <ouroboros/time.h>
+#include <ouroboros/utils.h>
 
 #include "ipcp.h"
 
@@ -267,7 +267,7 @@ int ipcp_wait_flow_req_arr(const uint8_t * dst,
                            const void *    data,
                            size_t          len)
 {
-        struct timespec ts  = {0, ALLOC_TIMEOUT};
+        struct timespec ts = TIMESPEC_INIT_MS(ALLOC_TIMEOUT);
         struct timespec abstime;
         int             fd;
 
@@ -294,7 +294,7 @@ int ipcp_wait_flow_req_arr(const uint8_t * dst,
         if (fd < 0) {
                 pthread_mutex_unlock(&ipcpi.alloc_lock);
                 log_err("Failed to get fd for flow.");
-                return -ENOTALLOC;
+                return fd;
         }
 
         ipcpi.alloc_id = fd;
@@ -308,7 +308,7 @@ int ipcp_wait_flow_req_arr(const uint8_t * dst,
 
 int ipcp_wait_flow_resp(const int fd)
 {
-        struct timespec      ts = {0, ALLOC_TIMEOUT};
+        struct timespec      ts = TIMESPEC_INIT_MS(ALLOC_TIMEOUT);
         struct timespec      abstime;
 
         clock_gettime(PTHREAD_COND_CLOCK, &abstime);
@@ -502,8 +502,8 @@ static void do_flow_alloc(pid_t        pid,
 {
         int fd;
 
-        log_info("Allocating flow %d to " HASH_FMT32 ".",
-                 flow_id, HASH_VAL32(dst));
+        log_info("Allocating flow %d for %d to " HASH_FMT32 ".",
+                 flow_id, pid, HASH_VAL32(dst));
 
         if (ipcpi.ops->ipcp_flow_alloc == NULL) {
                 log_err("Flow allocation unsupported.");
@@ -519,7 +519,8 @@ static void do_flow_alloc(pid_t        pid,
 
         fd = np1_flow_alloc(pid, flow_id);
         if (fd < 0) {
-                log_err("Failed allocating n + 1 fd on flow_id %d.", flow_id);
+                log_err("Failed allocating n + 1 fd on flow_id %d: %d",
+                        flow_id, fd);
                 ret_msg->result = -EFLOWDOWN;
                 goto finish;
         }
@@ -635,7 +636,6 @@ static void do_flow_dealloc(int          flow_id,
                  flow_id, ret_msg->result);
 }
 
-
 static void * mainloop(void * o)
 {
         int                 sfd;
@@ -929,7 +929,9 @@ int ipcp_init(int               argc,
 
 int ipcp_start(void)
 {
-        sigset_t  sigset;
+        sigset_t         sigset;
+        struct ipcp_info info;
+
         sigemptyset(&sigset);
         sigaddset(&sigset, SIGINT);
         sigaddset(&sigset, SIGQUIT);
@@ -938,6 +940,11 @@ int ipcp_start(void)
 
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
+        info.pid  = getpid();
+        info.type = ipcpi.type;
+        strcpy(info.name, ipcpi.name);
+        info.state = IPCP_OPERATIONAL;
+
         if (tpm_start(ipcpi.tpm))
                 goto fail_tpm_start;
 
@@ -946,7 +953,9 @@ int ipcp_start(void)
                 goto fail_acceptor;
         }
 
-        if (ipcp_create_r(0)) {
+        info.state = IPCP_OPERATIONAL;
+
+        if (ipcp_create_r(&info)) {
                 log_err("Failed to notify IRMd we are initialized.");
                 goto fail_create_r;
         }
@@ -957,11 +966,12 @@ int ipcp_start(void)
         pthread_cancel(ipcpi.acceptor);
         pthread_join(ipcpi.acceptor, NULL);
  fail_acceptor:
-        ipcp_set_state(IPCP_NULL);
         tpm_stop(ipcpi.tpm);
  fail_tpm_start:
         tpm_destroy(ipcpi.tpm);
-        ipcp_create_r(-1);
+        ipcp_set_state(IPCP_NULL);
+        info.state = IPCP_NULL;
+        ipcp_create_r(&info);
         return -1;
 }
 
diff --git a/src/ipcpd/local/main.c b/src/ipcpd/local/main.c
index 717e35ce..dd2c7209 100644
--- a/src/ipcpd/local/main.c
+++ b/src/ipcpd/local/main.c
@@ -271,7 +271,7 @@ static int local_ipcp_flow_dealloc(int fd)
 
         pthread_rwlock_unlock(&local_data.lock);
 
-        flow_dealloc(fd);
+        ipcp_flow_dealloc(fd);
 
         log_info("Flow with fd %d deallocated.", fd);
 
diff --git a/src/ipcpd/shim-data.c b/src/ipcpd/shim-data.c
index ecbbaf2f..1fac63ac 100644
--- a/src/ipcpd/shim-data.c
+++ b/src/ipcpd/shim-data.c
@@ -30,18 +30,18 @@
 
 #define OUROBOROS_PREFIX "shim-data"
 
+#include <ouroboros/errno.h>
 #include <ouroboros/hash.h>
-#include <ouroboros/logs.h>
 #include <ouroboros/list.h>
-#include <ouroboros/time_utils.h>
-#include <ouroboros/errno.h>
+#include <ouroboros/logs.h>
+#include <ouroboros/time.h>
 
 #include "shim-data.h"
 #include "ipcp.h"
 
-#include <string.h>
-#include <stdlib.h>
 #include <assert.h>
+#include <stdlib.h>
+#include <string.h>
 
 struct reg_entry {
         struct list_head list;
diff --git a/src/ipcpd/udp/main.c b/src/ipcpd/udp/main.c
index a8168f21..909ca0a5 100644
--- a/src/ipcpd/udp/main.c
+++ b/src/ipcpd/udp/main.c
@@ -592,10 +592,8 @@ static int udp_ipcp_bootstrap(const struct ipcp_config * conf)
 
         assert(conf);
         assert(conf->type == THIS_TYPE);
-        ((struct ipcp_config *) conf)->layer_info.dir_hash_algo =
-                (enum pol_dir_hash) HASH_MD5;
 
-        ipcpi.dir_hash_algo = (enum hash_algo) conf->layer_info.dir_hash_algo;
+        ipcpi.dir_hash_algo = HASH_MD5;
         strcpy(ipcpi.layer_name, conf->layer_info.name);
 
         if (inet4_ntop(&conf->udp.ip_addr, ipstr) == NULL) {
@@ -1080,7 +1078,7 @@ static int udp_ipcp_flow_dealloc(int fd)
 
         pthread_rwlock_unlock(&udp_data.flows_lock);
 
-        flow_dealloc(fd);
+        ipcp_flow_dealloc(fd);
 
         return 0;
 }
diff --git a/src/ipcpd/unicast/ca/mb-ecn.c b/src/ipcpd/unicast/ca/mb-ecn.c
index ed4bab12..d9a204b0 100644
--- a/src/ipcpd/unicast/ca/mb-ecn.c
+++ b/src/ipcpd/unicast/ca/mb-ecn.c
@@ -29,7 +29,7 @@
 #include "config.h"
 
 #include <ouroboros/ipcp-dev.h>
-#include <ouroboros/time_utils.h>
+#include <ouroboros/time.h>
 
 #include "mb-ecn.h"
 
@@ -187,7 +187,7 @@ ca_wnd_t mb_ecn_ctx_update_snd(void * _ctx,
 void mb_ecn_wnd_wait(ca_wnd_t wnd)
 {
         if (wnd.wait > 0) {
-                struct timespec s = {0, 0};
+                struct timespec s = TIMESPEC_INIT_S(0);
                 if (wnd.wait > BILLION) /* Don't care throttling < 1s */
                         s.tv_sec = 1;
                 else
diff --git a/src/ipcpd/unicast/dir/dht.c b/src/ipcpd/unicast/dir/dht.c
index 65dad5a3..08a5a5a9 100644
--- a/src/ipcpd/unicast/dir/dht.c
+++ b/src/ipcpd/unicast/dir/dht.c
@@ -40,7 +40,7 @@
 #include <ouroboros/list.h>
 #include <ouroboros/notifier.h>
 #include <ouroboros/random.h>
-#include <ouroboros/time_utils.h>
+#include <ouroboros/time.h>
 #include <ouroboros/tpm.h>
 #include <ouroboros/utils.h>
 #include <ouroboros/pthread.h>
@@ -472,12 +472,14 @@ static void kad_req_destroy(struct kad_req * req)
 static int kad_req_wait(struct kad_req * req,
                         time_t           t)
 {
-        struct timespec timeo = {t, 0};
+        struct timespec timeo = TIMESPEC_INIT_S(0);
         struct timespec abs;
         int ret = 0;
 
         assert(req);
 
+        timeo.tv_sec = t;
+
         clock_gettime(PTHREAD_COND_CLOCK, &abs);
 
         ts_add(&abs, &timeo, &abs);
@@ -995,7 +997,7 @@ static void cancel_lookup_wait(void * o)
 
 static enum lookup_state lookup_wait(struct lookup * lu)
 {
-        struct timespec   timeo = {KAD_T_RESP, 0};
+        struct timespec   timeo = TIMESPEC_INIT_S(KAD_T_RESP);
         struct timespec   abs;
         enum lookup_state state;
         int               ret = 0;
@@ -2764,7 +2766,7 @@ static void handle_event(void *       self,
                 pthread_t          thr;
                 struct join_info * inf;
                 struct conn *      c     = (struct conn *) o;
-                struct timespec    slack = {0, DHT_ENROLL_SLACK * MILLION};
+                struct timespec    slack = TIMESPEC_INIT_MS(DHT_ENROLL_SLACK);
 
                 /* Give the pff some time to update for the new link. */
                 nanosleep(&slack, NULL);
diff --git a/src/ipcpd/unicast/fa.c b/src/ipcpd/unicast/fa.c
index c9f60587..cea9483e 100644
--- a/src/ipcpd/unicast/fa.c
+++ b/src/ipcpd/unicast/fa.c
@@ -878,7 +878,7 @@ int fa_dealloc(int fd)
 
         pthread_rwlock_unlock(&fa.flows_lock);
 
-        flow_dealloc(fd);
+        ipcp_flow_dealloc(fd);
 
         return 0;
 }
diff --git a/src/ipcpd/unicast/main.c b/src/ipcpd/unicast/main.c
index fed08d93..e6cb2994 100644
--- a/src/ipcpd/unicast/main.c
+++ b/src/ipcpd/unicast/main.c
@@ -37,7 +37,7 @@
 #include <ouroboros/notifier.h>
 #include <ouroboros/random.h>
 #include <ouroboros/rib.h>
-#include <ouroboros/time_utils.h>
+#include <ouroboros/time.h>
 
 #include "common/connmgr.h"
 #include "common/enroll.h"