ipcpd: Revised locking

The state lock was reverted to an rwlock to avoid interference of
management functions with the fast path. IPCPs now close without
calling unsafe functions in the signal handler.

9 files changed, 390 insertions, 460 deletions

diff --git a/include/ouroboros/config.h.in b/include/ouroboros/config.h.in
index 84d56e85..3a246235 100644
--- a/include/ouroboros/config.h.in
+++ b/include/ouroboros/config.h.in
@@ -52,6 +52,7 @@
 #define SHM_DU_TIMEOUT_MICROS  15000
 #define IRMD_ACCEPT_TIMEOUT    100
 #define IRMD_FLOW_TIMEOUT      5000
+#define IPCP_ACCEPT_TIMEOUT    100
 #define SOCKET_TIMEOUT         4000
 #define CDAP_REPLY_TIMEOUT     1000
 #define ENROLL_TIMEOUT         2000
diff --git a/src/ipcpd/ipcp.c b/src/ipcpd/ipcp.c
index 1c4fab94..87bf9e51 100644
--- a/src/ipcpd/ipcp.c
+++ b/src/ipcpd/ipcp.c
@@ -22,6 +22,7 @@
 
 #include <ouroboros/config.h>
 #include <ouroboros/ipcp.h>
+#include <ouroboros/time_utils.h>
 
 #define OUROBOROS_PREFIX "ipcpd/ipcp"
 #include <ouroboros/logs.h>
@@ -44,7 +45,8 @@ struct ipcp * ipcp_instance_create()
         i->irmd_fd = -1;
         i->state   = IPCP_INIT;
 
-        pthread_mutex_init(&i->state_lock, NULL);
+        pthread_rwlock_init(&i->state_lock, NULL);
+        pthread_mutex_init(&i->state_mtx, NULL);
         pthread_condattr_init(&cattr);
         pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK);
         pthread_cond_init(&i->state_cond, &cattr);
@@ -52,14 +54,65 @@ struct ipcp * ipcp_instance_create()
         return i;
 }
 
-void ipcp_state_change(struct ipcp * ipcp,
-                       enum ipcp_state state)
+void ipcp_set_state(struct ipcp * ipcp,
+                    enum ipcp_state state)
 {
         if (ipcp == NULL)
             return;
 
+        pthread_mutex_lock(&ipcp->state_mtx);
+
         ipcp->state = state;
+
         pthread_cond_broadcast(&ipcp->state_cond);
+        pthread_mutex_unlock(&ipcp->state_mtx);
+}
+
+enum ipcp_state ipcp_get_state(struct ipcp * ipcp)
+{
+        enum ipcp_state state;
+
+        if (ipcp == NULL)
+                return IPCP_NULL;
+
+        pthread_mutex_lock(&ipcp->state_mtx);
+
+        state = ipcp->state;
+
+        pthread_mutex_unlock(&ipcp->state_mtx);
+
+        return state;
+}
+
+int ipcp_wait_state(struct ipcp *           ipcp,
+                    enum ipcp_state         state,
+                    const struct timespec * timeout)
+{
+        struct timespec abstime;
+
+        clock_gettime(PTHREAD_COND_CLOCK, &abstime);
+        ts_add(&abstime, timeout, &abstime);
+
+        pthread_mutex_lock(&ipcp->state_mtx);
+
+        while (ipcp->state != state && ipcp->state != IPCP_SHUTDOWN) {
+                int ret;
+                if (timeout == NULL)
+                        ret = pthread_cond_wait(&ipcp->state_cond,
+                                                &ipcp->state_mtx);
+                else
+                        ret = pthread_cond_timedwait(&ipcp->state_cond,
+                                                     &ipcp->state_mtx,
+                                                     &abstime);
+                if (ret) {
+                        pthread_mutex_unlock(&ipcp->state_mtx);
+                        return -ret;
+                }
+        }
+
+        pthread_mutex_unlock(&ipcp->state_mtx);
+
+        return 0;
 }
 
 int ipcp_parse_arg(int argc, char * argv[])
@@ -101,22 +154,6 @@ int ipcp_parse_arg(int argc, char * argv[])
         return 0;
 }
 
-static void close_ptr(void * o)
-{
-        char * name = ipcp_sock_path(getpid());
-        close(*((int *) o));
-
-        if (unlink(name))
-                LOG_DBG("Could not unlink %s.", name);
-
-        free(name);
-}
-
-static void clean_msg(void * msg)
-{
-         ipcp_msg__free_unpacked(msg, NULL);
-}
-
 void * ipcp_main_loop(void * o)
 {
         int     lsockfd;
@@ -135,9 +172,13 @@ void * ipcp_main_loop(void * o)
         char * sock_path;
         char * msg_name_dup;
 
-        struct timeval tv = {(SOCKET_TIMEOUT / 1000),
+        struct timeval tv = {(IPCP_ACCEPT_TIMEOUT / 1000),
+                             (IPCP_ACCEPT_TIMEOUT % 1000) * 1000};
+
+        struct timeval ltv = {(SOCKET_TIMEOUT / 1000),
                              (SOCKET_TIMEOUT % 1000) * 1000};
 
+
         if (_ipcp == NULL) {
                 LOG_ERR("Invalid ipcp struct.");
                 return (void *) 1;
@@ -150,28 +191,33 @@ void * ipcp_main_loop(void * o)
         sockfd = server_socket_open(sock_path);
         if (sockfd < 0) {
                 LOG_ERR("Could not open server socket.");
+                free(sock_path);
                 return (void *) 1;
         }
 
-        free(sock_path);
-
-        pthread_cleanup_push(close_ptr, (void *) &sockfd);
+        if (setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO,
+                       (void *) &tv, sizeof(tv)))
+                LOG_WARN("Failed to set timeout on socket.");
 
         while (true) {
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
+                if (ipcp_get_state(_ipcp) == IPCP_SHUTDOWN) {
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        break;
+                }
+
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+
                 ret_msg.code = IPCP_MSG_CODE__IPCP_REPLY;
 
                 lsockfd = accept(sockfd, 0, 0);
-                if (lsockfd < 0) {
-                        LOG_ERR("Cannot accept new connection");
-                        break;
-                }
+                if (lsockfd < 0)
+                        continue;
 
                 if (setsockopt(lsockfd, SOL_SOCKET, SO_RCVTIMEO,
-                               (void *) &tv, sizeof(tv)))
+                               (void *) &ltv, sizeof(ltv)))
                         LOG_WARN("Failed to set timeout on socket.");
 
-                pthread_cleanup_push(close_ptr, (void *) &lsockfd);
-
                 count = read(lsockfd, buf, IPCP_MSG_BUF_SIZE);
                 if (count <= 0) {
                         LOG_ERR("Failed to read from socket");
@@ -185,8 +231,6 @@ void * ipcp_main_loop(void * o)
                         continue;
                 }
 
-                pthread_cleanup_push(clean_msg, (void *) msg);
-
                 switch (msg->code) {
                 case IPCP_MSG_CODE__IPCP_BOOTSTRAP:
                         if (_ipcp->ops->ipcp_bootstrap == NULL) {
@@ -201,7 +245,6 @@ void * ipcp_main_loop(void * o)
                                 ret_msg.result = -1;
                                 break;
                         }
-
                         if (conf_msg->ipcp_type == IPCP_NORMAL) {
                                 conf.addr_size = conf_msg->addr_size;
                                 conf.cep_id_size = conf_msg->cep_id_size;
@@ -218,7 +261,6 @@ void * ipcp_main_loop(void * o)
                                 conf.ip_addr  = conf_msg->ip_addr;
                                 conf.dns_addr = conf_msg->dns_addr;
                         }
-
                         if (conf_msg->ipcp_type == IPCP_SHIM_ETH_LLC)
                                 conf.if_name = conf_msg->if_name;
 
@@ -295,8 +337,7 @@ void * ipcp_main_loop(void * o)
                         break;
                 }
 
-                pthread_cleanup_pop(true);
-                pthread_cleanup_pop(false);
+                ipcp_msg__free_unpacked(msg, NULL);
 
                 buffer.len = ipcp_msg__get_packed_size(&ret_msg);
                 if (buffer.len == 0) {
@@ -321,10 +362,13 @@ void * ipcp_main_loop(void * o)
 
                 free(buffer.data);
                 close(lsockfd);
-
         }
 
-        pthread_cleanup_pop(true);
+        close(sockfd);
+        if (unlink(sock_path))
+                LOG_DBG("Could not unlink %s.", sock_path);
+
+        free(sock_path);
 
-        return NULL;
+        return (void *) 0;
 }
diff --git a/src/ipcpd/ipcp.h b/src/ipcpd/ipcp.h
index 8b7d4ec9..edaea0fd 100644
--- a/src/ipcpd/ipcp.h
+++ b/src/ipcpd/ipcp.h
@@ -27,12 +27,14 @@
 #include <ouroboros/shared.h>
 
 #include <pthread.h>
+#include <time.h>
 
 #include "ipcp-ops.h"
 #include "ipcp-data.h"
 
 enum ipcp_state {
-        IPCP_INIT = 0,
+        IPCP_NULL = 0,
+        IPCP_INIT,
         IPCP_PENDING_ENROLL,
         IPCP_ENROLLED,
         IPCP_DISCONNECTED,
@@ -45,15 +47,27 @@ struct ipcp {
         int                irmd_fd;
 
         enum ipcp_state    state;
-        pthread_mutex_t    state_lock;
+        pthread_rwlock_t   state_lock;
+        pthread_mutex_t    state_mtx;
         pthread_cond_t     state_cond;
 };
 
-struct ipcp * ipcp_instance_create();
-void          ipcp_state_change(struct ipcp * ipcp,
-                                enum ipcp_state state);
-void *        ipcp_main_loop(void * o);
-void *        ipcp_sdu_loop(void * o);
-int           ipcp_parse_arg(int argc, char * argv[]);
+struct ipcp *   ipcp_instance_create();
+
+void            ipcp_set_state(struct ipcp *   ipcp,
+                               enum ipcp_state state);
+
+enum ipcp_state ipcp_get_state(struct ipcp * ipcp);
+
+int             ipcp_wait_state(struct ipcp *           ipcp,
+                                enum ipcp_state         state,
+                                const struct timespec * timeout);
+
+void *          ipcp_main_loop(void * o);
+
+void *          ipcp_sdu_loop(void * o);
+
+int             ipcp_parse_arg(int argc,
+                               char * argv[]);
 
 #endif
diff --git a/src/ipcpd/local/main.c b/src/ipcpd/local/main.c
index 72bf0e4b..6993cea2 100644
--- a/src/ipcpd/local/main.c
+++ b/src/ipcpd/local/main.c
@@ -33,7 +33,6 @@
 #include <ouroboros/bitmap.h>
 #include <ouroboros/shared.h>
 #include <ouroboros/dev.h>
-
 #define OUROBOROS_PREFIX "ipcpd/local"
 
 #include <ouroboros/logs.h>
@@ -133,7 +132,7 @@ void shim_ap_fini()
         if (_ap_instance == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up AP while not in shutdown.");
@@ -152,7 +151,7 @@ void shim_ap_fini()
                         shm_ap_rbuff_close(_ap_instance->flows[i].rb);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         free(_ap_instance);
 }
@@ -187,10 +186,10 @@ static void * ipcp_local_sdu_loop(void * o)
                         continue;
                 }
 
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         return (void *) 1; /* -ENOTENROLLED */
                 }
 
@@ -198,7 +197,7 @@ static void * ipcp_local_sdu_loop(void * o)
                 fd = _ap_instance->in_out[port_id_to_fd(e->port_id)];
                 if (fd == -1) {
                         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(e);
                         continue;
                 }
@@ -209,7 +208,7 @@ static void * ipcp_local_sdu_loop(void * o)
                         ;
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 free(e);
         }
@@ -229,25 +228,14 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
         case SIGHUP:
         case SIGQUIT:
                 if (info->si_pid == irmd_api) {
-                        bool clean_threads = false;
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
-
-                        if (_ipcp->state == IPCP_ENROLLED)
-                                clean_threads = true;
-
-                        _ipcp->state = IPCP_SHUTDOWN;
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
-                        if (clean_threads) {
-                                pthread_cancel(_ap_instance->sduloop);
-                                pthread_join(_ap_instance->sduloop, NULL);
-                        }
-
-                        pthread_cancel(_ap_instance->mainloop);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                 }
         default:
                 return;
@@ -261,22 +249,22 @@ static int ipcp_local_bootstrap(struct dif_config * conf)
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("IPCP in wrong state.");
                 return -1;
         }
 
-        _ipcp->state = IPCP_ENROLLED;
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         pthread_create(&_ap_instance->sduloop,
                        NULL,
                        ipcp_local_sdu_loop,
                        NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped local IPCP with api %d.",
                 getpid());
@@ -286,21 +274,15 @@ static int ipcp_local_bootstrap(struct dif_config * conf)
 
 static int ipcp_local_name_reg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBGF("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Failed to add %s to local registry.", name);
                 return -1;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Registered %s.", name);
 
@@ -309,11 +291,11 @@ static int ipcp_local_name_reg(char * name)
 
 static int ipcp_local_name_unreg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -336,17 +318,17 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
 
         /* This ipcpd has all QoS */
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBGF("Won't allocate flow with non-enrolled IPCP.");
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+                LOG_DBGF("Won't register with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
         }
 
         rb = shm_ap_rbuff_open(n_api);
         if (rb == NULL) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1; /* -ENORBUFF */
         }
 
@@ -355,7 +337,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
         in_fd = bmp_allocate(_ap_instance->fds);
         if (!bmp_is_id_valid(_ap_instance->fds, in_fd)) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -EMFILE;
         }
 
@@ -372,7 +354,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
 
         if (port_id < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Could not get port id from IRMd");
                 /* shm_ap_rbuff_close(n_api); */
                 return -1;
@@ -382,7 +364,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
         if (!bmp_is_id_valid(_ap_instance->fds, out_fd)) {
                 /* shm_ap_rbuff_close(n_api); */
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1; /* -ENOMOREFDS */
         }
 
@@ -394,7 +376,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
         _ap_instance->in_out[out_fd] = in_fd;
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBGF("Pending local allocation request, port_id %d.", port_id);
 
@@ -413,7 +395,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         if (response)
                 return 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         /* awaken pending flow */
 
@@ -422,14 +404,14 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         in_fd = port_id_to_fd(port_id);
         if (in_fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Could not find flow with port_id %d.", port_id);
                 return -1;
         }
 
         if (_ap_instance->flows[in_fd].state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Flow was not pending.");
                 return -1;
         }
@@ -441,7 +423,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
                 _ap_instance->flows[in_fd].port_id = -1;
                 _ap_instance->in_out[in_fd] = -1;
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -453,7 +435,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         out_fd = _ap_instance->in_out[in_fd];
         if (out_fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("No pending local flow with port_id %d.", port_id);
                 return -1;
         }
@@ -461,7 +443,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         if (_ap_instance->flows[out_fd].state != FLOW_PENDING) {
                  /* FIXME: clean up other end */
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Flow was not pending.");
                 return -1;
         }
@@ -469,7 +451,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         _ap_instance->flows[out_fd].state = FLOW_ALLOCATED;
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if ((ret = ipcp_flow_alloc_reply(getpid(),
                                          _ap_instance->flows[out_fd].port_id,
@@ -489,13 +471,13 @@ static int ipcp_local_flow_dealloc(int port_id)
         int fd = -1;
         struct shm_ap_rbuff * rb;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&_ap_instance->flows_lock);
 
         fd = port_id_to_fd(port_id);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Could not find flow with port_id %d.", port_id);
                 return 0;
         }
@@ -517,7 +499,7 @@ static int ipcp_local_flow_dealloc(int port_id)
         if (rb != NULL)
                 shm_ap_rbuff_close(rb);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBGF("Flow with port_id %d deallocated.", port_id);
 
@@ -611,16 +593,12 @@ int main(int argc, char * argv[])
                 exit(EXIT_FAILURE);
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&_ap_instance->mainloop, NULL, ipcp_main_loop, _ipcp);
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 close_logfile();
@@ -629,6 +607,9 @@ int main(int argc, char * argv[])
 
         pthread_join(_ap_instance->mainloop, NULL);
 
+        pthread_cancel(_ap_instance->sduloop);
+        pthread_join(_ap_instance->sduloop, NULL);
+
         shim_ap_fini();
 
         ipcp_data_destroy(_ipcp->data);
diff --git a/src/ipcpd/normal/fmgr.c b/src/ipcpd/normal/fmgr.c
index 70afff37..1c8330a8 100644
--- a/src/ipcpd/normal/fmgr.c
+++ b/src/ipcpd/normal/fmgr.c
@@ -81,17 +81,16 @@ static void * fmgr_listen(void * o)
         char * ae_name;
 
         while (true) {
-                pthread_mutex_lock(&_ipcp->state_lock);
-                while (!(_ipcp->state == IPCP_ENROLLED ||
-                         _ipcp->state == IPCP_SHUTDOWN))
-                        pthread_cond_wait(&_ipcp->state_cond,
-                                          &_ipcp->state_lock);
-
-                if (_ipcp->state == IPCP_SHUTDOWN) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                ipcp_wait_state(_ipcp, IPCP_ENROLLED, NULL);
+
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
+
+                if (ipcp_get_state(_ipcp) == IPCP_SHUTDOWN) {
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         return 0;
                 }
-                pthread_mutex_unlock(&_ipcp->state_lock);
+
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 fd = flow_accept(&ae_name);
                 if (fd < 0) {
diff --git a/src/ipcpd/normal/main.c b/src/ipcpd/normal/main.c
index 4173246d..9479b806 100644
--- a/src/ipcpd/normal/main.c
+++ b/src/ipcpd/normal/main.c
@@ -75,13 +75,11 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        ipcp_state_change(_ipcp, IPCP_SHUTDOWN);
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-
-                        pthread_cancel(normal_data(_ipcp)->mainloop);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
 
                         if (fmgr_fini())
                                 LOG_ERR("Failed to finalize flow manager.");
@@ -99,21 +97,15 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
 
 static int normal_ipcp_name_reg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_ERR("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to add %s to local registry.", name);
                 return -1;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Registered %s.", name);
 
@@ -122,11 +114,11 @@ static int normal_ipcp_name_reg(char * name)
 
 static int normal_ipcp_name_unreg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -135,62 +127,59 @@ static int normal_ipcp_enroll(char * dif_name)
 {
         struct timespec timeout = {(ENROLL_TIMEOUT / 1000),
                                    (ENROLL_TIMEOUT % 1000) * MILLION};
-        struct timespec abstime;
-
-        clock_gettime(PTHREAD_COND_CLOCK, &abstime);
-        ts_add(&abstime, &timeout, &abstime);
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Won't enroll an IPCP that is not in INIT.");
                 return -1; /* -ENOTINIT */
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (fmgr_mgmt_flow(dif_name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to establish management flow.");
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        while (_ipcp->state != IPCP_ENROLLED)
-                if (pthread_cond_timedwait(&_ipcp->state_cond,
-                                           &_ipcp->state_lock,
-                                           &abstime) == ETIMEDOUT) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        LOG_ERR("Enrollment didn't complete in time.");
-                        return -1;
-                }
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
+
+        if (ipcp_wait_state(_ipcp, IPCP_ENROLLED, &timeout) == -ETIMEDOUT) {
+                LOG_ERR("Enrollment timed out.");
+                return -1;
+        }
+
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
+
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+                return -1;
+        }
 
         return 0;
 }
 
 static int normal_ipcp_bootstrap(struct dif_config * conf)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Won't bootstrap an IPCP that is not in INIT.");
                 return -1; /* -ENOTINIT */
         }
 
         if (ribmgr_bootstrap(conf)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to bootstrap RIB manager.");
                 return -1;
         }
 
-        ipcp_state_change(_ipcp, IPCP_ENROLLED);
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         _ipcp->data->dif_name = conf->dif_name;
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped in DIF %s.", conf->dif_name);
 
@@ -247,9 +236,9 @@ void normal_ipcp_data_destroy()
         if (_ipcp == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_SHUTDOWN)
+        if (ipcp_get_state(_ipcp) != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up while not in shutdown.");
 
         if (normal_data(_ipcp)->dum != NULL)
@@ -257,7 +246,7 @@ void normal_ipcp_data_destroy()
         if (normal_data(_ipcp)->rb != NULL)
                 shm_ap_rbuff_close(normal_data(_ipcp)->rb);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_data_destroy(_ipcp->data);
 }
@@ -331,8 +320,6 @@ int main(int argc, char * argv[])
                 exit(EXIT_FAILURE);
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&normal_data(_ipcp)->mainloop, NULL,
@@ -340,8 +327,6 @@ int main(int argc, char * argv[])
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 normal_ipcp_data_destroy();
diff --git a/src/ipcpd/normal/ribmgr.c b/src/ipcpd/normal/ribmgr.c
index c8d517b5..9b61e180 100644
--- a/src/ipcpd/normal/ribmgr.c
+++ b/src/ipcpd/normal/ribmgr.c
@@ -254,15 +254,15 @@ int ribmgr_cdap_write(struct cdap * instance,
         static_info_msg_t * msg;
         int ret = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        if (_ipcp->state == IPCP_PENDING_ENROLL &&
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) == IPCP_PENDING_ENROLL &&
             strcmp(name, STATIC_INFO) == 0) {
                 LOG_DBG("Received static DIF information.");
 
                 msg = static_info_msg__unpack(NULL, len, data);
                 if (msg == NULL) {
-                        ipcp_state_change(_ipcp, IPCP_INIT);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        ipcp_set_state(_ipcp, IPCP_INIT);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         cdap_send_reply(instance, invoke_id, -1, NULL, 0);
                         LOG_ERR("Failed to unpack static info message.");
                         return -1;
@@ -280,8 +280,8 @@ int ribmgr_cdap_write(struct cdap * instance,
                 rib->address = msg->address;
 
                 if (frct_init(&rib->dtc, rib->address)) {
-                        ipcp_state_change(_ipcp, IPCP_INIT);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        ipcp_set_state(_ipcp, IPCP_INIT);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         cdap_send_reply(instance, invoke_id, -1, NULL, 0);
                         static_info_msg__free_unpacked(msg, NULL);
                         LOG_ERR("Failed to init FRCT");
@@ -289,9 +289,11 @@ int ribmgr_cdap_write(struct cdap * instance,
                 }
 
                 static_info_msg__free_unpacked(msg, NULL);
-        } else
+        } else {
                 ret = -1;
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        }
+
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (cdap_send_reply(instance, invoke_id, ret, NULL, 0)) {
                 LOG_ERR("Failed to send reply to write request.");
@@ -332,13 +334,13 @@ int ribmgr_cdap_start(struct cdap * instance,
         size_t len = 0;
         int iid = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        if (_ipcp->state == IPCP_ENROLLED &&
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) == IPCP_ENROLLED &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("New enrollment request.");
 
                 if (cdap_send_reply(instance, invoke_id, 0, NULL, 0)) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to send reply to enrollment request.");
                         return -1;
                 }
@@ -357,14 +359,14 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 len = static_info_msg__get_packed_size(&stat_info);
                 if (len == 0) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to get size of static information.");
                         return -1;
                 }
 
                 data = malloc(len);
                 if (data == NULL) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to allocate memory.");
                         return -1;
                 }
@@ -378,7 +380,7 @@ int ribmgr_cdap_start(struct cdap * instance,
                 iid = cdap_send_write(instance, STATIC_INFO, data, len, 0);
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to send static information.");
                         return -1;
@@ -386,7 +388,7 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 if (cdap_result_wait(instance, WRITE, STATIC_INFO, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Remote did not receive static information.");
                         return -1;
@@ -402,7 +404,7 @@ int ribmgr_cdap_start(struct cdap * instance,
                 iid = cdap_send_stop(instance, ENROLLMENT);
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to send stop of enrollment.");
                         return -1;
@@ -410,7 +412,7 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 if (cdap_result_wait(instance, STOP, ENROLLMENT, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Remote failed to complete enrollment.");
                         return -1;
@@ -420,12 +422,12 @@ int ribmgr_cdap_start(struct cdap * instance,
                 free(data);
         } else {
                 if (cdap_send_reply(instance, invoke_id, -1, NULL, 0)) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to send reply to start request.");
                         return -1;
                 }
         }
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -436,21 +438,21 @@ int ribmgr_cdap_stop(struct cdap * instance,
 {
         int ret = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        if (_ipcp->state == IPCP_PENDING_ENROLL &&
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) == IPCP_PENDING_ENROLL &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("Stop enrollment received.");
 
-                ipcp_state_change(_ipcp, IPCP_ENROLLED);
+                ipcp_set_state(_ipcp, IPCP_ENROLLED);
         } else
                 ret = -1;
 
         if (cdap_send_reply(instance, invoke_id, ret, NULL, 0)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to send reply to stop request.");
                 return -1;
         }
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -486,11 +488,11 @@ int ribmgr_add_flow(int fd)
         flow->instance = instance;
         flow->fd = fd;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&rib->flows_lock);
         if (list_empty(&rib->flows) &&
-            _ipcp->state == IPCP_INIT) {
-                ipcp_state_change(_ipcp, IPCP_PENDING_ENROLL);
+            ipcp_get_state(_ipcp) == IPCP_INIT) {
+                ipcp_set_state(_ipcp, IPCP_PENDING_ENROLL);
 
                 pthread_mutex_lock(&rib->cdap_reqs_lock);
                 iid = cdap_send_start(instance,
@@ -498,6 +500,7 @@ int ribmgr_add_flow(int fd)
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
                         pthread_rwlock_unlock(&rib->flows_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to start enrollment.");
                         cdap_destroy(instance);
                         free(flow);
@@ -507,6 +510,7 @@ int ribmgr_add_flow(int fd)
                 if (cdap_result_wait(instance, START, ENROLLMENT, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
                         pthread_rwlock_unlock(&rib->flows_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to start enrollment.");
                         cdap_destroy(instance);
                         free(flow);
@@ -514,10 +518,10 @@ int ribmgr_add_flow(int fd)
                 }
                 pthread_mutex_unlock(&rib->cdap_reqs_lock);
         }
-        pthread_mutex_unlock(&_ipcp->state_lock);
 
         list_add(&flow->next, &rib->flows);
         pthread_rwlock_unlock(&rib->flows_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -534,6 +538,7 @@ int ribmgr_remove_flow(int fd)
                         if (cdap_destroy(flow->instance))
                                 LOG_ERR("Failed to destroy CDAP instance.");
                         list_del(&flow->next);
+                        pthread_rwlock_unlock(&rib->flows_lock);
                         free(flow);
                         return 0;
                 }
diff --git a/src/ipcpd/shim-eth-llc/main.c b/src/ipcpd/shim-eth-llc/main.c
index e4d7429a..60b55009 100644
--- a/src/ipcpd/shim-eth-llc/main.c
+++ b/src/ipcpd/shim-eth-llc/main.c
@@ -197,9 +197,9 @@ void eth_llc_ipcp_data_destroy()
         if (_ipcp == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_SHUTDOWN)
+        if (ipcp_get_state(_ipcp) != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up while not in shutdown.");
 
         if (shim_data(_ipcp)->dum != NULL)
@@ -218,7 +218,7 @@ void eth_llc_ipcp_data_destroy()
                         shm_ap_rbuff_close(ipcp_flow(i)->rb);
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_data_destroy(_ipcp->data);
 }
@@ -478,12 +478,12 @@ static int eth_llc_ipcp_port_req(uint8_t r_sap,
         ssize_t index = 0;
         int i;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         index = bmp_allocate(shim_data(_ipcp)->indices);
         if (index < 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
                 LOG_ERR("Out of free indices.");
                 return -1;
@@ -496,7 +496,7 @@ static int eth_llc_ipcp_port_req(uint8_t r_sap,
 
         if (port_id < 0) {
                 bmp_release(shim_data(_ipcp)->indices, index);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
                 LOG_ERR("Could not get port id from IRMd.");
                 return -1;
@@ -511,7 +511,7 @@ static int eth_llc_ipcp_port_req(uint8_t r_sap,
         }
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("New flow request, port_id %d, remote SAP %d.", port_id, r_sap);
 
@@ -528,20 +528,20 @@ static int eth_llc_ipcp_port_alloc_reply(uint8_t ssap,
         int port_id = -1;
         int i;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&shim_data(_ipcp)->flows_lock);
 
         index = sap_to_index(ssap);
         if (index < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("No flow found with that SAP.");
                 return -1; /* -EFLOWNOTFOUND */
         }
 
         if (ipcp_flow(index)->state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1; /* -EFLOWNOTPENDING */
         }
 
@@ -558,7 +558,7 @@ static int eth_llc_ipcp_port_alloc_reply(uint8_t ssap,
         }
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Flow reply, port_id %d, remote SAP %d.", port_id, dsap);
 
@@ -578,13 +578,13 @@ static int eth_llc_ipcp_flow_dealloc_req(uint8_t ssap,
         int port_id = -1;
         int i = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         i = sap_to_index(ssap);
         if (i < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("No flow found for remote deallocation request.");
                 return 0;
         }
@@ -593,7 +593,7 @@ static int eth_llc_ipcp_flow_dealloc_req(uint8_t ssap,
         destroy_ipcp_flow(i);
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_flow_dealloc(0, port_id);
 
@@ -669,15 +669,6 @@ static void * eth_llc_ipcp_sdu_reader(void * o)
         memset(br_addr, 0xff, MAC_SIZE * sizeof(uint8_t));
 
         while (true) {
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
-
-                pthread_mutex_unlock(&_ipcp->state_lock);
-
 #if defined(PACKET_RX_RING) && defined(PACKET_TX_RING)
                 header = (void *) shim_data(_ipcp)->rx_ring +
                         (offset * SHM_DU_BUFF_BLOCK_SIZE);
@@ -789,13 +780,7 @@ static void * eth_llc_ipcp_sdu_writer(void * o)
                 if (e == NULL)
                         continue;
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 len = shm_du_map_read((uint8_t **) &buf,
                                       shim_data(_ipcp)->dum,
@@ -826,6 +811,8 @@ static void * eth_llc_ipcp_sdu_writer(void * o)
                 if (shim_data(_ipcp)->dum != NULL)
                         shm_du_map_remove(shim_data(_ipcp)->dum, e->index);
 
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+
                 free(e);
         }
 
@@ -843,30 +830,14 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
         case SIGTERM:
         case SIGHUP:
                 if (info->si_pid == irmd_api) {
-                        bool clean_threads = false;
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
-
-                        if (_ipcp->state == IPCP_ENROLLED)
-                                clean_threads = true;
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        _ipcp->state = IPCP_SHUTDOWN;
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-
-                        if (clean_threads) {
-                                pthread_cancel(shim_data(_ipcp)->sdu_reader);
-                                pthread_cancel(shim_data(_ipcp)->sdu_writer);
-
-                                pthread_join(shim_data(_ipcp)->sdu_writer,
-                                             NULL);
-                                pthread_join(shim_data(_ipcp)->sdu_reader,
-                                             NULL);
-                        }
-
-                        pthread_cancel(shim_data(_ipcp)->mainloop);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 }
         default:
@@ -1030,10 +1001,10 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
 
 #endif
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("IPCP in wrong state.");
                 close(fd);
                 return -1;
@@ -1043,7 +1014,7 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
         shim_data(_ipcp)->device = device;
         shim_data(_ipcp)->tx_offset = 0;
 
-        _ipcp->state = IPCP_ENROLLED;
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         pthread_create(&shim_data(_ipcp)->sdu_reader,
                        NULL,
@@ -1055,7 +1026,7 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
                        eth_llc_ipcp_sdu_writer,
                        NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped shim IPCP over Ethernet with LLC with api %d.",
                 getpid());
@@ -1065,21 +1036,15 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
 
 static int eth_llc_ipcp_name_reg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBGF("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to add %s to local registry.", name);
                 return -1;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Registered %s.", name);
 
@@ -1088,11 +1053,11 @@ static int eth_llc_ipcp_name_reg(char * name)
 
 static int eth_llc_ipcp_name_unreg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -1120,19 +1085,19 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
         if (rb == NULL)
                 return -1; /* -ENORBUFF */
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_ENROLLED) {
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 shm_ap_rbuff_close(rb);
-                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Won't allocate flow with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
         }
 
         index = bmp_allocate(shim_data(_ipcp)->indices);
         if (index < 0) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 shm_ap_rbuff_close(rb);
-                pthread_mutex_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1143,7 +1108,7 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
                 shm_ap_rbuff_close(rb);
                 bmp_release(shim_data(_ipcp)->indices, index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1153,7 +1118,7 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
         shim_data(_ipcp)->flows[index].sap = ssap;
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         memset(r_addr, 0xff, MAC_SIZE * sizeof(uint8_t));
 
@@ -1161,11 +1126,11 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
                                     dst_name,
                                     src_ae_name) < 0) {
                 LOG_DBGF("Port alloc returned -1.");
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
                 destroy_ipcp_flow(index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1183,20 +1148,20 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
         int index = -1;
         uint8_t ssap = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         index = port_id_to_index(port_id);
         if (index < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Could not find flow with port_id %d.", port_id);
                 return -1;
         }
 
         if (ipcp_flow(index)->state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Flow was not pending.");
                 return -1;
         }
@@ -1208,7 +1173,7 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
                 ipcp_flow(index)->port_id = -1;
                 bmp_release(shim_data(_ipcp)->indices, index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1219,7 +1184,7 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
                 shm_ap_rbuff_close(ipcp_flow(index)->rb);
                 bmp_release(shim_data(_ipcp)->indices, index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1228,17 +1193,17 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
         shim_data(_ipcp)->flows[index].sap = ssap;
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (eth_llc_ipcp_port_alloc_resp(shim_data(_ipcp)->flows[index].r_addr,
                                          shim_data(_ipcp)->flows[index].r_sap,
                                          ssap,
                                          response) < 0) {
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
                 destroy_ipcp_flow(index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 LOG_DBGF("Could not send response.");
                 return -1;
@@ -1257,13 +1222,13 @@ static int eth_llc_ipcp_flow_dealloc(int port_id)
         int i;
         int ret;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         index = port_id_to_index(port_id);
         if (index < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return 0;
         }
 
@@ -1277,7 +1242,7 @@ static int eth_llc_ipcp_flow_dealloc(int port_id)
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
 
         ret = eth_llc_ipcp_port_dealloc(addr, sap);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (ret < 0)
                 LOG_DBGF("Could not notify remote.");
@@ -1353,8 +1318,6 @@ int main(int argc, char * argv[])
         _ipcp->ops = &eth_llc_ops;
         _ipcp->state = IPCP_INIT;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&shim_data(_ipcp)->mainloop, NULL,
@@ -1362,8 +1325,6 @@ int main(int argc, char * argv[])
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 close_logfile();
@@ -1372,6 +1333,12 @@ int main(int argc, char * argv[])
 
         pthread_join(shim_data(_ipcp)->mainloop, NULL);
 
+        pthread_cancel(shim_data(_ipcp)->sdu_reader);
+        pthread_cancel(shim_data(_ipcp)->sdu_writer);
+
+        pthread_join(shim_data(_ipcp)->sdu_writer, NULL);
+        pthread_join(shim_data(_ipcp)->sdu_reader, NULL);
+
         eth_llc_ipcp_data_destroy();
 
         free(_ipcp);
diff --git a/src/ipcpd/shim-udp/main.c b/src/ipcpd/shim-udp/main.c
index 672941da..292d347d 100644
--- a/src/ipcpd/shim-udp/main.c
+++ b/src/ipcpd/shim-udp/main.c
@@ -99,7 +99,8 @@ struct shim_ap_data {
         pthread_t             handler;
         pthread_t             sdu_reader;
 
-        bool                  fd_set_sync;
+        bool                  fd_set_mod;
+        pthread_cond_t        fd_set_cond;
         pthread_mutex_t       fd_set_lock;
 } * _ap_instance;
 
@@ -142,6 +143,7 @@ static int shim_ap_init()
         }
 
         pthread_rwlock_init(&_ap_instance->flows_lock, NULL);
+        pthread_cond_init(&_ap_instance->fd_set_cond, NULL);
         pthread_mutex_init(&_ap_instance->fd_set_lock, NULL);
 
         return 0;
@@ -154,7 +156,7 @@ void shim_ap_fini()
         if (_ap_instance == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up AP while not in shutdown.");
@@ -173,7 +175,7 @@ void shim_ap_fini()
                         shm_ap_rbuff_close(_ap_instance->flows[i].rb);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         free(_ap_instance);
 }
@@ -197,31 +199,28 @@ static ssize_t ipcp_udp_flow_write(int fd, void * buf, size_t count)
         ssize_t index;
         struct rb_entry e;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                return -1; /* -ENOTENROLLED */
-        }
-
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
-        while ((index = shm_du_map_write(_ap_instance->dum,
-                                         _ap_instance->flows[fd].api,
-                                         0,
-                                         0,
-                                         (uint8_t *) buf,
-                                         count)) < 0)
-                ;
+        index = shm_du_map_write_b(_ap_instance->dum,
+                                   _ap_instance->flows[fd].api,
+                                   0,
+                                   0,
+                                   (uint8_t *) buf,
+                                   count);
+        if (index < 0) {
+                pthread_rwlock_unlock(&_ap_instance->flows_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+                return -1;
+        }
 
         e.index   = index;
         e.port_id = _ap_instance->flows[fd].port_id;
 
-        while (shm_ap_rbuff_write(_ap_instance->flows[fd].rb, &e) < 0)
-                ;
+        shm_ap_rbuff_write(_ap_instance->flows[fd].rb, &e);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -292,40 +291,30 @@ struct ipcp_udp_data * ipcp_udp_data_create()
 
 static void set_fd(int fd)
 {
-        bool fd_wait = true;
-
         pthread_mutex_lock(&_ap_instance->fd_set_lock);
 
-        _ap_instance->fd_set_sync =  true;
+        _ap_instance->fd_set_mod = true;
         FD_SET(fd, &shim_data(_ipcp)->flow_fd_s);
 
-        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
-
+        while (_ap_instance->fd_set_mod)
+                pthread_cond_wait(&_ap_instance->fd_set_cond,
+                                  &_ap_instance->fd_set_lock);
 
-        while (fd_wait) {
-                pthread_mutex_lock(&_ap_instance->fd_set_lock);
-                fd_wait = _ap_instance->fd_set_sync;
-                pthread_mutex_unlock(&_ap_instance->fd_set_lock);
-        }
+        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
 }
 
 static void clr_fd(int fd)
 {
-        bool fd_wait = true;
-
         pthread_mutex_lock(&_ap_instance->fd_set_lock);
 
-        _ap_instance->fd_set_sync =  true;
+        _ap_instance->fd_set_mod = true;
         FD_CLR(fd, &shim_data(_ipcp)->flow_fd_s);
 
-        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
+        while (_ap_instance->fd_set_mod)
+                pthread_cond_wait(&_ap_instance->fd_set_cond,
+                                  &_ap_instance->fd_set_lock);
 
-        while (fd_wait) {
-                sched_yield();
-                pthread_mutex_lock(&_ap_instance->fd_set_lock);
-                fd_wait = _ap_instance->fd_set_sync;
-                pthread_mutex_unlock(&_ap_instance->fd_set_lock);
-        }
+        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
 }
 
 
@@ -463,7 +452,7 @@ static int ipcp_udp_port_req(struct sockaddr_in * c_saddr,
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&_ap_instance->flows_lock);
 
         /* reply to IRM */
@@ -473,7 +462,7 @@ static int ipcp_udp_port_req(struct sockaddr_in * c_saddr,
 
         if (port_id < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Could not get port id from IRMd");
                 close(fd);
                 return -1;
@@ -484,7 +473,7 @@ static int ipcp_udp_port_req(struct sockaddr_in * c_saddr,
         _ap_instance->flows[fd].state   = FLOW_PENDING;
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Pending allocation request, port_id %d, UDP port (%d, %d).",
                 port_id, ntohs(f_saddr.sin_port), ntohs(c_saddr->sin_port));
@@ -506,20 +495,20 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
         LOG_DBG("Received reply for flow on udp port %d.",
                 ntohs(dst_udp_port));
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         fd = udp_port_to_fd(dst_udp_port);
         if (fd == -1) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Unknown flow on UDP port %d.", ntohs(dst_udp_port));
                 return -1; /* -EUNKNOWNFLOW */
         }
 
         if (_ap_instance->flows[fd].state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Flow on UDP port %d not pending.",
                          ntohs(dst_udp_port));
                 return -1; /* -EFLOWNOTPENDING */
@@ -538,7 +527,7 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
                                 (struct sockaddr *) &t_saddr,
                                 &t_saddr_len) < 0) {
                         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_DBG("Flow with port_id %d has no peer.", port_id);
                         return -1;
                 }
@@ -550,7 +539,7 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
                             (struct sockaddr *) &t_saddr,
                             sizeof(t_saddr)) < 0) {
                         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         close(fd);
                         return -1;
                 }
@@ -559,7 +548,7 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
         }
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
 
         if ((ret = ipcp_flow_alloc_reply(getpid(),
@@ -582,13 +571,13 @@ static int ipcp_udp_flow_dealloc_req(int udp_port)
 
         struct shm_ap_rbuff * rb;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         fd = udp_port_to_fd(udp_port);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Could not find flow on UDP port %d.",
                          ntohs(udp_port));
                 return 0;
@@ -610,7 +599,7 @@ static int ipcp_udp_flow_dealloc_req(int udp_port)
         if (rb != NULL)
                 shm_ap_rbuff_close(rb);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_flow_dealloc(0, port_id);
 
@@ -632,16 +621,11 @@ static void * ipcp_udp_listener()
                 int sfd = 0;
                 shim_udp_msg_t * msg = NULL;
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 sfd = shim_data(_ipcp)->s_fd;
 
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 memset(&buf, 0, SHIM_UDP_MSG_SIZE);
                 n = sizeof(c_saddr);
@@ -705,23 +689,17 @@ static void * ipcp_udp_sdu_reader()
         while (true) {
                 struct timeval tv = {0, FD_UPDATE_TIMEOUT};
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
-
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_rdlock(&_ap_instance->flows_lock);
-
                 pthread_mutex_lock(&_ap_instance->fd_set_lock);
 
                 read_fds = shim_data(_ipcp)->flow_fd_s;
-                _ap_instance->fd_set_sync = false;
+                _ap_instance->fd_set_mod = false;
+                pthread_cond_broadcast(&_ap_instance->fd_set_cond);
 
                 pthread_mutex_unlock(&_ap_instance->fd_set_lock);
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 if (select(FD_SETSIZE, &read_fds, NULL, NULL, &tv) <= 0) {
                         continue;
@@ -765,18 +743,13 @@ static void * ipcp_udp_sdu_loop(void * o)
                         continue;
                 }
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 len = shm_du_map_read((uint8_t **) &buf,
                                       _ap_instance->dum,
                                       e->index);
                 if (len <= 0) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(e);
                         continue;
                 }
@@ -786,7 +759,7 @@ static void * ipcp_udp_sdu_loop(void * o)
                 fd = port_id_to_fd(e->port_id);
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 if (fd == -1) {
                         free(e);
@@ -796,12 +769,12 @@ static void * ipcp_udp_sdu_loop(void * o)
                 if (send(fd, buf, len, 0) < 0)
                         LOG_ERR("Failed to send SDU.");
 
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 if (_ap_instance->dum != NULL)
                         shm_du_map_remove(_ap_instance->dum, e->index);
 
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 free(e);
         }
@@ -820,31 +793,14 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
         case SIGTERM:
         case SIGHUP:
                 if (info->si_pid == irmd_api) {
-                        bool clean_threads = false;
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        if (_ipcp->state == IPCP_ENROLLED)
-                                clean_threads = true;
-
-                        _ipcp->state = IPCP_SHUTDOWN;
-
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-
-                        if (clean_threads) {
-                                pthread_cancel(_ap_instance->handler);
-                                pthread_cancel(_ap_instance->sdu_reader);
-                                pthread_cancel(_ap_instance->sduloop);
-
-                                pthread_join(_ap_instance->sduloop, NULL);
-                                pthread_join(_ap_instance->handler, NULL);
-                                pthread_join(_ap_instance->sdu_reader, NULL);
-                        }
-
-                        pthread_cancel(_ap_instance->mainloop);
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                 }
         default:
                 return;
@@ -916,10 +872,10 @@ static int ipcp_udp_bootstrap(struct dif_config * conf)
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("IPCP in wrong state.");
                 close(fd);
                 return -1;
@@ -931,7 +887,7 @@ static int ipcp_udp_bootstrap(struct dif_config * conf)
 
         FD_CLR(shim_data(_ipcp)->s_fd, &shim_data(_ipcp)->flow_fd_s);
 
-        _ipcp->state = IPCP_ENROLLED;
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         pthread_create(&_ap_instance->handler,
                        NULL,
@@ -947,7 +903,7 @@ static int ipcp_udp_bootstrap(struct dif_config * conf)
                        ipcp_udp_sdu_loop,
                        NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped shim IPCP over UDP with api %d.",
                 getpid());
@@ -1098,16 +1054,10 @@ static int ipcp_udp_name_reg(char * name)
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBG("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to add %s to local registry.", name);
                 return -1;
         }
@@ -1117,7 +1067,7 @@ static int ipcp_udp_name_reg(char * name)
 
         dns_addr = shim_data(_ipcp)->dns_addr;
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (dns_addr != 0) {
                 ip_addr = shim_data(_ipcp)->ip_addr;
@@ -1136,14 +1086,14 @@ static int ipcp_udp_name_reg(char * name)
                         dnsstr, name, DNS_TTL, ipstr);
 
                 if (ddns_send(cmd)) {
-                        pthread_mutex_lock(&_ipcp->state_lock);
+                        pthread_rwlock_rdlock(&_ipcp->state_lock);
                         ipcp_data_del_reg_entry(_ipcp->data, name);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         return -1;
                 }
         }
 #else
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 #endif
         LOG_DBG("Registered %s.", name);
 
@@ -1167,17 +1117,11 @@ static int ipcp_udp_name_unreg(char * name)
 #ifdef CONFIG_OUROBOROS_ENABLE_DNS
         /* unregister application with DNS server */
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBG("IPCP is not enrolled");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         dns_addr = shim_data(_ipcp)->dns_addr;
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (dns_addr != 0) {
                 if (inet_ntop(AF_INET, &dns_addr, dnsstr, INET_ADDRSTRLEN)
@@ -1191,11 +1135,11 @@ static int ipcp_udp_name_unreg(char * name)
         }
 #endif
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -1253,10 +1197,10 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Won't allocate flow with non-enrolled IPCP.");
                 close(fd);
                 return -1; /* -ENOTENROLLED */
@@ -1266,7 +1210,7 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
         dns_addr = shim_data(_ipcp)->dns_addr;
 
         if (dns_addr != 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 ip_addr = ddns_resolve(dst_name, dns_addr);
                 if (ip_addr == 0) {
@@ -1275,9 +1219,9 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
                         return -1;
                 }
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
+                if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_DBG("Won't allocate flow with non-enrolled IPCP.");
                         close(fd);
                         return -1; /* -ENOTENROLLED */
@@ -1320,13 +1264,13 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (ipcp_udp_port_alloc(ip_addr,
                                 f_saddr.sin_port,
                                 dst_name,
                                 src_ae_name) < 0) {
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
                 clr_fd(fd);
@@ -1340,7 +1284,7 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
                  _ap_instance->flows[fd].rb     = NULL;
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 close(fd);
                 return -1;
         }
@@ -1363,14 +1307,7 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
         if (response)
                 return 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBG("IPCP in wrong state.");
-                close(fd);
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         /* awaken pending flow */
 
@@ -1379,14 +1316,14 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
         fd = port_id_to_fd(port_id);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Could not find flow with port_id %d.", port_id);
                 return -1;
         }
 
         if (_ap_instance->flows[fd].state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Flow was not pending.");
                 return -1;
         }
@@ -1397,7 +1334,7 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
                 _ap_instance->flows[fd].state   = FLOW_NULL;
                 _ap_instance->flows[fd].port_id = -1;
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1413,20 +1350,20 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
 
         _ap_instance->flows[fd].state = FLOW_ALLOCATED;
         _ap_instance->flows[fd].rb    = rb;
-        pthread_rwlock_unlock(&_ap_instance->flows_lock);
 
+        pthread_rwlock_unlock(&_ap_instance->flows_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         set_fd(fd);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (ipcp_udp_port_alloc_resp(r_saddr.sin_addr.s_addr,
                                      f_saddr.sin_port,
                                      r_saddr.sin_port,
                                      response) < 0) {
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
                 clr_fd(fd);
@@ -1439,7 +1376,7 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
                 _ap_instance->flows[fd].rb    = NULL;
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 return -1;
         }
@@ -1457,13 +1394,13 @@ static int ipcp_udp_flow_dealloc(int port_id)
         struct sockaddr_in    r_saddr;
         socklen_t             r_saddr_len = sizeof(r_saddr);
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         fd = port_id_to_fd(port_id);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Could not find flow with port_id %d.", port_id);
                 return 0;
         }
@@ -1484,7 +1421,7 @@ static int ipcp_udp_flow_dealloc(int port_id)
                 shm_ap_rbuff_close(rb);
 
         if (getpeername(fd, (struct sockaddr *) &r_saddr, &r_saddr_len) < 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Flow with port_id %d has no peer.", port_id);
                 close(fd);
                 return 0;
@@ -1494,7 +1431,7 @@ static int ipcp_udp_flow_dealloc(int port_id)
         r_saddr.sin_port = LISTEN_PORT;
 
         if (connect(fd, (struct sockaddr *) &r_saddr, sizeof(r_saddr)) < 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 close(fd);
                 return 0 ;
         }
@@ -1502,12 +1439,12 @@ static int ipcp_udp_flow_dealloc(int port_id)
         if (ipcp_udp_port_dealloc(r_saddr.sin_addr.s_addr,
                                   remote_udp) < 0) {
                 LOG_DBG("Could not notify remote.");
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 close(fd);
                 return 0;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         close(fd);
 
@@ -1516,11 +1453,20 @@ static int ipcp_udp_flow_dealloc(int port_id)
         return 0;
 }
 
+static struct ipcp_ops udp_ops = {
+        .ipcp_bootstrap       = ipcp_udp_bootstrap,
+        .ipcp_enroll          = NULL,                       /* shim */
+        .ipcp_name_reg        = ipcp_udp_name_reg,
+        .ipcp_name_unreg      = ipcp_udp_name_unreg,
+        .ipcp_flow_alloc      = ipcp_udp_flow_alloc,
+        .ipcp_flow_alloc_resp = ipcp_udp_flow_alloc_resp,
+        .ipcp_flow_dealloc    = ipcp_udp_flow_dealloc
+};
+
 static struct ipcp * ipcp_udp_create()
 {
         struct ipcp * i;
         struct ipcp_udp_data * data;
-        struct ipcp_ops *      ops;
 
         i = ipcp_instance_create();
         if (i == NULL)
@@ -1532,23 +1478,8 @@ static struct ipcp * ipcp_udp_create()
                 return NULL;
         }
 
-        ops = malloc(sizeof(*ops));
-        if (ops == NULL) {
-                free(data);
-                free(i);
-                return NULL;
-        }
-
-        ops->ipcp_bootstrap       = ipcp_udp_bootstrap;
-        ops->ipcp_enroll          = NULL;                       /* shim */
-        ops->ipcp_name_reg        = ipcp_udp_name_reg;
-        ops->ipcp_name_unreg      = ipcp_udp_name_unreg;
-        ops->ipcp_flow_alloc      = ipcp_udp_flow_alloc;
-        ops->ipcp_flow_alloc_resp = ipcp_udp_flow_alloc_resp;
-        ops->ipcp_flow_dealloc    = ipcp_udp_flow_dealloc;
-
         i->data = (struct ipcp_data *) data;
-        i->ops  = ops;
+        i->ops  = &udp_ops;
 
         i->state = IPCP_INIT;
 
@@ -1599,16 +1530,12 @@ int main(int argc, char * argv[])
                 exit(EXIT_FAILURE);
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&_ap_instance->mainloop, NULL, ipcp_main_loop, _ipcp);
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 close_logfile();
@@ -1617,10 +1544,17 @@ int main(int argc, char * argv[])
 
         pthread_join(_ap_instance->mainloop, NULL);
 
+        pthread_cancel(_ap_instance->handler);
+        pthread_cancel(_ap_instance->sdu_reader);
+        pthread_cancel(_ap_instance->sduloop);
+
+        pthread_join(_ap_instance->sduloop, NULL);
+        pthread_join(_ap_instance->handler, NULL);
+        pthread_join(_ap_instance->sdu_reader, NULL);
+
         shim_ap_fini();
 
         ipcp_data_destroy(_ipcp->data);
-        free(_ipcp->ops);
         free(_ipcp);
 
         close_logfile();