Merged in dstaesse/ouroboros/be-ipcp-locks (pull request #221)

ipcpd: Revised locking

9 files changed, 390 insertions, 460 deletions

diff --git a/include/ouroboros/config.h.in b/include/ouroboros/config.h.in
index 84d56e85..3a246235 100644
--- a/include/ouroboros/config.h.in
+++ b/include/ouroboros/config.h.in
@@ -52,6 +52,7 @@
 #define SHM_DU_TIMEOUT_MICROS  15000
 #define IRMD_ACCEPT_TIMEOUT    100
 #define IRMD_FLOW_TIMEOUT      5000
+#define IPCP_ACCEPT_TIMEOUT    100
 #define SOCKET_TIMEOUT         4000
 #define CDAP_REPLY_TIMEOUT     1000
 #define ENROLL_TIMEOUT         2000
diff --git a/src/ipcpd/ipcp.c b/src/ipcpd/ipcp.c
index 1c4fab94..87bf9e51 100644
--- a/src/ipcpd/ipcp.c
+++ b/src/ipcpd/ipcp.c
@@ -22,6 +22,7 @@
 
 #include <ouroboros/config.h>
 #include <ouroboros/ipcp.h>
+#include <ouroboros/time_utils.h>
 
 #define OUROBOROS_PREFIX "ipcpd/ipcp"
 #include <ouroboros/logs.h>
@@ -44,7 +45,8 @@ struct ipcp * ipcp_instance_create()
         i->irmd_fd = -1;
         i->state   = IPCP_INIT;
 
-        pthread_mutex_init(&i->state_lock, NULL);
+        pthread_rwlock_init(&i->state_lock, NULL);
+        pthread_mutex_init(&i->state_mtx, NULL);
         pthread_condattr_init(&cattr);
         pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK);
         pthread_cond_init(&i->state_cond, &cattr);
@@ -52,14 +54,65 @@ struct ipcp * ipcp_instance_create()
         return i;
 }
 
-void ipcp_state_change(struct ipcp * ipcp,
-                       enum ipcp_state state)
+void ipcp_set_state(struct ipcp * ipcp,
+                    enum ipcp_state state)
 {
         if (ipcp == NULL)
             return;
 
+        pthread_mutex_lock(&ipcp->state_mtx);
+
         ipcp->state = state;
+
         pthread_cond_broadcast(&ipcp->state_cond);
+        pthread_mutex_unlock(&ipcp->state_mtx);
+}
+
+enum ipcp_state ipcp_get_state(struct ipcp * ipcp)
+{
+        enum ipcp_state state;
+
+        if (ipcp == NULL)
+                return IPCP_NULL;
+
+        pthread_mutex_lock(&ipcp->state_mtx);
+
+        state = ipcp->state;
+
+        pthread_mutex_unlock(&ipcp->state_mtx);
+
+        return state;
+}
+
+int ipcp_wait_state(struct ipcp *           ipcp,
+                    enum ipcp_state         state,
+                    const struct timespec * timeout)
+{
+        struct timespec abstime;
+
+        clock_gettime(PTHREAD_COND_CLOCK, &abstime);
+        ts_add(&abstime, timeout, &abstime);
+
+        pthread_mutex_lock(&ipcp->state_mtx);
+
+        while (ipcp->state != state && ipcp->state != IPCP_SHUTDOWN) {
+                int ret;
+                if (timeout == NULL)
+                        ret = pthread_cond_wait(&ipcp->state_cond,
+                                                &ipcp->state_mtx);
+                else
+                        ret = pthread_cond_timedwait(&ipcp->state_cond,
+                                                     &ipcp->state_mtx,
+                                                     &abstime);
+                if (ret) {
+                        pthread_mutex_unlock(&ipcp->state_mtx);
+                        return -ret;
+                }
+        }
+
+        pthread_mutex_unlock(&ipcp->state_mtx);
+
+        return 0;
 }
 
 int ipcp_parse_arg(int argc, char * argv[])
@@ -101,22 +154,6 @@ int ipcp_parse_arg(int argc, char * argv[])
         return 0;
 }
 
-static void close_ptr(void * o)
-{
-        char * name = ipcp_sock_path(getpid());
-        close(*((int *) o));
-
-        if (unlink(name))
-                LOG_DBG("Could not unlink %s.", name);
-
-        free(name);
-}
-
-static void clean_msg(void * msg)
-{
-         ipcp_msg__free_unpacked(msg, NULL);
-}
-
 void * ipcp_main_loop(void * o)
 {
         int     lsockfd;
@@ -135,9 +172,13 @@ void * ipcp_main_loop(void * o)
         char * sock_path;
         char * msg_name_dup;
 
-        struct timeval tv = {(SOCKET_TIMEOUT / 1000),
+        struct timeval tv = {(IPCP_ACCEPT_TIMEOUT / 1000),
+                             (IPCP_ACCEPT_TIMEOUT % 1000) * 1000};
+
+        struct timeval ltv = {(SOCKET_TIMEOUT / 1000),
                              (SOCKET_TIMEOUT % 1000) * 1000};
 
+
         if (_ipcp == NULL) {
                 LOG_ERR("Invalid ipcp struct.");
                 return (void *) 1;
@@ -150,28 +191,33 @@ void * ipcp_main_loop(void * o)
         sockfd = server_socket_open(sock_path);
         if (sockfd < 0) {
                 LOG_ERR("Could not open server socket.");
+                free(sock_path);
                 return (void *) 1;
         }
 
-        free(sock_path);
-
-        pthread_cleanup_push(close_ptr, (void *) &sockfd);
+        if (setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO,
+                       (void *) &tv, sizeof(tv)))
+                LOG_WARN("Failed to set timeout on socket.");
 
         while (true) {
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
+                if (ipcp_get_state(_ipcp) == IPCP_SHUTDOWN) {
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        break;
+                }
+
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+
                 ret_msg.code = IPCP_MSG_CODE__IPCP_REPLY;
 
                 lsockfd = accept(sockfd, 0, 0);
-                if (lsockfd < 0) {
-                        LOG_ERR("Cannot accept new connection");
-                        break;
-                }
+                if (lsockfd < 0)
+                        continue;
 
                 if (setsockopt(lsockfd, SOL_SOCKET, SO_RCVTIMEO,
-                               (void *) &tv, sizeof(tv)))
+                               (void *) &ltv, sizeof(ltv)))
                         LOG_WARN("Failed to set timeout on socket.");
 
-                pthread_cleanup_push(close_ptr, (void *) &lsockfd);
-
                 count = read(lsockfd, buf, IPCP_MSG_BUF_SIZE);
                 if (count <= 0) {
                         LOG_ERR("Failed to read from socket");
@@ -185,8 +231,6 @@ void * ipcp_main_loop(void * o)
                         continue;
                 }
 
-                pthread_cleanup_push(clean_msg, (void *) msg);
-
                 switch (msg->code) {
                 case IPCP_MSG_CODE__IPCP_BOOTSTRAP:
                         if (_ipcp->ops->ipcp_bootstrap == NULL) {
@@ -201,7 +245,6 @@ void * ipcp_main_loop(void * o)
                                 ret_msg.result = -1;
                                 break;
                         }
-
                         if (conf_msg->ipcp_type == IPCP_NORMAL) {
                                 conf.addr_size = conf_msg->addr_size;
                                 conf.cep_id_size = conf_msg->cep_id_size;
@@ -218,7 +261,6 @@ void * ipcp_main_loop(void * o)
                                 conf.ip_addr  = conf_msg->ip_addr;
                                 conf.dns_addr = conf_msg->dns_addr;
                         }
-
                         if (conf_msg->ipcp_type == IPCP_SHIM_ETH_LLC)
                                 conf.if_name = conf_msg->if_name;
 
@@ -295,8 +337,7 @@ void * ipcp_main_loop(void * o)
                         break;
                 }
 
-                pthread_cleanup_pop(true);
-                pthread_cleanup_pop(false);
+                ipcp_msg__free_unpacked(msg, NULL);
 
                 buffer.len = ipcp_msg__get_packed_size(&ret_msg);
                 if (buffer.len == 0) {
@@ -321,10 +362,13 @@ void * ipcp_main_loop(void * o)
 
                 free(buffer.data);
                 close(lsockfd);
-
         }
 
-        pthread_cleanup_pop(true);
+        close(sockfd);
+        if (unlink(sock_path))
+                LOG_DBG("Could not unlink %s.", sock_path);
+
+        free(sock_path);
 
-        return NULL;
+        return (void *) 0;
 }
diff --git a/src/ipcpd/ipcp.h b/src/ipcpd/ipcp.h
index 8b7d4ec9..edaea0fd 100644
--- a/src/ipcpd/ipcp.h
+++ b/src/ipcpd/ipcp.h
@@ -27,12 +27,14 @@
 #include <ouroboros/shared.h>
 
 #include <pthread.h>
+#include <time.h>
 
 #include "ipcp-ops.h"
 #include "ipcp-data.h"
 
 enum ipcp_state {
-        IPCP_INIT = 0,
+        IPCP_NULL = 0,
+        IPCP_INIT,
         IPCP_PENDING_ENROLL,
         IPCP_ENROLLED,
         IPCP_DISCONNECTED,
@@ -45,15 +47,27 @@ struct ipcp {
         int                irmd_fd;
 
         enum ipcp_state    state;
-        pthread_mutex_t    state_lock;
+        pthread_rwlock_t   state_lock;
+        pthread_mutex_t    state_mtx;
         pthread_cond_t     state_cond;
 };
 
-struct ipcp * ipcp_instance_create();
-void          ipcp_state_change(struct ipcp * ipcp,
-                                enum ipcp_state state);
-void *        ipcp_main_loop(void * o);
-void *        ipcp_sdu_loop(void * o);
-int           ipcp_parse_arg(int argc, char * argv[]);
+struct ipcp *   ipcp_instance_create();
+
+void            ipcp_set_state(struct ipcp *   ipcp,
+                               enum ipcp_state state);
+
+enum ipcp_state ipcp_get_state(struct ipcp * ipcp);
+
+int             ipcp_wait_state(struct ipcp *           ipcp,
+                                enum ipcp_state         state,
+                                const struct timespec * timeout);
+
+void *          ipcp_main_loop(void * o);
+
+void *          ipcp_sdu_loop(void * o);
+
+int             ipcp_parse_arg(int argc,
+                               char * argv[]);
 
 #endif
diff --git a/src/ipcpd/local/main.c b/src/ipcpd/local/main.c
index 72bf0e4b..6993cea2 100644
--- a/src/ipcpd/local/main.c
+++ b/src/ipcpd/local/main.c
@@ -33,7 +33,6 @@
 #include <ouroboros/bitmap.h>
 #include <ouroboros/shared.h>
 #include <ouroboros/dev.h>
-
 #define OUROBOROS_PREFIX "ipcpd/local"
 
 #include <ouroboros/logs.h>
@@ -133,7 +132,7 @@ void shim_ap_fini()
         if (_ap_instance == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up AP while not in shutdown.");
@@ -152,7 +151,7 @@ void shim_ap_fini()
                         shm_ap_rbuff_close(_ap_instance->flows[i].rb);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         free(_ap_instance);
 }
@@ -187,10 +186,10 @@ static void * ipcp_local_sdu_loop(void * o)
                         continue;
                 }
 
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         return (void *) 1; /* -ENOTENROLLED */
                 }
 
@@ -198,7 +197,7 @@ static void * ipcp_local_sdu_loop(void * o)
                 fd = _ap_instance->in_out[port_id_to_fd(e->port_id)];
                 if (fd == -1) {
                         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(e);
                         continue;
                 }
@@ -209,7 +208,7 @@ static void * ipcp_local_sdu_loop(void * o)
                         ;
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 free(e);
         }
@@ -229,25 +228,14 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
         case SIGHUP:
         case SIGQUIT:
                 if (info->si_pid == irmd_api) {
-                        bool clean_threads = false;
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
-
-                        if (_ipcp->state == IPCP_ENROLLED)
-                                clean_threads = true;
-
-                        _ipcp->state = IPCP_SHUTDOWN;
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
-                        if (clean_threads) {
-                                pthread_cancel(_ap_instance->sduloop);
-                                pthread_join(_ap_instance->sduloop, NULL);
-                        }
-
-                        pthread_cancel(_ap_instance->mainloop);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                 }
         default:
                 return;
@@ -261,22 +249,22 @@ static int ipcp_local_bootstrap(struct dif_config * conf)
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("IPCP in wrong state.");
                 return -1;
         }
 
-        _ipcp->state = IPCP_ENROLLED;
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         pthread_create(&_ap_instance->sduloop,
                        NULL,
                        ipcp_local_sdu_loop,
                        NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped local IPCP with api %d.",
                 getpid());
@@ -286,21 +274,15 @@ static int ipcp_local_bootstrap(struct dif_config * conf)
 
 static int ipcp_local_name_reg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBGF("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Failed to add %s to local registry.", name);
                 return -1;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Registered %s.", name);
 
@@ -309,11 +291,11 @@ static int ipcp_local_name_reg(char * name)
 
 static int ipcp_local_name_unreg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -336,17 +318,17 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
 
         /* This ipcpd has all QoS */
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBGF("Won't allocate flow with non-enrolled IPCP.");
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+                LOG_DBGF("Won't register with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
         }
 
         rb = shm_ap_rbuff_open(n_api);
         if (rb == NULL) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1; /* -ENORBUFF */
         }
 
@@ -355,7 +337,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
         in_fd = bmp_allocate(_ap_instance->fds);
         if (!bmp_is_id_valid(_ap_instance->fds, in_fd)) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -EMFILE;
         }
 
@@ -372,7 +354,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
 
         if (port_id < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Could not get port id from IRMd");
                 /* shm_ap_rbuff_close(n_api); */
                 return -1;
@@ -382,7 +364,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
         if (!bmp_is_id_valid(_ap_instance->fds, out_fd)) {
                 /* shm_ap_rbuff_close(n_api); */
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1; /* -ENOMOREFDS */
         }
 
@@ -394,7 +376,7 @@ static int ipcp_local_flow_alloc(pid_t         n_api,
         _ap_instance->in_out[out_fd] = in_fd;
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBGF("Pending local allocation request, port_id %d.", port_id);
 
@@ -413,7 +395,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         if (response)
                 return 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         /* awaken pending flow */
 
@@ -422,14 +404,14 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         in_fd = port_id_to_fd(port_id);
         if (in_fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Could not find flow with port_id %d.", port_id);
                 return -1;
         }
 
         if (_ap_instance->flows[in_fd].state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Flow was not pending.");
                 return -1;
         }
@@ -441,7 +423,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
                 _ap_instance->flows[in_fd].port_id = -1;
                 _ap_instance->in_out[in_fd] = -1;
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -453,7 +435,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         out_fd = _ap_instance->in_out[in_fd];
         if (out_fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("No pending local flow with port_id %d.", port_id);
                 return -1;
         }
@@ -461,7 +443,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         if (_ap_instance->flows[out_fd].state != FLOW_PENDING) {
                  /* FIXME: clean up other end */
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Flow was not pending.");
                 return -1;
         }
@@ -469,7 +451,7 @@ static int ipcp_local_flow_alloc_resp(pid_t n_api,
         _ap_instance->flows[out_fd].state = FLOW_ALLOCATED;
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if ((ret = ipcp_flow_alloc_reply(getpid(),
                                          _ap_instance->flows[out_fd].port_id,
@@ -489,13 +471,13 @@ static int ipcp_local_flow_dealloc(int port_id)
         int fd = -1;
         struct shm_ap_rbuff * rb;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&_ap_instance->flows_lock);
 
         fd = port_id_to_fd(port_id);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Could not find flow with port_id %d.", port_id);
                 return 0;
         }
@@ -517,7 +499,7 @@ static int ipcp_local_flow_dealloc(int port_id)
         if (rb != NULL)
                 shm_ap_rbuff_close(rb);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBGF("Flow with port_id %d deallocated.", port_id);
 
@@ -611,16 +593,12 @@ int main(int argc, char * argv[])
                 exit(EXIT_FAILURE);
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&_ap_instance->mainloop, NULL, ipcp_main_loop, _ipcp);
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 close_logfile();
@@ -629,6 +607,9 @@ int main(int argc, char * argv[])
 
         pthread_join(_ap_instance->mainloop, NULL);
 
+        pthread_cancel(_ap_instance->sduloop);
+        pthread_join(_ap_instance->sduloop, NULL);
+
         shim_ap_fini();
 
         ipcp_data_destroy(_ipcp->data);
diff --git a/src/ipcpd/normal/fmgr.c b/src/ipcpd/normal/fmgr.c
index 58ae1dc8..23d9514a 100644
--- a/src/ipcpd/normal/fmgr.c
+++ b/src/ipcpd/normal/fmgr.c
@@ -98,17 +98,16 @@ static void * fmgr_listen(void * o)
         char * ae_name;
 
         while (true) {
-                pthread_mutex_lock(&_ipcp->state_lock);
-                while (!(_ipcp->state == IPCP_ENROLLED ||
-                         _ipcp->state == IPCP_SHUTDOWN))
-                        pthread_cond_wait(&_ipcp->state_cond,
-                                          &_ipcp->state_lock);
-
-                if (_ipcp->state == IPCP_SHUTDOWN) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                ipcp_wait_state(_ipcp, IPCP_ENROLLED, NULL);
+
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
+
+                if (ipcp_get_state(_ipcp) == IPCP_SHUTDOWN) {
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         return 0;
                 }
-                pthread_mutex_unlock(&_ipcp->state_lock);
+
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 fd = flow_accept(&ae_name);
                 if (fd < 0) {
diff --git a/src/ipcpd/normal/main.c b/src/ipcpd/normal/main.c
index 4173246d..9479b806 100644
--- a/src/ipcpd/normal/main.c
+++ b/src/ipcpd/normal/main.c
@@ -75,13 +75,11 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        ipcp_state_change(_ipcp, IPCP_SHUTDOWN);
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-
-                        pthread_cancel(normal_data(_ipcp)->mainloop);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
 
                         if (fmgr_fini())
                                 LOG_ERR("Failed to finalize flow manager.");
@@ -99,21 +97,15 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
 
 static int normal_ipcp_name_reg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_ERR("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to add %s to local registry.", name);
                 return -1;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Registered %s.", name);
 
@@ -122,11 +114,11 @@ static int normal_ipcp_name_reg(char * name)
 
 static int normal_ipcp_name_unreg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -135,62 +127,59 @@ static int normal_ipcp_enroll(char * dif_name)
 {
         struct timespec timeout = {(ENROLL_TIMEOUT / 1000),
                                    (ENROLL_TIMEOUT % 1000) * MILLION};
-        struct timespec abstime;
-
-        clock_gettime(PTHREAD_COND_CLOCK, &abstime);
-        ts_add(&abstime, &timeout, &abstime);
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Won't enroll an IPCP that is not in INIT.");
                 return -1; /* -ENOTINIT */
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (fmgr_mgmt_flow(dif_name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to establish management flow.");
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        while (_ipcp->state != IPCP_ENROLLED)
-                if (pthread_cond_timedwait(&_ipcp->state_cond,
-                                           &_ipcp->state_lock,
-                                           &abstime) == ETIMEDOUT) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        LOG_ERR("Enrollment didn't complete in time.");
-                        return -1;
-                }
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
+
+        if (ipcp_wait_state(_ipcp, IPCP_ENROLLED, &timeout) == -ETIMEDOUT) {
+                LOG_ERR("Enrollment timed out.");
+                return -1;
+        }
+
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
+
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+                return -1;
+        }
 
         return 0;
 }
 
 static int normal_ipcp_bootstrap(struct dif_config * conf)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Won't bootstrap an IPCP that is not in INIT.");
                 return -1; /* -ENOTINIT */
         }
 
         if (ribmgr_bootstrap(conf)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to bootstrap RIB manager.");
                 return -1;
         }
 
-        ipcp_state_change(_ipcp, IPCP_ENROLLED);
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         _ipcp->data->dif_name = conf->dif_name;
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped in DIF %s.", conf->dif_name);
 
@@ -247,9 +236,9 @@ void normal_ipcp_data_destroy()
         if (_ipcp == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_SHUTDOWN)
+        if (ipcp_get_state(_ipcp) != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up while not in shutdown.");
 
         if (normal_data(_ipcp)->dum != NULL)
@@ -257,7 +246,7 @@ void normal_ipcp_data_destroy()
         if (normal_data(_ipcp)->rb != NULL)
                 shm_ap_rbuff_close(normal_data(_ipcp)->rb);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_data_destroy(_ipcp->data);
 }
@@ -331,8 +320,6 @@ int main(int argc, char * argv[])
                 exit(EXIT_FAILURE);
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&normal_data(_ipcp)->mainloop, NULL,
@@ -340,8 +327,6 @@ int main(int argc, char * argv[])
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 normal_ipcp_data_destroy();
diff --git a/src/ipcpd/normal/ribmgr.c b/src/ipcpd/normal/ribmgr.c
index c8d517b5..9b61e180 100644
--- a/src/ipcpd/normal/ribmgr.c
+++ b/src/ipcpd/normal/ribmgr.c
@@ -254,15 +254,15 @@ int ribmgr_cdap_write(struct cdap * instance,
         static_info_msg_t * msg;
         int ret = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        if (_ipcp->state == IPCP_PENDING_ENROLL &&
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) == IPCP_PENDING_ENROLL &&
             strcmp(name, STATIC_INFO) == 0) {
                 LOG_DBG("Received static DIF information.");
 
                 msg = static_info_msg__unpack(NULL, len, data);
                 if (msg == NULL) {
-                        ipcp_state_change(_ipcp, IPCP_INIT);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        ipcp_set_state(_ipcp, IPCP_INIT);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         cdap_send_reply(instance, invoke_id, -1, NULL, 0);
                         LOG_ERR("Failed to unpack static info message.");
                         return -1;
@@ -280,8 +280,8 @@ int ribmgr_cdap_write(struct cdap * instance,
                 rib->address = msg->address;
 
                 if (frct_init(&rib->dtc, rib->address)) {
-                        ipcp_state_change(_ipcp, IPCP_INIT);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        ipcp_set_state(_ipcp, IPCP_INIT);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         cdap_send_reply(instance, invoke_id, -1, NULL, 0);
                         static_info_msg__free_unpacked(msg, NULL);
                         LOG_ERR("Failed to init FRCT");
@@ -289,9 +289,11 @@ int ribmgr_cdap_write(struct cdap * instance,
                 }
 
                 static_info_msg__free_unpacked(msg, NULL);
-        } else
+        } else {
                 ret = -1;
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        }
+
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (cdap_send_reply(instance, invoke_id, ret, NULL, 0)) {
                 LOG_ERR("Failed to send reply to write request.");
@@ -332,13 +334,13 @@ int ribmgr_cdap_start(struct cdap * instance,
         size_t len = 0;
         int iid = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        if (_ipcp->state == IPCP_ENROLLED &&
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) == IPCP_ENROLLED &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("New enrollment request.");
 
                 if (cdap_send_reply(instance, invoke_id, 0, NULL, 0)) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to send reply to enrollment request.");
                         return -1;
                 }
@@ -357,14 +359,14 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 len = static_info_msg__get_packed_size(&stat_info);
                 if (len == 0) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to get size of static information.");
                         return -1;
                 }
 
                 data = malloc(len);
                 if (data == NULL) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to allocate memory.");
                         return -1;
                 }
@@ -378,7 +380,7 @@ int ribmgr_cdap_start(struct cdap * instance,
                 iid = cdap_send_write(instance, STATIC_INFO, data, len, 0);
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to send static information.");
                         return -1;
@@ -386,7 +388,7 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 if (cdap_result_wait(instance, WRITE, STATIC_INFO, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Remote did not receive static information.");
                         return -1;
@@ -402,7 +404,7 @@ int ribmgr_cdap_start(struct cdap * instance,
                 iid = cdap_send_stop(instance, ENROLLMENT);
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to send stop of enrollment.");
                         return -1;
@@ -410,7 +412,7 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 if (cdap_result_wait(instance, STOP, ENROLLMENT, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Remote failed to complete enrollment.");
                         return -1;
@@ -420,12 +422,12 @@ int ribmgr_cdap_start(struct cdap * instance,
                 free(data);
         } else {
                 if (cdap_send_reply(instance, invoke_id, -1, NULL, 0)) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to send reply to start request.");
                         return -1;
                 }
         }
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -436,21 +438,21 @@ int ribmgr_cdap_stop(struct cdap * instance,
 {
         int ret = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-        if (_ipcp->state == IPCP_PENDING_ENROLL &&
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) == IPCP_PENDING_ENROLL &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("Stop enrollment received.");
 
-                ipcp_state_change(_ipcp, IPCP_ENROLLED);
+                ipcp_set_state(_ipcp, IPCP_ENROLLED);
         } else
                 ret = -1;
 
         if (cdap_send_reply(instance, invoke_id, ret, NULL, 0)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to send reply to stop request.");
                 return -1;
         }
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -486,11 +488,11 @@ int ribmgr_add_flow(int fd)
         flow->instance = instance;
         flow->fd = fd;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&rib->flows_lock);
         if (list_empty(&rib->flows) &&
-            _ipcp->state == IPCP_INIT) {
-                ipcp_state_change(_ipcp, IPCP_PENDING_ENROLL);
+            ipcp_get_state(_ipcp) == IPCP_INIT) {
+                ipcp_set_state(_ipcp, IPCP_PENDING_ENROLL);
 
                 pthread_mutex_lock(&rib->cdap_reqs_lock);
                 iid = cdap_send_start(instance,
@@ -498,6 +500,7 @@ int ribmgr_add_flow(int fd)
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
                         pthread_rwlock_unlock(&rib->flows_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to start enrollment.");
                         cdap_destroy(instance);
                         free(flow);
@@ -507,6 +510,7 @@ int ribmgr_add_flow(int fd)
                 if (cdap_result_wait(instance, START, ENROLLMENT, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
                         pthread_rwlock_unlock(&rib->flows_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to start enrollment.");
                         cdap_destroy(instance);
                         free(flow);
@@ -514,10 +518,10 @@ int ribmgr_add_flow(int fd)
                 }
                 pthread_mutex_unlock(&rib->cdap_reqs_lock);
         }
-        pthread_mutex_unlock(&_ipcp->state_lock);
 
         list_add(&flow->next, &rib->flows);
         pthread_rwlock_unlock(&rib->flows_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -534,6 +538,7 @@ int ribmgr_remove_flow(int fd)
                         if (cdap_destroy(flow->instance))
                                 LOG_ERR("Failed to destroy CDAP instance.");
                         list_del(&flow->next);
+                        pthread_rwlock_unlock(&rib->flows_lock);
                         free(flow);
                         return 0;
                 }
diff --git a/src/ipcpd/shim-eth-llc/main.c b/src/ipcpd/shim-eth-llc/main.c
index e4d7429a..60b55009 100644
--- a/src/ipcpd/shim-eth-llc/main.c
+++ b/src/ipcpd/shim-eth-llc/main.c
@@ -197,9 +197,9 @@ void eth_llc_ipcp_data_destroy()
         if (_ipcp == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_SHUTDOWN)
+        if (ipcp_get_state(_ipcp) != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up while not in shutdown.");
 
         if (shim_data(_ipcp)->dum != NULL)
@@ -218,7 +218,7 @@ void eth_llc_ipcp_data_destroy()
                         shm_ap_rbuff_close(ipcp_flow(i)->rb);
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_data_destroy(_ipcp->data);
 }
@@ -478,12 +478,12 @@ static int eth_llc_ipcp_port_req(uint8_t r_sap,
         ssize_t index = 0;
         int i;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         index = bmp_allocate(shim_data(_ipcp)->indices);
         if (index < 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
                 LOG_ERR("Out of free indices.");
                 return -1;
@@ -496,7 +496,7 @@ static int eth_llc_ipcp_port_req(uint8_t r_sap,
 
         if (port_id < 0) {
                 bmp_release(shim_data(_ipcp)->indices, index);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
                 LOG_ERR("Could not get port id from IRMd.");
                 return -1;
@@ -511,7 +511,7 @@ static int eth_llc_ipcp_port_req(uint8_t r_sap,
         }
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("New flow request, port_id %d, remote SAP %d.", port_id, r_sap);
 
@@ -528,20 +528,20 @@ static int eth_llc_ipcp_port_alloc_reply(uint8_t ssap,
         int port_id = -1;
         int i;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&shim_data(_ipcp)->flows_lock);
 
         index = sap_to_index(ssap);
         if (index < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("No flow found with that SAP.");
                 return -1; /* -EFLOWNOTFOUND */
         }
 
         if (ipcp_flow(index)->state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1; /* -EFLOWNOTPENDING */
         }
 
@@ -558,7 +558,7 @@ static int eth_llc_ipcp_port_alloc_reply(uint8_t ssap,
         }
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Flow reply, port_id %d, remote SAP %d.", port_id, dsap);
 
@@ -578,13 +578,13 @@ static int eth_llc_ipcp_flow_dealloc_req(uint8_t ssap,
         int port_id = -1;
         int i = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         i = sap_to_index(ssap);
         if (i < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("No flow found for remote deallocation request.");
                 return 0;
         }
@@ -593,7 +593,7 @@ static int eth_llc_ipcp_flow_dealloc_req(uint8_t ssap,
         destroy_ipcp_flow(i);
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_flow_dealloc(0, port_id);
 
@@ -669,15 +669,6 @@ static void * eth_llc_ipcp_sdu_reader(void * o)
         memset(br_addr, 0xff, MAC_SIZE * sizeof(uint8_t));
 
         while (true) {
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
-
-                pthread_mutex_unlock(&_ipcp->state_lock);
-
 #if defined(PACKET_RX_RING) && defined(PACKET_TX_RING)
                 header = (void *) shim_data(_ipcp)->rx_ring +
                         (offset * SHM_DU_BUFF_BLOCK_SIZE);
@@ -789,13 +780,7 @@ static void * eth_llc_ipcp_sdu_writer(void * o)
                 if (e == NULL)
                         continue;
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 len = shm_du_map_read((uint8_t **) &buf,
                                       shim_data(_ipcp)->dum,
@@ -826,6 +811,8 @@ static void * eth_llc_ipcp_sdu_writer(void * o)
                 if (shim_data(_ipcp)->dum != NULL)
                         shm_du_map_remove(shim_data(_ipcp)->dum, e->index);
 
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+
                 free(e);
         }
 
@@ -843,30 +830,14 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
         case SIGTERM:
         case SIGHUP:
                 if (info->si_pid == irmd_api) {
-                        bool clean_threads = false;
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
-
-                        if (_ipcp->state == IPCP_ENROLLED)
-                                clean_threads = true;
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        _ipcp->state = IPCP_SHUTDOWN;
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-
-                        if (clean_threads) {
-                                pthread_cancel(shim_data(_ipcp)->sdu_reader);
-                                pthread_cancel(shim_data(_ipcp)->sdu_writer);
-
-                                pthread_join(shim_data(_ipcp)->sdu_writer,
-                                             NULL);
-                                pthread_join(shim_data(_ipcp)->sdu_reader,
-                                             NULL);
-                        }
-
-                        pthread_cancel(shim_data(_ipcp)->mainloop);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 }
         default:
@@ -1030,10 +1001,10 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
 
 #endif
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("IPCP in wrong state.");
                 close(fd);
                 return -1;
@@ -1043,7 +1014,7 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
         shim_data(_ipcp)->device = device;
         shim_data(_ipcp)->tx_offset = 0;
 
-        _ipcp->state = IPCP_ENROLLED;
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         pthread_create(&shim_data(_ipcp)->sdu_reader,
                        NULL,
@@ -1055,7 +1026,7 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
                        eth_llc_ipcp_sdu_writer,
                        NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped shim IPCP over Ethernet with LLC with api %d.",
                 getpid());
@@ -1065,21 +1036,15 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
 
 static int eth_llc_ipcp_name_reg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBGF("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to add %s to local registry.", name);
                 return -1;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Registered %s.", name);
 
@@ -1088,11 +1053,11 @@ static int eth_llc_ipcp_name_reg(char * name)
 
 static int eth_llc_ipcp_name_unreg(char * name)
 {
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -1120,19 +1085,19 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
         if (rb == NULL)
                 return -1; /* -ENORBUFF */
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_ENROLLED) {
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 shm_ap_rbuff_close(rb);
-                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Won't allocate flow with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
         }
 
         index = bmp_allocate(shim_data(_ipcp)->indices);
         if (index < 0) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 shm_ap_rbuff_close(rb);
-                pthread_mutex_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1143,7 +1108,7 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
                 shm_ap_rbuff_close(rb);
                 bmp_release(shim_data(_ipcp)->indices, index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1153,7 +1118,7 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
         shim_data(_ipcp)->flows[index].sap = ssap;
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         memset(r_addr, 0xff, MAC_SIZE * sizeof(uint8_t));
 
@@ -1161,11 +1126,11 @@ static int eth_llc_ipcp_flow_alloc(pid_t         n_api,
                                     dst_name,
                                     src_ae_name) < 0) {
                 LOG_DBGF("Port alloc returned -1.");
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
                 destroy_ipcp_flow(index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1183,20 +1148,20 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
         int index = -1;
         uint8_t ssap = 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         index = port_id_to_index(port_id);
         if (index < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Could not find flow with port_id %d.", port_id);
                 return -1;
         }
 
         if (ipcp_flow(index)->state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Flow was not pending.");
                 return -1;
         }
@@ -1208,7 +1173,7 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
                 ipcp_flow(index)->port_id = -1;
                 bmp_release(shim_data(_ipcp)->indices, index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1219,7 +1184,7 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
                 shm_ap_rbuff_close(ipcp_flow(index)->rb);
                 bmp_release(shim_data(_ipcp)->indices, index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1228,17 +1193,17 @@ static int eth_llc_ipcp_flow_alloc_resp(pid_t n_api,
         shim_data(_ipcp)->flows[index].sap = ssap;
 
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (eth_llc_ipcp_port_alloc_resp(shim_data(_ipcp)->flows[index].r_addr,
                                          shim_data(_ipcp)->flows[index].r_sap,
                                          ssap,
                                          response) < 0) {
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
                 destroy_ipcp_flow(index);
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 LOG_DBGF("Could not send response.");
                 return -1;
@@ -1257,13 +1222,13 @@ static int eth_llc_ipcp_flow_dealloc(int port_id)
         int i;
         int ret;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&shim_data(_ipcp)->flows_lock);
 
         index = port_id_to_index(port_id);
         if (index < 0) {
                 pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return 0;
         }
 
@@ -1277,7 +1242,7 @@ static int eth_llc_ipcp_flow_dealloc(int port_id)
         pthread_rwlock_unlock(&shim_data(_ipcp)->flows_lock);
 
         ret = eth_llc_ipcp_port_dealloc(addr, sap);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (ret < 0)
                 LOG_DBGF("Could not notify remote.");
@@ -1353,8 +1318,6 @@ int main(int argc, char * argv[])
         _ipcp->ops = &eth_llc_ops;
         _ipcp->state = IPCP_INIT;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&shim_data(_ipcp)->mainloop, NULL,
@@ -1362,8 +1325,6 @@ int main(int argc, char * argv[])
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 close_logfile();
@@ -1372,6 +1333,12 @@ int main(int argc, char * argv[])
 
         pthread_join(shim_data(_ipcp)->mainloop, NULL);
 
+        pthread_cancel(shim_data(_ipcp)->sdu_reader);
+        pthread_cancel(shim_data(_ipcp)->sdu_writer);
+
+        pthread_join(shim_data(_ipcp)->sdu_writer, NULL);
+        pthread_join(shim_data(_ipcp)->sdu_reader, NULL);
+
         eth_llc_ipcp_data_destroy();
 
         free(_ipcp);
diff --git a/src/ipcpd/shim-udp/main.c b/src/ipcpd/shim-udp/main.c
index 672941da..292d347d 100644
--- a/src/ipcpd/shim-udp/main.c
+++ b/src/ipcpd/shim-udp/main.c
@@ -99,7 +99,8 @@ struct shim_ap_data {
         pthread_t             handler;
         pthread_t             sdu_reader;
 
-        bool                  fd_set_sync;
+        bool                  fd_set_mod;
+        pthread_cond_t        fd_set_cond;
         pthread_mutex_t       fd_set_lock;
 } * _ap_instance;
 
@@ -142,6 +143,7 @@ static int shim_ap_init()
         }
 
         pthread_rwlock_init(&_ap_instance->flows_lock, NULL);
+        pthread_cond_init(&_ap_instance->fd_set_cond, NULL);
         pthread_mutex_init(&_ap_instance->fd_set_lock, NULL);
 
         return 0;
@@ -154,7 +156,7 @@ void shim_ap_fini()
         if (_ap_instance == NULL)
                 return;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up AP while not in shutdown.");
@@ -173,7 +175,7 @@ void shim_ap_fini()
                         shm_ap_rbuff_close(_ap_instance->flows[i].rb);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         free(_ap_instance);
 }
@@ -197,31 +199,28 @@ static ssize_t ipcp_udp_flow_write(int fd, void * buf, size_t count)
         ssize_t index;
         struct rb_entry e;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                return -1; /* -ENOTENROLLED */
-        }
-
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
-        while ((index = shm_du_map_write(_ap_instance->dum,
-                                         _ap_instance->flows[fd].api,
-                                         0,
-                                         0,
-                                         (uint8_t *) buf,
-                                         count)) < 0)
-                ;
+        index = shm_du_map_write_b(_ap_instance->dum,
+                                   _ap_instance->flows[fd].api,
+                                   0,
+                                   0,
+                                   (uint8_t *) buf,
+                                   count);
+        if (index < 0) {
+                pthread_rwlock_unlock(&_ap_instance->flows_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
+                return -1;
+        }
 
         e.index   = index;
         e.port_id = _ap_instance->flows[fd].port_id;
 
-        while (shm_ap_rbuff_write(_ap_instance->flows[fd].rb, &e) < 0)
-                ;
+        shm_ap_rbuff_write(_ap_instance->flows[fd].rb, &e);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -292,40 +291,30 @@ struct ipcp_udp_data * ipcp_udp_data_create()
 
 static void set_fd(int fd)
 {
-        bool fd_wait = true;
-
         pthread_mutex_lock(&_ap_instance->fd_set_lock);
 
-        _ap_instance->fd_set_sync =  true;
+        _ap_instance->fd_set_mod = true;
         FD_SET(fd, &shim_data(_ipcp)->flow_fd_s);
 
-        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
-
+        while (_ap_instance->fd_set_mod)
+                pthread_cond_wait(&_ap_instance->fd_set_cond,
+                                  &_ap_instance->fd_set_lock);
 
-        while (fd_wait) {
-                pthread_mutex_lock(&_ap_instance->fd_set_lock);
-                fd_wait = _ap_instance->fd_set_sync;
-                pthread_mutex_unlock(&_ap_instance->fd_set_lock);
-        }
+        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
 }
 
 static void clr_fd(int fd)
 {
-        bool fd_wait = true;
-
         pthread_mutex_lock(&_ap_instance->fd_set_lock);
 
-        _ap_instance->fd_set_sync =  true;
+        _ap_instance->fd_set_mod = true;
         FD_CLR(fd, &shim_data(_ipcp)->flow_fd_s);
 
-        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
+        while (_ap_instance->fd_set_mod)
+                pthread_cond_wait(&_ap_instance->fd_set_cond,
+                                  &_ap_instance->fd_set_lock);
 
-        while (fd_wait) {
-                sched_yield();
-                pthread_mutex_lock(&_ap_instance->fd_set_lock);
-                fd_wait = _ap_instance->fd_set_sync;
-                pthread_mutex_unlock(&_ap_instance->fd_set_lock);
-        }
+        pthread_mutex_unlock(&_ap_instance->fd_set_lock);
 }
 
 
@@ -463,7 +452,7 @@ static int ipcp_udp_port_req(struct sockaddr_in * c_saddr,
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&_ap_instance->flows_lock);
 
         /* reply to IRM */
@@ -473,7 +462,7 @@ static int ipcp_udp_port_req(struct sockaddr_in * c_saddr,
 
         if (port_id < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Could not get port id from IRMd");
                 close(fd);
                 return -1;
@@ -484,7 +473,7 @@ static int ipcp_udp_port_req(struct sockaddr_in * c_saddr,
         _ap_instance->flows[fd].state   = FLOW_PENDING;
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Pending allocation request, port_id %d, UDP port (%d, %d).",
                 port_id, ntohs(f_saddr.sin_port), ntohs(c_saddr->sin_port));
@@ -506,20 +495,20 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
         LOG_DBG("Received reply for flow on udp port %d.",
                 ntohs(dst_udp_port));
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         fd = udp_port_to_fd(dst_udp_port);
         if (fd == -1) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Unknown flow on UDP port %d.", ntohs(dst_udp_port));
                 return -1; /* -EUNKNOWNFLOW */
         }
 
         if (_ap_instance->flows[fd].state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Flow on UDP port %d not pending.",
                          ntohs(dst_udp_port));
                 return -1; /* -EFLOWNOTPENDING */
@@ -538,7 +527,7 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
                                 (struct sockaddr *) &t_saddr,
                                 &t_saddr_len) < 0) {
                         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_DBG("Flow with port_id %d has no peer.", port_id);
                         return -1;
                 }
@@ -550,7 +539,7 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
                             (struct sockaddr *) &t_saddr,
                             sizeof(t_saddr)) < 0) {
                         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         close(fd);
                         return -1;
                 }
@@ -559,7 +548,7 @@ static int ipcp_udp_port_alloc_reply(int src_udp_port,
         }
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
 
         if ((ret = ipcp_flow_alloc_reply(getpid(),
@@ -582,13 +571,13 @@ static int ipcp_udp_flow_dealloc_req(int udp_port)
 
         struct shm_ap_rbuff * rb;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         fd = udp_port_to_fd(udp_port);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Could not find flow on UDP port %d.",
                          ntohs(udp_port));
                 return 0;
@@ -610,7 +599,7 @@ static int ipcp_udp_flow_dealloc_req(int udp_port)
         if (rb != NULL)
                 shm_ap_rbuff_close(rb);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         ipcp_flow_dealloc(0, port_id);
 
@@ -632,16 +621,11 @@ static void * ipcp_udp_listener()
                 int sfd = 0;
                 shim_udp_msg_t * msg = NULL;
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 sfd = shim_data(_ipcp)->s_fd;
 
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 memset(&buf, 0, SHIM_UDP_MSG_SIZE);
                 n = sizeof(c_saddr);
@@ -705,23 +689,17 @@ static void * ipcp_udp_sdu_reader()
         while (true) {
                 struct timeval tv = {0, FD_UPDATE_TIMEOUT};
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
-
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_rdlock(&_ap_instance->flows_lock);
-
                 pthread_mutex_lock(&_ap_instance->fd_set_lock);
 
                 read_fds = shim_data(_ipcp)->flow_fd_s;
-                _ap_instance->fd_set_sync = false;
+                _ap_instance->fd_set_mod = false;
+                pthread_cond_broadcast(&_ap_instance->fd_set_cond);
 
                 pthread_mutex_unlock(&_ap_instance->fd_set_lock);
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 if (select(FD_SETSIZE, &read_fds, NULL, NULL, &tv) <= 0) {
                         continue;
@@ -765,18 +743,13 @@ static void * ipcp_udp_sdu_loop(void * o)
                         continue;
                 }
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-                        return (void *) 1; /* -ENOTENROLLED */
-                }
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 len = shm_du_map_read((uint8_t **) &buf,
                                       _ap_instance->dum,
                                       e->index);
                 if (len <= 0) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         free(e);
                         continue;
                 }
@@ -786,7 +759,7 @@ static void * ipcp_udp_sdu_loop(void * o)
                 fd = port_id_to_fd(e->port_id);
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 if (fd == -1) {
                         free(e);
@@ -796,12 +769,12 @@ static void * ipcp_udp_sdu_loop(void * o)
                 if (send(fd, buf, len, 0) < 0)
                         LOG_ERR("Failed to send SDU.");
 
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
 
                 if (_ap_instance->dum != NULL)
                         shm_du_map_remove(_ap_instance->dum, e->index);
 
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 free(e);
         }
@@ -820,31 +793,14 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
         case SIGTERM:
         case SIGHUP:
                 if (info->si_pid == irmd_api) {
-                        bool clean_threads = false;
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_mutex_lock(&_ipcp->state_lock);
+                        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-                        if (_ipcp->state == IPCP_ENROLLED)
-                                clean_threads = true;
-
-                        _ipcp->state = IPCP_SHUTDOWN;
-
-                        pthread_mutex_unlock(&_ipcp->state_lock);
-
-                        if (clean_threads) {
-                                pthread_cancel(_ap_instance->handler);
-                                pthread_cancel(_ap_instance->sdu_reader);
-                                pthread_cancel(_ap_instance->sduloop);
-
-                                pthread_join(_ap_instance->sduloop, NULL);
-                                pthread_join(_ap_instance->handler, NULL);
-                                pthread_join(_ap_instance->sdu_reader, NULL);
-                        }
-
-                        pthread_cancel(_ap_instance->mainloop);
+                        ipcp_set_state(_ipcp, IPCP_SHUTDOWN);
 
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                 }
         default:
                 return;
@@ -916,10 +872,10 @@ static int ipcp_udp_bootstrap(struct dif_config * conf)
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_wrlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_INIT) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_INIT) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("IPCP in wrong state.");
                 close(fd);
                 return -1;
@@ -931,7 +887,7 @@ static int ipcp_udp_bootstrap(struct dif_config * conf)
 
         FD_CLR(shim_data(_ipcp)->s_fd, &shim_data(_ipcp)->flow_fd_s);
 
-        _ipcp->state = IPCP_ENROLLED;
+        ipcp_set_state(_ipcp, IPCP_ENROLLED);
 
         pthread_create(&_ap_instance->handler,
                        NULL,
@@ -947,7 +903,7 @@ static int ipcp_udp_bootstrap(struct dif_config * conf)
                        ipcp_udp_sdu_loop,
                        NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Bootstrapped shim IPCP over UDP with api %d.",
                 getpid());
@@ -1098,16 +1054,10 @@ static int ipcp_udp_name_reg(char * name)
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBG("Won't register with non-enrolled IPCP.");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to add %s to local registry.", name);
                 return -1;
         }
@@ -1117,7 +1067,7 @@ static int ipcp_udp_name_reg(char * name)
 
         dns_addr = shim_data(_ipcp)->dns_addr;
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (dns_addr != 0) {
                 ip_addr = shim_data(_ipcp)->ip_addr;
@@ -1136,14 +1086,14 @@ static int ipcp_udp_name_reg(char * name)
                         dnsstr, name, DNS_TTL, ipstr);
 
                 if (ddns_send(cmd)) {
-                        pthread_mutex_lock(&_ipcp->state_lock);
+                        pthread_rwlock_rdlock(&_ipcp->state_lock);
                         ipcp_data_del_reg_entry(_ipcp->data, name);
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         return -1;
                 }
         }
 #else
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 #endif
         LOG_DBG("Registered %s.", name);
 
@@ -1167,17 +1117,11 @@ static int ipcp_udp_name_unreg(char * name)
 #ifdef CONFIG_OUROBOROS_ENABLE_DNS
         /* unregister application with DNS server */
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBG("IPCP is not enrolled");
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         dns_addr = shim_data(_ipcp)->dns_addr;
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (dns_addr != 0) {
                 if (inet_ntop(AF_INET, &dns_addr, dnsstr, INET_ADDRSTRLEN)
@@ -1191,11 +1135,11 @@ static int ipcp_udp_name_unreg(char * name)
         }
 #endif
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -1253,10 +1197,10 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
                 return -1;
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+        if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Won't allocate flow with non-enrolled IPCP.");
                 close(fd);
                 return -1; /* -ENOTENROLLED */
@@ -1266,7 +1210,7 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
         dns_addr = shim_data(_ipcp)->dns_addr;
 
         if (dns_addr != 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 ip_addr = ddns_resolve(dst_name, dns_addr);
                 if (ip_addr == 0) {
@@ -1275,9 +1219,9 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
                         return -1;
                 }
 
-                pthread_mutex_lock(&_ipcp->state_lock);
-                if (_ipcp->state != IPCP_ENROLLED) {
-                        pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
+                if (ipcp_get_state(_ipcp) != IPCP_ENROLLED) {
+                        pthread_rwlock_unlock(&_ipcp->state_lock);
                         LOG_DBG("Won't allocate flow with non-enrolled IPCP.");
                         close(fd);
                         return -1; /* -ENOTENROLLED */
@@ -1320,13 +1264,13 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (ipcp_udp_port_alloc(ip_addr,
                                 f_saddr.sin_port,
                                 dst_name,
                                 src_ae_name) < 0) {
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
                 clr_fd(fd);
@@ -1340,7 +1284,7 @@ static int ipcp_udp_flow_alloc(pid_t         n_api,
                  _ap_instance->flows[fd].rb     = NULL;
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 close(fd);
                 return -1;
         }
@@ -1363,14 +1307,7 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
         if (response)
                 return 0;
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
-        if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
-                LOG_DBG("IPCP in wrong state.");
-                close(fd);
-                return -1; /* -ENOTENROLLED */
-        }
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
 
         /* awaken pending flow */
 
@@ -1379,14 +1316,14 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
         fd = port_id_to_fd(port_id);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Could not find flow with port_id %d.", port_id);
                 return -1;
         }
 
         if (_ap_instance->flows[fd].state != FLOW_PENDING) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Flow was not pending.");
                 return -1;
         }
@@ -1397,7 +1334,7 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
                 _ap_instance->flows[fd].state   = FLOW_NULL;
                 _ap_instance->flows[fd].port_id = -1;
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 return -1;
         }
 
@@ -1413,20 +1350,20 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
 
         _ap_instance->flows[fd].state = FLOW_ALLOCATED;
         _ap_instance->flows[fd].rb    = rb;
-        pthread_rwlock_unlock(&_ap_instance->flows_lock);
 
+        pthread_rwlock_unlock(&_ap_instance->flows_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         set_fd(fd);
 
         pthread_rwlock_unlock(&_ap_instance->flows_lock);
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         if (ipcp_udp_port_alloc_resp(r_saddr.sin_addr.s_addr,
                                      f_saddr.sin_port,
                                      r_saddr.sin_port,
                                      response) < 0) {
-                pthread_mutex_lock(&_ipcp->state_lock);
+                pthread_rwlock_rdlock(&_ipcp->state_lock);
                 pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
                 clr_fd(fd);
@@ -1439,7 +1376,7 @@ static int ipcp_udp_flow_alloc_resp(pid_t n_api,
                 _ap_instance->flows[fd].rb    = NULL;
 
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
 
                 return -1;
         }
@@ -1457,13 +1394,13 @@ static int ipcp_udp_flow_dealloc(int port_id)
         struct sockaddr_in    r_saddr;
         socklen_t             r_saddr_len = sizeof(r_saddr);
 
-        pthread_mutex_lock(&_ipcp->state_lock);
+        pthread_rwlock_rdlock(&_ipcp->state_lock);
         pthread_rwlock_rdlock(&_ap_instance->flows_lock);
 
         fd = port_id_to_fd(port_id);
         if (fd < 0) {
                 pthread_rwlock_unlock(&_ap_instance->flows_lock);
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Could not find flow with port_id %d.", port_id);
                 return 0;
         }
@@ -1484,7 +1421,7 @@ static int ipcp_udp_flow_dealloc(int port_id)
                 shm_ap_rbuff_close(rb);
 
         if (getpeername(fd, (struct sockaddr *) &r_saddr, &r_saddr_len) < 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 LOG_DBG("Flow with port_id %d has no peer.", port_id);
                 close(fd);
                 return 0;
@@ -1494,7 +1431,7 @@ static int ipcp_udp_flow_dealloc(int port_id)
         r_saddr.sin_port = LISTEN_PORT;
 
         if (connect(fd, (struct sockaddr *) &r_saddr, sizeof(r_saddr)) < 0) {
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 close(fd);
                 return 0 ;
         }
@@ -1502,12 +1439,12 @@ static int ipcp_udp_flow_dealloc(int port_id)
         if (ipcp_udp_port_dealloc(r_saddr.sin_addr.s_addr,
                                   remote_udp) < 0) {
                 LOG_DBG("Could not notify remote.");
-                pthread_mutex_unlock(&_ipcp->state_lock);
+                pthread_rwlock_unlock(&_ipcp->state_lock);
                 close(fd);
                 return 0;
         }
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
+        pthread_rwlock_unlock(&_ipcp->state_lock);
 
         close(fd);
 
@@ -1516,11 +1453,20 @@ static int ipcp_udp_flow_dealloc(int port_id)
         return 0;
 }
 
+static struct ipcp_ops udp_ops = {
+        .ipcp_bootstrap       = ipcp_udp_bootstrap,
+        .ipcp_enroll          = NULL,                       /* shim */
+        .ipcp_name_reg        = ipcp_udp_name_reg,
+        .ipcp_name_unreg      = ipcp_udp_name_unreg,
+        .ipcp_flow_alloc      = ipcp_udp_flow_alloc,
+        .ipcp_flow_alloc_resp = ipcp_udp_flow_alloc_resp,
+        .ipcp_flow_dealloc    = ipcp_udp_flow_dealloc
+};
+
 static struct ipcp * ipcp_udp_create()
 {
         struct ipcp * i;
         struct ipcp_udp_data * data;
-        struct ipcp_ops *      ops;
 
         i = ipcp_instance_create();
         if (i == NULL)
@@ -1532,23 +1478,8 @@ static struct ipcp * ipcp_udp_create()
                 return NULL;
         }
 
-        ops = malloc(sizeof(*ops));
-        if (ops == NULL) {
-                free(data);
-                free(i);
-                return NULL;
-        }
-
-        ops->ipcp_bootstrap       = ipcp_udp_bootstrap;
-        ops->ipcp_enroll          = NULL;                       /* shim */
-        ops->ipcp_name_reg        = ipcp_udp_name_reg;
-        ops->ipcp_name_unreg      = ipcp_udp_name_unreg;
-        ops->ipcp_flow_alloc      = ipcp_udp_flow_alloc;
-        ops->ipcp_flow_alloc_resp = ipcp_udp_flow_alloc_resp;
-        ops->ipcp_flow_dealloc    = ipcp_udp_flow_dealloc;
-
         i->data = (struct ipcp_data *) data;
-        i->ops  = ops;
+        i->ops  = &udp_ops;
 
         i->state = IPCP_INIT;
 
@@ -1599,16 +1530,12 @@ int main(int argc, char * argv[])
                 exit(EXIT_FAILURE);
         }
 
-        pthread_mutex_lock(&_ipcp->state_lock);
-
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
         pthread_create(&_ap_instance->mainloop, NULL, ipcp_main_loop, _ipcp);
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_mutex_unlock(&_ipcp->state_lock);
-
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
                 close_logfile();
@@ -1617,10 +1544,17 @@ int main(int argc, char * argv[])
 
         pthread_join(_ap_instance->mainloop, NULL);
 
+        pthread_cancel(_ap_instance->handler);
+        pthread_cancel(_ap_instance->sdu_reader);
+        pthread_cancel(_ap_instance->sduloop);
+
+        pthread_join(_ap_instance->sduloop, NULL);
+        pthread_join(_ap_instance->handler, NULL);
+        pthread_join(_ap_instance->sdu_reader, NULL);
+
         shim_ap_fini();
 
         ipcp_data_destroy(_ipcp->data);
-        free(_ipcp->ops);
         free(_ipcp);
 
         close_logfile();