lib, ipcpd, irmd: Wrap pthread unlocks for cleanup

This add an ouroboros/pthread.h header that wraps the
pthread_..._unlock() functions for cleanup using
pthread_cleanup_push() as this casting is not safe (and there were
definitely bad casts in the code). The close() function is now also
wrapped for cleanup in ouroboros/sockets.h.
This allows enabling more compiler checks.

Signed-off-by: Dimitri Staessens <[email protected]>
Signed-off-by: Sander Vrijders <[email protected]>

27 files changed, 136 insertions, 123 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dd26c828..e0c7bef3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -60,7 +60,8 @@ include(CompilerUtils)
 test_and_set_c_compiler_flag_global(-std=c89)
 test_and_set_c_compiler_flag_global(-Wall)
 # -Wextra may fail on clobbered warning due to pthread_cleanup
-# test_and_set_c_compiler_flag_global(-Wextra)
+test_and_set_c_compiler_flag_global(-Wno-clobbered)
+test_and_set_c_compiler_flag_global(-Wextra)
 # explicitly add other flags in -Wextra
 test_and_set_c_compiler_flag_global(-Wempty-body)
 test_and_set_c_compiler_flag_global(-Wignored-qualifiers)
@@ -75,14 +76,13 @@ test_and_set_c_compiler_flag_global(-Wuninitialized)
 test_and_set_c_compiler_flag_global(-Wshift-negative-value)
 test_and_set_c_compiler_flag_global(-Wunused-parameter)
 test_and_set_c_compiler_flag_global(-Wunused-but-set-parameter)
-# -Wextra parameters
 test_and_set_c_compiler_flag_global(-Werror)
 test_and_set_c_compiler_flag_global(-Wundef)
 test_and_set_c_compiler_flag_global(-Wpointer-arith)
 test_and_set_c_compiler_flag_global(-Wstrict-prototypes)
 test_and_set_c_compiler_flag_global(-Wvla)
-#Wswitch-default check fails on the swig-generated code
-#test_and_set_c_compiler_flag_global(-Wswitch-default)
+test_and_set_c_compiler_flag_global(-Wswitch-default)
+test_and_set_c_compiler_flag_global(-Wreturn-type)
 test_and_set_c_compiler_flag_global(-Wunreachable-code)
 test_and_set_c_compiler_flag_global(-Wdeclaration-after-statement)
 test_and_set_c_compiler_flag_global(-Winfinite-recursion)
diff --git a/include/ouroboros/pthread.h b/include/ouroboros/pthread.h
new file mode 100644
index 00000000..735557d2
--- /dev/null
+++ b/include/ouroboros/pthread.h
@@ -0,0 +1,39 @@
+/*
+ * Ouroboros - Copyright (C) 2016 - 2021
+ *
+ * Useful cleanup functions for pthreads
+ *
+ *    Dimitri Staessens <[email protected]>
+ *    Sander Vrijders   <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., http://www.fsf.org/about/contact/.
+ */
+
+#ifndef OUROBOROS_PTHREAD_H
+#define OUROBOROS_PTHREAD_H
+
+#include <pthread.h>
+
+/* various cleanup functions for pthread_cleanup_push */
+static void __attribute__((unused)) __cleanup_rwlock_unlock(void * rwlock)
+{
+        pthread_rwlock_unlock((pthread_rwlock_t *) rwlock);
+}
+
+static void __attribute__((unused)) __cleanup_mutex_unlock(void * mutex)
+{
+        pthread_mutex_unlock((pthread_mutex_t *) mutex);
+}
+
+#endif /* OUROBOROS_PTHREAD_H */
diff --git a/include/ouroboros/sockets.h.in b/include/ouroboros/sockets.h.in
index 857b1353..f1162163 100644
--- a/include/ouroboros/sockets.h.in
+++ b/include/ouroboros/sockets.h.in
@@ -65,4 +65,7 @@ qosspec_msg_t spec_to_msg(const qosspec_t * qs);
 
 qosspec_t     msg_to_spec(const qosspec_msg_t * msg);
 
+/* cleanup socket when cancelling thread */
+void __cleanup_close_ptr(void * o);
+
 #endif
diff --git a/src/ipcpd/broadcast/dt.c b/src/ipcpd/broadcast/dt.c
index 8b15008b..00476027 100644
--- a/src/ipcpd/broadcast/dt.c
+++ b/src/ipcpd/broadcast/dt.c
@@ -40,6 +40,7 @@
 #include <ouroboros/logs.h>
 #include <ouroboros/notifier.h>
 #include <ouroboros/utils.h>
+#include <ouroboros/pthread.h>
 
 #include "common/comp.h"
 #include "common/connmgr.h"
@@ -49,7 +50,6 @@
 #include <stdlib.h>
 #include <inttypes.h>
 #include <string.h>
-#include <pthread.h>
 
 struct nb {
         struct list_head next;
@@ -156,8 +156,7 @@ static void dt_packet(uint8_t * buf,
 
         pthread_rwlock_rdlock(&fwd.nbs_lock);
 
-        pthread_cleanup_push((void (*))(void *) pthread_rwlock_unlock,
-                             &fwd.nbs_lock);
+        pthread_cleanup_push(__cleanup_rwlock_unlock, &fwd.nbs_lock);
 
         list_for_each(p, &fwd.nbs) {
                 struct nb * nb = list_entry(p, struct nb, next);
@@ -168,6 +167,11 @@ static void dt_packet(uint8_t * buf,
         pthread_cleanup_pop(true);
 }
 
+static void __cleanup_fqueue_destroy(void * fq)
+{
+        fqueue_destroy((fqueue_t *) fq);
+}
+
 static void * dt_reader(void * o)
 {
         fqueue_t *   fq;
@@ -182,8 +186,7 @@ static void * dt_reader(void * o)
         if (fq == NULL)
                 return (void *) -1;
 
-        pthread_cleanup_push((void (*) (void *)) fqueue_destroy,
-                             (void *) fq);
+        pthread_cleanup_push(__cleanup_fqueue_destroy, (void *) fq);
 
         while (true) {
                 ret = fevent(fwd.set, fq, NULL);
diff --git a/src/ipcpd/common/connmgr.c b/src/ipcpd/common/connmgr.c
index 9002467c..53a66992 100644
--- a/src/ipcpd/common/connmgr.c
+++ b/src/ipcpd/common/connmgr.c
@@ -28,11 +28,11 @@
 #include <ouroboros/list.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/notifier.h>
+#include <ouroboros/pthread.h>
 
 #include "connmgr.h"
 #include "ipcp.h"
 
-#include <pthread.h>
 #include <string.h>
 #include <stdlib.h>
 #include <assert.h>
@@ -493,8 +493,7 @@ int connmgr_wait(enum comp_id  id,
 
         pthread_mutex_lock(&comp->lock);
 
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) &comp->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &comp->lock);
 
         while (list_is_empty(&comp->pending))
                 pthread_cond_wait(&comp->cond, &comp->lock);
diff --git a/src/ipcpd/eth/eth.c b/src/ipcpd/eth/eth.c
index 13a4a7e3..932034d5 100644
--- a/src/ipcpd/eth/eth.c
+++ b/src/ipcpd/eth/eth.c
@@ -48,13 +48,13 @@
 #include <ouroboros/logs.h>
 #include <ouroboros/time_utils.h>
 #include <ouroboros/fccntl.h>
+#include <ouroboros/pthread.h>
 
 #include "ipcp.h"
 #include "shim-data.h"
 
 #include <signal.h>
 #include <stdlib.h>
-#include <pthread.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <string.h>
@@ -807,8 +807,7 @@ static void * eth_ipcp_mgmt_handler(void * o)
 
         (void) o;
 
-        pthread_cleanup_push((void (*)(void *)) pthread_mutex_unlock,
-                             (void *) &eth_data.mgmt_lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &eth_data.mgmt_lock);
 
         while (true) {
                 ret = 0;
@@ -1162,12 +1161,6 @@ static void change_flows_state(bool up)
         pthread_rwlock_unlock(&eth_data.flows_lock);
 }
 
-static void close_ptr(void * o)
-{
-        close(*((int *) o));
-}
-
-
 static void * eth_ipcp_if_monitor(void * o)
 {
         int                fd;
@@ -1188,7 +1181,7 @@ static void * eth_ipcp_if_monitor(void * o)
                 return (void *) -1;
         }
 
-        pthread_cleanup_push(close_ptr, &fd);
+        pthread_cleanup_push(__cleanup_close_ptr, &fd);
 
         while (true) {
                 status = recvmsg(fd, &msg, 0);
diff --git a/src/ipcpd/ipcp.c b/src/ipcpd/ipcp.c
index 8c9d3525..56cdb5d2 100644
--- a/src/ipcpd/ipcp.c
+++ b/src/ipcpd/ipcp.c
@@ -54,6 +54,7 @@
 #include <ouroboros/bitmap.h>
 #include <ouroboros/np1_flow.h>
 #include <ouroboros/rib.h>
+#include <ouroboros/pthread.h>
 
 #include "ipcp.h"
 
@@ -207,11 +208,6 @@ static struct rib_ops r_ops = {
         .getattr = ipcp_stat_getattr
 };
 
-static void close_ptr(void * o)
-{
-        close(*((int *) o));
-}
-
 static void * acceptloop(void * o)
 {
         int            csockfd;
@@ -239,7 +235,7 @@ static void * acceptloop(void * o)
                         break;
                 }
 
-                pthread_cleanup_push(close_ptr, &csockfd);
+                pthread_cleanup_push(__cleanup_close_ptr, &csockfd);
                 pthread_cleanup_push(free, cmd);
 
                 cmd->len = read(csockfd, cmd->cbuf, SOCK_BUF_SIZE);
@@ -295,8 +291,7 @@ static void * mainloop(void * o)
 
                 pthread_mutex_lock(&ipcpi.cmd_lock);
 
-                pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                                     &ipcpi.cmd_lock);
+                pthread_cleanup_push(__cleanup_mutex_unlock, &ipcpi.cmd_lock);
 
                 while (list_is_empty(&ipcpi.cmds))
                         pthread_cond_wait(&ipcpi.cmd_cond, &ipcpi.cmd_lock);
@@ -318,7 +313,7 @@ static void * mainloop(void * o)
 
                 tpm_dec(ipcpi.tpm);
 
-                pthread_cleanup_push(close_ptr, &sfd);
+                pthread_cleanup_push(__cleanup_close_ptr, &sfd);
                 pthread_cleanup_push(free_msg, msg);
 
                 switch (msg->code) {
@@ -665,7 +660,7 @@ static void * mainloop(void * o)
 
                 ipcp_msg__pack(&ret_msg, buffer.data);
 
-                pthread_cleanup_push(close_ptr, &sfd);
+                pthread_cleanup_push(__cleanup_close_ptr, &sfd);
 
                 if (write(sfd, buffer.data, buffer.len) == -1)
                         log_warn("Failed to send reply message");
@@ -955,8 +950,7 @@ int ipcp_wait_state(enum ipcp_state         state,
 
         pthread_mutex_lock(&ipcpi.state_mtx);
 
-        pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                             &ipcpi.state_mtx);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &ipcpi.state_mtx);
 
         while (ipcpi.state != state
                && ipcpi.state != IPCP_SHUTDOWN
diff --git a/src/ipcpd/udp/main.c b/src/ipcpd/udp/main.c
index 13db0c1f..03de24ab 100644
--- a/src/ipcpd/udp/main.c
+++ b/src/ipcpd/udp/main.c
@@ -39,6 +39,7 @@
 #include <ouroboros/fqueue.h>
 #include <ouroboros/errno.h>
 #include <ouroboros/logs.h>
+#include <ouroboros/pthread.h>
 
 #include "ipcp.h"
 #include "shim-data.h"
@@ -51,7 +52,6 @@
 #include <netinet/in.h>
 #include <signal.h>
 #include <stdlib.h>
-#include <pthread.h>
 #include <sys/wait.h>
 #include <fcntl.h>
 
@@ -393,8 +393,7 @@ static void * ipcp_udp_mgmt_handler(void * o)
 {
         (void) o;
 
-        pthread_cleanup_push((void (*)(void *)) pthread_mutex_unlock,
-                             (void *) &udp_data.mgmt_lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &udp_data.mgmt_lock);
 
         while (true) {
                 struct mgmt_frame * frame;
@@ -485,9 +484,14 @@ static void * ipcp_udp_packet_reader(void * o)
         return 0;
 }
 
-static void cleanup_writer(void * o)
+static void cleanup_fqueue(void * fq)
 {
-        fqueue_destroy((fqueue_t *) o);
+        fqueue_destroy((fqueue_t *) fq);
+}
+
+static void cleanup_sdb(void * sdb)
+{
+        ipcp_sdb_release((struct shm_du_buff *) sdb);
 }
 
 static void * ipcp_udp_packet_writer(void * o)
@@ -502,7 +506,7 @@ static void * ipcp_udp_packet_writer(void * o)
 
         ipcp_lock_to_core();
 
-        pthread_cleanup_push(cleanup_writer, fq);
+        pthread_cleanup_push(cleanup_fqueue, fq);
 
         while (true) {
                 struct sockaddr_in saddr;
@@ -545,8 +549,7 @@ static void * ipcp_udp_packet_writer(void * o)
 
                         memcpy(buf, &eid, sizeof(eid));
 
-                        pthread_cleanup_push((void (*)(void *))
-                                             ipcp_sdb_release, (void *) sdb);
+                        pthread_cleanup_push(cleanup_sdb, sdb);
 
                         if (sendto(udp_data.s_fd, buf, len + OUR_HEADER_LEN,
                                    MSG_CONFIRM,
diff --git a/src/ipcpd/unicast/dht.c b/src/ipcpd/unicast/dht.c
index 3dc592db..2b668f9f 100644
--- a/src/ipcpd/unicast/dht.c
+++ b/src/ipcpd/unicast/dht.c
@@ -42,12 +42,12 @@
 #include <ouroboros/time_utils.h>
 #include <ouroboros/tpm.h>
 #include <ouroboros/utils.h>
+#include <ouroboros/pthread.h>
 
 #include "common/connmgr.h"
 #include "dht.h"
 #include "dt.h"
 
-#include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
@@ -308,8 +308,7 @@ int dht_wait_running(struct dht * dht)
 
         pthread_mutex_lock(&dht->mtx);
 
-        pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                             &dht->mtx);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &dht->mtx);
 
         while (dht->state == DHT_JOINING)
                 pthread_cond_wait(&dht->cond, &dht->mtx);
@@ -466,8 +465,7 @@ static int kad_req_wait(struct kad_req * req,
 
         req->state = REQ_PENDING;
 
-        pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                             &req->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &req->lock);
 
         while (req->state == REQ_PENDING && ret != -ETIMEDOUT)
                 ret = -pthread_cond_timedwait(&req->cond, &req->lock, &abs);
@@ -819,8 +817,7 @@ static void lookup_update(struct dht *    dht,
                 return;
         }
 
-        pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                             &lu->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &lu->lock);
 
         while (lu->state == LU_INIT) {
                 pthread_rwlock_unlock(&dht->lock);
@@ -967,7 +964,7 @@ static void lookup_set_state(struct lookup *   lu,
         pthread_mutex_unlock(&lu->lock);
 }
 
-static void cleanup_wait(void * o)
+static void cancel_lookup_wait(void * o)
 {
         struct lookup * lu = (struct lookup *) o;
         lu->state = LU_NULL;
@@ -991,7 +988,7 @@ static enum lookup_state lookup_wait(struct lookup * lu)
         if (lu->state == LU_INIT || lu->state == LU_UPDATE)
                 lu->state = LU_PENDING;
 
-        pthread_cleanup_push(cleanup_wait, lu);
+        pthread_cleanup_push(cancel_lookup_wait, lu);
 
         while (lu->state == LU_PENDING && ret != -ETIMEDOUT)
                 ret = -pthread_cond_timedwait(&lu->cond, &lu->lock, &abs);
@@ -2421,8 +2418,7 @@ static void * dht_handle_packet(void * o)
 
                 pthread_mutex_lock(&dht->mtx);
 
-                pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                                     &dht->mtx);
+                pthread_cleanup_push(__cleanup_mutex_unlock, &dht->mtx);
 
                 while (list_is_empty(&dht->cmds))
                         pthread_cond_wait(&dht->cond, &dht->mtx);
diff --git a/src/ipcpd/unicast/fa.c b/src/ipcpd/unicast/fa.c
index b3b9b42c..3ebdc1f1 100644
--- a/src/ipcpd/unicast/fa.c
+++ b/src/ipcpd/unicast/fa.c
@@ -38,6 +38,7 @@
 #include <ouroboros/ipcp-dev.h>
 #include <ouroboros/rib.h>
 #include <ouroboros/random.h>
+#include <ouroboros/pthread.h>
 
 #include "dir.h"
 #include "fa.h"
@@ -47,7 +48,6 @@
 #include "ca.h"
 
 #include <inttypes.h>
-#include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -442,8 +442,7 @@ static void * fa_handle_packet(void * o)
 
                 pthread_mutex_lock(&fa.mtx);
 
-                pthread_cleanup_push((void (*)(void *)) pthread_mutex_unlock,
-                                     &fa.mtx);
+                pthread_cleanup_push(__cleanup_mutex_unlock, &fa.mtx);
 
                 while (list_is_empty(&fa.cmds))
                         pthread_cond_wait(&fa.cond, &fa.mtx);
diff --git a/src/ipcpd/unicast/pol/link_state.c b/src/ipcpd/unicast/pol/link_state.c
index e8983ec6..cca85d63 100644
--- a/src/ipcpd/unicast/pol/link_state.c
+++ b/src/ipcpd/unicast/pol/link_state.c
@@ -38,6 +38,7 @@
 #include <ouroboros/list.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/notifier.h>
+#include <ouroboros/pthread.h>
 #include <ouroboros/rib.h>
 #include <ouroboros/utils.h>
 
@@ -52,7 +53,6 @@
 #include <stdlib.h>
 #include <inttypes.h>
 #include <string.h>
-#include <pthread.h>
 
 #define RECALC_TIME    4
 #define LS_UPDATE_TIME 15
@@ -648,8 +648,7 @@ static void * lsupdate(void * o)
 
                 pthread_rwlock_wrlock(&ls.db_lock);
 
-                pthread_cleanup_push((void (*) (void *)) pthread_rwlock_unlock,
-                                     (void *) &ls.db_lock);
+                pthread_cleanup_push(__cleanup_rwlock_unlock, &ls.db_lock);
 
                 list_for_each_safe(p, h, &ls.db) {
                         struct adjacency * adj;
@@ -709,8 +708,7 @@ static void forward_lsm(uint8_t * buf,
 
         pthread_rwlock_rdlock(&ls.db_lock);
 
-        pthread_cleanup_push((void (*))(void *) pthread_rwlock_unlock,
-                             &ls.db_lock);
+        pthread_cleanup_push(__cleanup_rwlock_unlock, &ls.db_lock);
 
         list_for_each(p, &ls.nbs) {
                 struct nb * nb = list_entry(p, struct nb, next);
@@ -721,6 +719,11 @@ static void forward_lsm(uint8_t * buf,
         pthread_cleanup_pop(true);
 }
 
+static void cleanup_fqueue(void * fq)
+{
+        fqueue_destroy((fqueue_t *) fq);
+}
+
 static void * lsreader(void * o)
 {
         fqueue_t *   fq;
@@ -739,8 +742,7 @@ static void * lsreader(void * o)
         if (fq == NULL)
                 return (void *) -1;
 
-        pthread_cleanup_push((void (*) (void *)) fqueue_destroy,
-                             (void *) fq);
+        pthread_cleanup_push(cleanup_fqueue, fq);
 
         while (true) {
                 ret = fevent(ls.mgmt_set, fq, NULL);
@@ -813,8 +815,7 @@ static void handle_event(void *       self,
         case NOTIFY_DT_CONN_ADD:
                 pthread_rwlock_rdlock(&ls.db_lock);
 
-                pthread_cleanup_push((void (*) (void *)) pthread_rwlock_unlock,
-                                     (void *) &ls.db_lock);
+                pthread_cleanup_push(__cleanup_rwlock_unlock, &ls.db_lock);
 
                 send_lsm(ipcpi.dt_addr, c->conn_info.addr, 0);
                 pthread_cleanup_pop(true);
diff --git a/src/irmd/ipcp.c b/src/irmd/ipcp.c
index a6b0d20a..ae5325c5 100644
--- a/src/irmd/ipcp.c
+++ b/src/irmd/ipcp.c
@@ -44,11 +44,6 @@
 #include <sys/time.h>
 #include <spawn.h>
 
-static void close_ptr(void * o)
-{
-        close(*(int *) o);
-}
-
 ipcp_msg_t * send_recv_ipcp_msg(pid_t        pid,
                                 ipcp_msg_t * msg)
 {
@@ -111,7 +106,7 @@ ipcp_msg_t * send_recv_ipcp_msg(pid_t        pid,
                       (void *) &tv, sizeof(tv)))
                log_warn("Failed to set timeout on socket.");
 
-       pthread_cleanup_push(close_ptr, (void *) &sockfd);
+       pthread_cleanup_push(__cleanup_close_ptr, (void *) &sockfd);
 
        ipcp_msg__pack(msg, buf);
 
diff --git a/src/irmd/irm_flow.c b/src/irmd/irm_flow.c
index 92bf74ef..75df7a80 100644
--- a/src/irmd/irm_flow.c
+++ b/src/irmd/irm_flow.c
@@ -29,6 +29,7 @@
 #include <ouroboros/errno.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/time_utils.h>
+#include <ouroboros/pthread.h>
 
 #include "irm_flow.h"
 
@@ -195,8 +196,7 @@ int irm_flow_wait_state(struct irm_flow * f,
 
         assert(f->state != FLOW_NULL);
 
-        pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                             &f->state_lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &f->state_lock);
 
         while (!(f->state == state ||
                  f->state == FLOW_DESTROY ||
diff --git a/src/irmd/main.c b/src/irmd/main.c
index 31f244d9..471e9c36 100644
--- a/src/irmd/main.c
+++ b/src/irmd/main.c
@@ -45,6 +45,7 @@
 #include <ouroboros/tpm.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/version.h>
+#include <ouroboros/pthread.h>
 
 #include "utils.h"
 #include "registry.h"
@@ -58,7 +59,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
-#include <pthread.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <spawn.h>
@@ -1969,11 +1969,6 @@ void * irm_sanitize(void * o)
         }
 }
 
-static void close_ptr(void * o)
-{
-        close(*((int *) o));
-}
-
 static void * acceptloop(void * o)
 {
         int            csockfd;
@@ -2000,7 +1995,7 @@ static void * acceptloop(void * o)
                         break;
                 }
 
-                pthread_cleanup_push(close_ptr, &csockfd);
+                pthread_cleanup_push(__cleanup_close_ptr, &csockfd);
                 pthread_cleanup_push(free, cmd);
 
                 cmd->len = read(csockfd, cmd->cbuf, SOCK_BUF_SIZE);
@@ -2063,8 +2058,7 @@ static void * mainloop(void * o)
                 pthread_mutex_lock(&irmd.cmd_lock);
 
                 pthread_cleanup_push(free_msg, ret_msg);
-                pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                                     &irmd.cmd_lock);
+                pthread_cleanup_push(__cleanup_mutex_unlock, &irmd.cmd_lock);
 
                 while (list_is_empty(&irmd.cmds))
                         pthread_cond_wait(&irmd.cmd_cond, &irmd.cmd_lock);
@@ -2096,7 +2090,7 @@ static void * mainloop(void * o)
                         timeo = &ts;
                 }
 
-                pthread_cleanup_push(close_ptr, &sfd);
+                pthread_cleanup_push(__cleanup_close_ptr, &sfd);
                 pthread_cleanup_push(free_msg, msg);
                 pthread_cleanup_push(free_msg, ret_msg);
 
@@ -2271,7 +2265,7 @@ static void * mainloop(void * o)
                 ret_msg->qosspec = NULL;
                 irm_msg__free_unpacked(ret_msg, NULL);
 
-                pthread_cleanup_push(close_ptr, &sfd);
+                pthread_cleanup_push(__cleanup_close_ptr, &sfd);
 
                 if (write(sfd, buffer.data, buffer.len) == -1)
                         if (result != -EIRMD)
diff --git a/src/irmd/registry.c b/src/irmd/registry.c
index f0bd3e62..69e3ae97 100644
--- a/src/irmd/registry.c
+++ b/src/irmd/registry.c
@@ -33,6 +33,7 @@
 #include <ouroboros/errno.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/time_utils.h>
+#include <ouroboros/pthread.h>
 
 #include "registry.h"
 #include "utils.h"
@@ -431,8 +432,7 @@ int reg_entry_leave_state(struct reg_entry *  e,
 
         pthread_mutex_lock(&e->state_lock);
 
-        pthread_cleanup_push((void *)(void *) pthread_mutex_unlock,
-                             &e->state_lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &e->state_lock);
 
         while (e->state == state && ret != -ETIMEDOUT)
                 if (timeout)
diff --git a/src/lib/dev.c b/src/lib/dev.c
index 032d57f5..ec4561b2 100644
--- a/src/lib/dev.c
+++ b/src/lib/dev.c
@@ -37,6 +37,7 @@
 #include <ouroboros/sockets.h>
 #include <ouroboros/fccntl.h>
 #include <ouroboros/bitmap.h>
+#include <ouroboros/pthread.h>
 #include <ouroboros/random.h>
 #include <ouroboros/shm_flow_set.h>
 #include <ouroboros/shm_rdrbuff.h>
diff --git a/src/lib/frct.c b/src/lib/frct.c
index 615d499c..7aaf037c 100644
--- a/src/lib/frct.c
+++ b/src/lib/frct.c
@@ -413,8 +413,7 @@ static int __frcti_window_wait(struct frcti *    frcti,
                         frcti->open   = false;
                 }
 
-                pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                                     (void *) &frcti->mtx);
+                pthread_cleanup_push(__cleanup_mutex_unlock, &frcti->mtx);
 
                 ret = -pthread_cond_timedwait(&frcti->cond,
                                               &frcti->mtx,
diff --git a/src/lib/notifier.c b/src/lib/notifier.c
index 06f15681..1f600765 100644
--- a/src/lib/notifier.c
+++ b/src/lib/notifier.c
@@ -25,8 +25,9 @@
 #include <ouroboros/errno.h>
 #include <ouroboros/notifier.h>
 #include <ouroboros/list.h>
+#include <ouroboros/utils.h>
+#include <ouroboros/pthread.h>
 
-#include <pthread.h>
 #include <stdlib.h>
 
 struct listener {
@@ -75,8 +76,7 @@ void notifier_event(int          event,
 
         pthread_rwlock_rdlock(&notifier.lock);
 
-        pthread_cleanup_push((void (*) (void *)) pthread_rwlock_unlock,
-                             (void *) &notifier.lock)
+        pthread_cleanup_push(__cleanup_rwlock_unlock, &notifier.lock)
 
         list_for_each(p, &notifier.listeners) {
                 struct listener * l = list_entry(p, struct listener, next);
diff --git a/src/lib/shm_flow_set.c b/src/lib/shm_flow_set.c
index 553a7a78..5a9bee6c 100644
--- a/src/lib/shm_flow_set.c
+++ b/src/lib/shm_flow_set.c
@@ -28,8 +28,8 @@
 #include <ouroboros/time_utils.h>
 #include <ouroboros/shm_flow_set.h>
 #include <ouroboros/errno.h>
+#include <ouroboros/pthread.h>
 
-#include <pthread.h>
 #include <sys/mman.h>
 #include <fcntl.h>
 #include <stdlib.h>
@@ -346,8 +346,7 @@ ssize_t shm_flow_set_wait(const struct shm_flow_set * set,
                 pthread_mutex_consistent(set->lock);
 #endif
 
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) set->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, set->lock);
 
         while (set->heads[idx] == 0 && ret != -ETIMEDOUT) {
                 if (abstime != NULL) {
diff --git a/src/lib/shm_rbuff.c b/src/lib/shm_rbuff.c
index b75477ba..361d5bb0 100644
--- a/src/lib/shm_rbuff.c
+++ b/src/lib/shm_rbuff.c
@@ -29,8 +29,8 @@
 #include <ouroboros/time_utils.h>
 #include <ouroboros/errno.h>
 #include <ouroboros/fccntl.h>
+#include <ouroboros/pthread.h>
 
-#include <pthread.h>
 #include <sys/mman.h>
 #include <fcntl.h>
 #include <stdlib.h>
diff --git a/src/lib/shm_rbuff_ll.c b/src/lib/shm_rbuff_ll.c
index 1a81b926..eef8a2fb 100644
--- a/src/lib/shm_rbuff_ll.c
+++ b/src/lib/shm_rbuff_ll.c
@@ -100,8 +100,7 @@ int shm_rbuff_write_b(struct shm_rbuff *      rb,
                 goto err;
         }
 
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) rb->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, rb->lock);
 
         while (!shm_rbuff_free(rb) && ret != -ETIMEDOUT) {
                 if (abstime != NULL)
@@ -174,8 +173,7 @@ ssize_t shm_rbuff_read_b(struct shm_rbuff *      rb,
         if (pthread_mutex_lock(rb->lock) == EOWNERDEAD)
                 pthread_mutex_consistent(rb->lock);
 #endif
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) rb->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, rb->lock);
 
         while (shm_rbuff_empty(rb) && (idx != -ETIMEDOUT)) {
                 if (abstime != NULL)
@@ -230,8 +228,7 @@ void shm_rbuff_fini(struct shm_rbuff * rb)
                 pthread_mutex_consistent(rb->lock);
 #endif
 
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) rb->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, rb->lock);
 
         while (!shm_rbuff_empty(rb))
 #ifndef HAVE_ROBUST_MUTEX
diff --git a/src/lib/shm_rbuff_pthr.c b/src/lib/shm_rbuff_pthr.c
index aa79b477..cedbc7b1 100644
--- a/src/lib/shm_rbuff_pthr.c
+++ b/src/lib/shm_rbuff_pthr.c
@@ -106,8 +106,7 @@ int shm_rbuff_write_b(struct shm_rbuff *      rb,
                 goto err;
         }
 
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) rb->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, rb->lock);
 
         while (!shm_rbuff_free(rb)
                && ret != -ETIMEDOUT
@@ -186,8 +185,7 @@ ssize_t shm_rbuff_read_b(struct shm_rbuff *      rb,
                 return -EFLOWDOWN;
         }
 
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) rb->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, rb->lock);
 
         while (shm_rbuff_empty(rb)
                && (idx != -ETIMEDOUT)
@@ -263,8 +261,7 @@ void shm_rbuff_fini(struct shm_rbuff * rb)
         if (pthread_mutex_lock(rb->lock) == EOWNERDEAD)
                 pthread_mutex_consistent(rb->lock);
 #endif
-        pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock,
-                             (void *) rb->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, rb->lock);
 
         while (!shm_rbuff_empty(rb))
 #ifndef HAVE_ROBUST_MUTEX
diff --git a/src/lib/shm_rdrbuff.c b/src/lib/shm_rdrbuff.c
index 2c966376..e3552100 100644
--- a/src/lib/shm_rdrbuff.c
+++ b/src/lib/shm_rdrbuff.c
@@ -28,8 +28,8 @@
 #include <ouroboros/shm_rdrbuff.h>
 #include <ouroboros/shm_du_buff.h>
 #include <ouroboros/time_utils.h>
+#include <ouroboros/pthread.h>
 
-#include <pthread.h>
 #include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
@@ -392,8 +392,7 @@ ssize_t shm_rdrbuff_alloc_b(struct shm_rdrbuff *    rdrb,
         if (pthread_mutex_lock(rdrb->lock) == EOWNERDEAD)
                 sanitize(rdrb);
 #endif
-        pthread_cleanup_push((void (*) (void *)) pthread_mutex_unlock,
-                             (void *) rdrb->lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, rdrb->lock);
 
 #ifdef SHM_RDRB_MULTI_BLOCK
         if (blocks + *rdrb->head > (SHM_BUFFER_SIZE))
diff --git a/src/lib/sockets.c b/src/lib/sockets.c
index 391d41d5..8179d2b3 100644
--- a/src/lib/sockets.c
+++ b/src/lib/sockets.c
@@ -40,6 +40,11 @@
 #define SOCK_TYPE SOCK_SEQPACKET
 #endif
 
+void __cleanup_close_ptr(void * fd)
+{
+        close(*(int *) fd);
+}
+
 int client_socket_open(char * file_name)
 {
         int sockfd;
@@ -95,11 +100,6 @@ int server_socket_open(char * file_name)
         return sockfd;
 }
 
-static void close_ptr(void * o)
-{
-        close(*(int *) o);
-}
-
 irm_msg_t * send_recv_irm_msg(irm_msg_t * msg)
 {
         int         sockfd;
@@ -117,7 +117,7 @@ irm_msg_t * send_recv_irm_msg(irm_msg_t * msg)
                 return NULL;
         }
 
-        pthread_cleanup_push(close_ptr, &sockfd);
+        pthread_cleanup_push(__cleanup_close_ptr, &sockfd);
 
         irm_msg__pack(msg, buf);
 
diff --git a/src/lib/timerwheel.c b/src/lib/timerwheel.c
index 55433a58..3c1a44b4 100644
--- a/src/lib/timerwheel.c
+++ b/src/lib/timerwheel.c
@@ -147,8 +147,7 @@ static void timerwheel_move(void)
 
         pthread_mutex_lock(&rw.lock);
 
-        pthread_cleanup_push((void (*) (void *)) pthread_mutex_unlock,
-                             (void *) &rw.lock);
+        pthread_cleanup_push(__cleanup_mutex_unlock, &rw.lock);
 
         clock_gettime(PTHREAD_COND_CLOCK, &now);
 
diff --git a/src/tools/ocbr/ocbr_server.c b/src/tools/ocbr/ocbr_server.c
index 7f347971..05659dc4 100644
--- a/src/tools/ocbr/ocbr_server.c
+++ b/src/tools/ocbr/ocbr_server.c
@@ -142,6 +142,11 @@ static void handle_flow(int fd)
         flow_dealloc(fd);
 }
 
+static void __cleanup_mutex_unlock(void * mutex)
+{
+        pthread_mutex_unlock((pthread_mutex_t *) mutex);
+}
+
 static void * worker(void * o)
 {
         int cli_fd;
@@ -150,8 +155,7 @@ static void * worker(void * o)
 
         while (true) {
                 pthread_mutex_lock(&fds_lock);
-                pthread_cleanup_push((void(*)(void *)) pthread_mutex_unlock,
-                                     (void *) &fds_lock);
+                pthread_cleanup_push(__cleanup_mutex_unlock, &fds_lock);
                 while (fds[fds_index] == -1)
                         pthread_cond_wait(&fds_signal, &fds_lock);
 
@@ -184,8 +188,7 @@ static void * listener(void * o)
 
         while (true) {
                 pthread_mutex_lock(&fds_lock);
-                pthread_cleanup_push((void(*)(void *)) pthread_mutex_unlock,
-                                     (void *) &fds_lock);
+                pthread_cleanup_push(__cleanup_mutex_unlock, &fds_lock);
 
                 while (fds_count == THREADS_SIZE) {
                         printf("Can't accept any more flows, waiting.\n");
diff --git a/src/tools/oping/oping_client.c b/src/tools/oping/oping_client.c
index a310338d..0ab5a4af 100644
--- a/src/tools/oping/oping_client.c
+++ b/src/tools/oping/oping_client.c
@@ -163,7 +163,7 @@ void * writer(void * o)
                 printf("Pinging %s with %d bytes of data (%u packets):\n\n",
                        client.s_apn, client.size, client.count);
 
-        pthread_cleanup_push((void (*) (void *)) free, buf);
+        pthread_cleanup_push(free, buf);
 
         while (!stop && client.sent < client.count) {
                 nanosleep(&wait, NULL);