ipcpd: Deprecate gam as autonomous component

The graph adjacency manager has been deprecated in favor of providing
an external interface into the connectivity manager so that
adjacencies can be controlled from the command line, user scripts or
user applications.

The gam and its associated policies were removed from the normal IPCP
and the IRM configuration tools. The "/members" part of the RIB was
deprecated. Removal of the gam means that initial connectivity based
on changes in the RIB can't be provided, so some changes were
required throughout the normal IPCP.

The enrollment procedure was revised to establish its own
connectivity. First, it gets boot information from a peer by
establishing a connection to the remote enrollment component and
downloading the IPCP configuratoin. This is now done using its own
protocol buffers message in anticipation of deprecation of the RIB and
CDAP for communication within a DIF.

After the boot information is downloaded, it establishes a data
transfer flow for enrolling the directory (DHT). After the DHT has
enrolled, it signals the peer to that enrollment is done, and the data
transfer connection is torn down.

Signaling connections is done via the nbs struct, which is now passed
to the connmgr, which enables control of the connectivity graph from
external sources.

1 files changed, 240 insertions, 211 deletions

diff --git a/src/ipcpd/normal/enroll.c b/src/ipcpd/normal/enroll.c
index a33239a0..2f7dd9bc 100644
--- a/src/ipcpd/normal/enroll.c
+++ b/src/ipcpd/normal/enroll.c
@@ -22,19 +22,19 @@
 
 #define _POSIX_C_SOURCE 199309L
 
-#define OUROBOROS_PREFIX "enrollment"
+#define OUROBOROS_PREFIX "Enrollment"
 
 #include <ouroboros/endian.h>
 #include <ouroboros/errno.h>
-#include <ouroboros/cdap.h>
 #include <ouroboros/time_utils.h>
 #include <ouroboros/dev.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/rib.h>
 #include <ouroboros/errno.h>
 
-#include "ae.h"
 #include "connmgr.h"
+#include "enroll.h"
+#include "ipcp.h"
 #include "ribconfig.h"
 
 #include <assert.h>
@@ -42,302 +42,331 @@
 #include <string.h>
 #include <pthread.h>
 
-/* Symbolic, will return current time */
-#define TIME_NAME               "localtime"
-#define TIME_PATH               DLR TIME_NAME
+#include "enroll.pb-c.h"
+typedef EnrollMsg enroll_msg_t;
+typedef IpcpConfigMsg ipcp_config_msg_t;
+typedef DifInfoMsg dif_info_msg_t;
+
+#define ENROLL_AE               "Enrollment"
+#define ENROLL_PROTO            "OEP" /* Ouroboros enrollment protocol */
 #define ENROLL_WARN_TIME_OFFSET 20
+#define ENROLL_BUF_LEN          1024
+
+enum enroll_state {
+        ENROLL_NULL = 0,
+        ENROLL_INIT,
+        ENROLL_RUNNING
+};
 
 struct {
-        struct ae *   ae;
-        struct cdap * cdap;
-        pthread_t     listener;
+        struct ipcp_config conf;
+        enum enroll_state  state;
+        pthread_t          listener;
 } enroll;
 
-static void * enroll_handle(void * o)
+static int  send_rcv_enroll_msg(int fd)
 {
-        struct cdap *    cdap;
-        struct conn      conn;
-        cdap_key_t       key;
-        enum cdap_opcode oc;
-        char *           name;
-        uint8_t *        buf;
-        uint8_t *        data;
-        ssize_t          len;
-        uint32_t         flags;
+        enroll_msg_t    req = ENROLL_MSG__INIT;
+        enroll_msg_t *  reply;
+        uint8_t         buf[ENROLL_BUF_LEN];
+        ssize_t         len;
+        ssize_t         delta_t;
+        struct timespec t0;
+        struct timespec rtt;
 
-        bool boot_r     = false;
-        bool members_r  = false;
+        req.code = ENROLL_CODE__ENROLL_REQ;
 
-        char * boot_ro    = BOOT_PATH;
-        char * members_ro = MEMBERS_PATH;
+        len = enroll_msg__get_packed_size(&req);
+        if (len < 0) {
+                log_dbg("Failed pack request message.");
+                return -1;
+        }
 
-        cdap = (struct cdap *) o;
+        enroll_msg__pack(&req, buf);
 
-        assert(cdap);
+        clock_gettime(CLOCK_REALTIME, &t0);
 
-        while (true) {
-                if (connmgr_wait(enroll.ae, &conn)) {
-                        log_err("Failed to get next connection.");
-                        continue;
-                }
+        if (flow_write(fd, buf, len)) {
+                log_dbg("Failed to send request message.");
+                return -1;
+        }
 
-                if (cdap_add_flow(cdap, conn.flow_info.fd)) {
-                        log_warn("Failed to add flow to CDAP.");
-                        flow_dealloc(conn.flow_info.fd);
-                        continue;
-                }
+        len = flow_read(fd, buf, ENROLL_BUF_LEN);
+        if (len < 0) {
+                log_dbg("No enrollment reply received.");
+                return -1;
+        }
 
-                while (!(boot_r && members_r)) {
-                        key = cdap_request_wait(cdap, &oc, &name, &data,
-                                                (size_t *) &len , &flags);
-                        assert(key >= 0);
-                        assert(name);
-
-                        if (data != NULL) {
-                                free(data);
-                                log_warn("Received data with enroll request.");
-                        }
-
-                        if (oc != CDAP_READ) {
-                                log_warn("Invalid request.");
-                                cdap_reply_send(cdap, key, -1, NULL, 0);
-                                free(name);
-                                continue;
-                        }
-
-                        if (strcmp(name, boot_ro) == 0) {
-                                boot_r = true;
-                        } else if (strcmp(name, members_ro) == 0) {
-                                members_r = true;
-                        } else if (strcmp(name, TIME_PATH) == 0) {
-                                struct timespec t;
-                                uint64_t buf[2];
-                                clock_gettime(CLOCK_REALTIME, &t);
-                                buf[0] = hton64(t.tv_sec);
-                                buf[1] = hton64(t.tv_nsec);
-                                cdap_reply_send(cdap, key, 0, buf, sizeof(buf));
-                                free(name);
-                                continue;
-                        } else {
-                                log_warn("Illegal read: %s.", name);
-                                cdap_reply_send(cdap, key, -1, NULL, 0);
-                                free(name);
-                                continue;
-                        }
-
-                        len = rib_pack(name, &buf, PACK_HASH_ROOT);
-                        if (len < 0) {
-                                log_err("Failed to pack %s.", name);
-                                cdap_reply_send(cdap, key, -1, NULL, 0);
-                                free(name);
-                                continue;
-                        }
-
-                        log_dbg("Packed %s (%zu bytes).", name, len);
-
-                        free(name);
-
-                        if (cdap_reply_send(cdap, key, 0, buf, len)) {
-                                log_err("Failed to send CDAP reply.");
-                                free(buf);
-                                continue;
-                        }
-
-                        free(buf);
-                }
+        log_dbg("Received enrollment info (%zd bytes).", len);
 
-                log_dbg("Sent boot info to new member.");
+        reply = enroll_msg__unpack(NULL, len, buf);
+        if (reply == NULL) {
+                log_dbg("No enrollment response.");
+                return -1;
+        }
 
-                cdap_del_flow(cdap, conn.flow_info.fd);
-                flow_dealloc(conn.flow_info.fd);
+        if (reply->code != ENROLL_CODE__ENROLL_BOOT) {
+                log_dbg("Failed to unpack enrollment response.");
+                enroll_msg__free_unpacked(reply, NULL);
+                return -1;
+        }
 
-                boot_r = members_r = false;
+        if (!(reply->has_t_sec && reply->has_t_nsec)) {
+                log_dbg("No time in response message.");
+                enroll_msg__free_unpacked(reply, NULL);
+                return -1;
         }
 
+        clock_gettime(CLOCK_REALTIME, &rtt);
+
+        delta_t = ts_diff_ms(&t0, &rtt);
+
+        rtt.tv_sec  = reply->t_sec;
+        rtt.tv_nsec = reply->t_nsec;
+
+        if (labs(ts_diff_ms(&t0, &rtt)) - delta_t > ENROLL_WARN_TIME_OFFSET)
+                log_warn("Clock offset above threshold.");
+
+        strcpy(enroll.conf.dif_info.dif_name, reply->conf->dif_info->dif_name);
+        enroll.conf.type           = reply->conf->ipcp_type;
+        enroll.conf.addr_size      = reply->conf->addr_size;
+        enroll.conf.fd_size        = reply->conf->fd_size;
+        enroll.conf.has_ttl        = reply->conf->has_ttl;
+        enroll.conf.addr_auth_type = reply->conf->addr_auth_type;
+        enroll.conf.routing_type   = reply->conf->routing_type;
+        enroll.conf.dif_info.dir_hash_algo
+                = reply->conf->dif_info->dir_hash_algo;
+
+        enroll_msg__free_unpacked(reply, NULL);
+
         return 0;
 }
 
-int enroll_boot(const char * dst)
+static ssize_t enroll_pack(uint8_t ** buf)
 {
-        struct cdap * cdap;
-        cdap_key_t *  key;
-        uint8_t *     data;
-        size_t        len;
-        struct conn   conn;
+        enroll_msg_t      msg      = ENROLL_MSG__INIT;
+        ipcp_config_msg_t config   = IPCP_CONFIG_MSG__INIT;
+        dif_info_msg_t    dif_info = DIF_INFO_MSG__INIT;
+        struct timespec   now;
+        ssize_t           len;
+
+        clock_gettime(CLOCK_REALTIME, &now);
+
+        msg.code       = ENROLL_CODE__ENROLL_BOOT;
+        msg.has_t_sec  = true;
+        msg.t_sec      = now.tv_sec;
+        msg.has_t_nsec = true;
+        msg.t_nsec     = now.tv_nsec;
+        msg.conf       = &config;
+
+        config.ipcp_type          = enroll.conf.type;
+        config.has_addr_size      = true;
+        config.addr_size          = enroll.conf.addr_size;
+        config.has_fd_size        = true;
+        config.fd_size            = enroll.conf.fd_size;
+        config.has_has_ttl        = true;
+        config.has_ttl            = enroll.conf.has_ttl;
+        config.has_addr_auth_type = true;
+        config.addr_auth_type     = enroll.conf.addr_auth_type;
+        config.has_routing_type   = true;
+        config.routing_type       = enroll.conf.routing_type;
+        config.dif_info           = &dif_info;
+
+        dif_info.dif_name      = (char *) enroll.conf.dif_info.dif_name;
+        dif_info.dir_hash_algo = enroll.conf.dif_info.dir_hash_algo;
+
+        len = enroll_msg__get_packed_size(&msg);
+
+        *buf = malloc(len);
+        if (*buf == NULL)
+                return -1;
 
-        struct timespec t0;
-        struct timespec rtt;
+        enroll_msg__pack(&msg, *buf);
 
-        ssize_t delta_t;
+        return len;
+}
 
-        char * boot_ro    = BOOT_PATH;
-        char * members_ro = MEMBERS_PATH;
+static void * enroll_handle(void * o)
+{
+        struct conn    conn;
+        uint8_t        buf[ENROLL_BUF_LEN];
+        uint8_t *      reply;
+        ssize_t        len;
+        enroll_msg_t * msg;
 
-        cdap = cdap_create();
-        if (cdap == NULL) {
-                log_err("Failed to instantiate CDAP.");
-                return -1;
-        }
+        (void) o;
 
-        if (connmgr_alloc(enroll.ae, dst, NULL, &conn)) {
-                log_err("Failed to get connection.");
-                cdap_destroy(cdap);
-                return -1;
-        }
+        while (true) {
+                if (connmgr_wait(AEID_ENROLL, &conn)) {
+                        log_err("Failed to get next connection.");
+                        continue;
+                }
 
-        if (cdap_add_flow(cdap, conn.flow_info.fd)) {
-                log_warn("Failed to add flow to CDAP.");
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
-                return -1;
-        }
+                len = flow_read(conn.flow_info.fd, buf, ENROLL_BUF_LEN);
+                if (len < 0) {
+                        log_err("Failed to read from flow.");
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        continue;
+                }
 
-        log_dbg("Getting boot information from %s.", dst);
+                msg = enroll_msg__unpack(NULL, len, buf);
+                if (msg == NULL) {
+                        log_err("Failed to unpack message.");
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        continue;
+                }
 
-        clock_gettime(CLOCK_REALTIME, &t0);
+                if (msg->code != ENROLL_CODE__ENROLL_REQ) {
+                        log_err("Wrong message type.");
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        enroll_msg__free_unpacked(msg, NULL);
+                        continue;
+                }
 
-        key = cdap_request_send(cdap, CDAP_READ, TIME_PATH, NULL, 0, 0);
-        if (key == NULL || key[0] == INVALID_CDAP_KEY) {
-                log_err("Failed to send CDAP request.");
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
-                return -1;
-        }
+                log_dbg("Enrolling a new neighbor.");
 
-        if (cdap_reply_wait(cdap, key[0], &data, &len)) {
-                log_err("Failed to get CDAP reply.");
-                free(key);
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
-                return -1;
-        }
+                enroll_msg__free_unpacked(msg, NULL);
 
-        free(key);
+                len = enroll_pack(&reply);
+                if (reply == NULL) {
+                        log_err("Failed to pack enrollment message.");
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        continue;
+                }
 
-        clock_gettime(CLOCK_REALTIME, &rtt);
+                log_dbg("Sending enrollment info (%zd bytes).", len);
 
-        delta_t = ts_diff_ms(&t0, &rtt);
+                if (flow_write(conn.flow_info.fd, reply, len)) {
+                        log_err("Failed respond to enrollment request.");
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        free(reply);
+                        continue;
+                }
 
-        assert(len == 2 * sizeof (uint64_t));
+                free(reply);
 
-        rtt.tv_sec  = ntoh64(((uint64_t *) data)[0]);
-        rtt.tv_nsec = ntoh64(((uint64_t *) data)[1]);
+                len = flow_read(conn.flow_info.fd, buf, ENROLL_BUF_LEN);
+                if (len < 0) {
+                        log_err("Failed to read from flow.");
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        continue;
+                }
 
-        if (labs(ts_diff_ms(&t0, &rtt)) - delta_t > ENROLL_WARN_TIME_OFFSET)
-                log_warn("Clock offset above threshold.");
+                msg = enroll_msg__unpack(NULL, len, buf);
+                if (msg == NULL) {
+                        log_err("Failed to unpack message.");
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        continue;
+                }
 
-        free(data);
+                if (msg->code != ENROLL_CODE__ENROLL_DONE) {
+                        log_err("Wrong message type.");
+                        enroll_msg__free_unpacked(msg, NULL);
+                        connmgr_dealloc(AEID_ENROLL, &conn);
+                        continue;
+                }
 
-        key = cdap_request_send(cdap, CDAP_READ, boot_ro, NULL, 0, 0);
-        if (key == NULL || key[0] == INVALID_CDAP_KEY) {
-                log_err("Failed to send CDAP request.");
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
-                return -1;
-        }
+                log_dbg("Neighbor enrollment successful.");
 
-        if (cdap_reply_wait(cdap, key[0], &data, &len)) {
-                log_err("Failed to get CDAP reply.");
-                free(key);
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
-                return -1;
+                connmgr_dealloc(AEID_ENROLL, &conn);
         }
 
-        free(key);
+        return 0;
+}
 
-        log_dbg("Packed information received (%zu bytes).", len);
+int enroll_boot(struct conn * conn,
+                const char *  dst)
+{
+        log_dbg("Getting boot information from %s.", dst);
 
-        if (rib_unpack(data, len, UNPACK_CREATE)) {
-                log_warn("Error unpacking RIB data.");
-                rib_del(boot_ro);
-                free(data);
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
+        if (send_rcv_enroll_msg(conn->flow_info.fd)) {
+                log_err("Failed to enroll.");
                 return -1;
         }
 
-        log_dbg("Packed information inserted into RIB.");
+        return 0;
+}
 
-        key = cdap_request_send(cdap, CDAP_READ, members_ro, NULL, 0, 0);
-        if (key == NULL || key[0] == INVALID_CDAP_KEY) {
-                log_err("Failed to send CDAP request.");
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
-                return -1;
-        }
+int enroll_done(struct conn * conn)
+{
+        enroll_msg_t msg = ENROLL_MSG__INIT;
+        uint8_t      buf[ENROLL_BUF_LEN];
+        ssize_t       len;
 
-        if (cdap_reply_wait(cdap, key[0], &data, &len)) {
-                log_err("Failed to get CDAP reply.");
-                free(key);
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
+        msg.code = ENROLL_CODE__ENROLL_DONE;
+
+        len = enroll_msg__get_packed_size(&msg);
+        if (len < 0) {
+                log_dbg("Failed pack request message.");
                 return -1;
         }
 
-        free(key);
-
-        log_dbg("Packed information received (%zu bytes).", len);
+        enroll_msg__pack(&msg, buf);
 
-        if (rib_unpack(data, len, UNPACK_CREATE)) {
-                log_warn("Error unpacking RIB data.");
-                rib_del(boot_ro);
-                free(data);
-                cdap_destroy(cdap);
-                flow_dealloc(conn.flow_info.fd);
+        if (flow_write(conn->flow_info.fd, buf, len)) {
+                log_dbg("Failed to send acknowledgment.");
                 return -1;
         }
 
-        log_dbg("Packed information inserted into RIB.");
+        return 0;
+}
+
+void enroll_bootstrap(const struct ipcp_config * conf)
+{
+        assert(conf);
 
-        cdap_destroy(cdap);
-        flow_dealloc(conn.flow_info.fd);
+        memcpy(&enroll.conf, conf, sizeof(enroll.conf));
+}
 
-        return 0;
+struct ipcp_config * enroll_get_conf(void)
+{
+        return &enroll.conf;
 }
 
 int enroll_init(void)
 {
         struct conn_info info;
 
-        enroll.cdap = cdap_create();
-        if (enroll.cdap == NULL) {
-                log_err("Failed to instantiate CDAP.");
-                return -1;
-        }
-
         memset(&info, 0, sizeof(info));
 
         strcpy(info.ae_name, ENROLL_AE);
-        strcpy(info.protocol, CDAP_PROTO);
+        strcpy(info.protocol, ENROLL_PROTO);
         info.pref_version = 1;
         info.pref_syntax  = PROTO_GPB;
+        info.addr         = 0;
 
-        enroll.ae = connmgr_ae_create(info);
-        if (enroll.ae == NULL) {
-                cdap_destroy(enroll.cdap);
+        if (connmgr_ae_init(AEID_ENROLL, &info, NULL)) {
+                log_err("Failed to register with connmgr.");
                 return -1;
         }
 
+        enroll.state = ENROLL_INIT;
+
         return 0;
 }
 
 void enroll_fini(void)
 {
-        pthread_join(enroll.listener, NULL);
-        cdap_destroy(enroll.cdap);
-        connmgr_ae_destroy(enroll.ae);
+        if (enroll.state == ENROLL_RUNNING)
+                pthread_join(enroll.listener, NULL);
+
+        connmgr_ae_fini(AEID_ENROLL);
 }
 
 int enroll_start(void)
 {
-        if (pthread_create(&enroll.listener, NULL, enroll_handle, enroll.cdap))
+        if (pthread_create(&enroll.listener, NULL, enroll_handle, NULL))
                 return -1;
 
+        enroll.state = ENROLL_RUNNING;
+
         return 0;
 }
 
 void enroll_stop(void)
 {
-        pthread_cancel(enroll.listener);
+        if (enroll.state == ENROLL_RUNNING)
+                pthread_cancel(enroll.listener);
+
+        enroll.state = ENROLL_INIT;
 }