FS-9775: First round of integration of DHT into libblade, requires ongoing…

FS-9775: First round of integration of DHT into libblade, requires ongoing changes to DHT for proper exposure to blade level

libs/libblade/src/blade_peer.c

@@ -33,14 +33,22 @@
 
 #include "blade.h"
 
+#define KS_DHT_TPOOL_MIN 2
+#define KS_DHT_TPOOL_MAX 8
+#define KS_DHT_TPOOL_STACK (1024 * 256)
+#define KS_DHT_TPOOL_IDLE 10
+
 typedef enum {
 	BP_NONE = 0,
-	BP_MYPOOL = (1 << 0)
+	BP_MYPOOL = (1 << 0),
+	BP_MYTPOOL = (1 << 1)
 } bppvt_flag_t;
 
 struct blade_peer_s {
 	bppvt_flag_t flags;
 	ks_pool_t *pool;
+	ks_thread_pool_t *tpool;
+	ks_dht_t *dht;
 };
 
 
@@ -60,33 +68,75 @@ KS_DECLARE(ks_status_t) blade_peer_destroy(blade_peer_t **bpP)
 	flags = bp->flags;
 	pool = bp->pool;
 
+	if (bp->dht) ks_dht_destroy(&bp->dht);
+	if (bp->tpool && (flags & BP_MYTPOOL)) ks_thread_pool_destroy(&bp->tpool);
+	
 	ks_pool_free(bp->pool, &bp);
 
-	if (pool && (flags & BP_MYPOOL)) {
-		ks_pool_close(&pool);
-	}
+	if (pool && (flags & BP_MYPOOL)) ks_pool_close(&pool);
 
 	return KS_STATUS_SUCCESS;
 }
 
-KS_DECLARE(ks_status_t) blade_peer_create(blade_peer_t **bpP, ks_pool_t *pool)
+KS_DECLARE(ks_status_t) blade_peer_create(blade_peer_t **bpP, ks_pool_t *pool, ks_thread_pool_t *tpool, ks_dht_nodeid_t *nodeid)
 {
 	bppvt_flag_t newflags = BP_NONE;
 	blade_peer_t *bp = NULL;
+	ks_dht_t *dht = NULL;
 
 	if (!pool) {
 		newflags |= BP_MYPOOL;
 		ks_pool_open(&pool);
+		ks_assert(pool);
+	}
+	if (!tpool) {
+		newflags |= BP_MYTPOOL;
+		ks_thread_pool_create(&tpool, BLADE_PEER_TPOOL_MIN, BLADE_PEER_TPOOL_MAX, BLADE_PEER_TPOOL_STACK, KS_PRI_NORMAL, BLADE_PEER_TPOOL_IDLE);
+		ks_assert(tpool);
 	}
+	ks_dht_create(&dht, pool, tpool, nodeid);
+	ks_assert(dht);
 
 	bp = ks_pool_alloc(pool, sizeof(*bp));
-	bp->pool = pool;
 	bp->flags = newflags;
+	bp->pool = pool;
+	bp->tpool = tpool;
+	bp->dht = dht;
 	*bpP = bp;
 
 	return KS_STATUS_SUCCESS;
 }
 
+KS_DECLARE(ks_dht_nodeid_t *) blade_peer_myid(blade_peer_t *bp)
+{
+	ks_assert(bp);
+	ks_assert(bp->dht);
+
+	return &bp->dht->nodeid;
+}
+
+KS_DECLARE(void) blade_peer_autoroute(blade_peer_t *bp, ks_bool_t autoroute, ks_port_t port)
+{
+	ks_assert(bp);
+
+	ks_dht_autoroute(bp->dht, autoroute, port);
+}
+
+KS_DECLARE(ks_status_t) blade_peer_bind(blade_peer_t *bp, const ks_sockaddr_t *addr, ks_dht_endpoint_t **endpoint)
+{
+	ks_assert(bp);
+	ks_assert(addr);
+
+	return ks_dht_bind(bp->dht, addr, endpoint);
+}
+
+KS_DECLARE(void) blade_peer_pulse(blade_peer_t *bp, int32_t timeout)
+{
+	ks_assert(bp);
+	ks_assert(timeout >= 0);
+
+	ks_dht_pulse(bp->dht, timeout);
+}
 
 /* For Emacs:
  * Local Variables:

libs/libblade/src/blade_stack.c

@@ -35,12 +35,14 @@
 
 typedef enum {
 	BH_NONE = 0,
-	BH_MYPOOL = (1 << 0)
+	BH_MYPOOL = (1 << 0),
+	BH_MYTPOOL = (1 << 1)
 } bhpvt_flag_t;
 
 struct blade_handle_s {
-	ks_pool_t *pool;
 	bhpvt_flag_t flags;
+	ks_pool_t *pool;
+	ks_thread_pool_t *tpool;
 	blade_peer_t *peer;
 };
 
@@ -62,6 +64,7 @@ KS_DECLARE(ks_status_t) blade_handle_destroy(blade_handle_t **bhP)
 	pool = bh->pool;
 
 	blade_peer_destroy(&bh->peer);
+    if (bh->tpool && (flags & BH_MYTPOOL)) ks_thread_pool_destroy(&bh->tpool);
 
 	ks_pool_free(bh->pool, &bh);
 
@@ -72,26 +75,78 @@ KS_DECLARE(ks_status_t) blade_handle_destroy(blade_handle_t **bhP)
 	return KS_STATUS_SUCCESS;
 }
 
-KS_DECLARE(ks_status_t) blade_handle_create(blade_handle_t **bhP, ks_pool_t *pool)
+KS_DECLARE(ks_status_t) blade_handle_create(blade_handle_t **bhP, ks_pool_t *pool, ks_thread_pool_t *tpool, const char *nodeid)
 {
 	bhpvt_flag_t newflags = BH_NONE;
 	blade_handle_t *bh = NULL;
+	ks_dht_nodeid_t nid;
+
+	ks_assert(nodeid);
+	ks_assert(strlen(nodeid) == (KS_DHT_NODEID_SIZE * 2));
 
 	if (!pool) {
 		newflags |= BH_MYPOOL;
 		ks_pool_open(&pool);
 	}
-
+    if (!tpool) {
+		newflags |= BH_MYTPOOL;
+		ks_thread_pool_create(&tpool, BLADE_HANDLE_TPOOL_MIN, BLADE_HANDLE_TPOOL_MAX, BLADE_HANDLE_TPOOL_STACK, KS_PRI_NORMAL, BLADE_HANDLE_TPOOL_IDLE);
+		ks_assert(tpool);
+	}
+	
 	bh = ks_pool_alloc(pool, sizeof(*bh));
-	bh->pool = pool;
 	bh->flags = newflags;
-	blade_peer_create(&bh->peer, bh->pool);
+	bh->pool = pool;
+	bh->tpool = tpool;
+	ks_dht_dehex(nid.id, nodeid, KS_DHT_NODEID_SIZE);
+	blade_peer_create(&bh->peer, bh->pool, bh->tpool, &nid);
 
 	*bhP = bh;
 
 	return KS_STATUS_SUCCESS;
 }
 
+KS_DECLARE(void) blade_handle_myid(blade_handle_t *bh, char *buffer)
+{
+	ks_dht_nodeid_t *nodeid = NULL;
+
+	ks_assert(bh);
+	ks_assert(bh->peer);
+
+	nodeid = blade_peer_myid(bh->peer);
+	ks_dht_hex(nodeid->id, buffer, KS_DHT_NODEID_SIZE);
+}
+
+KS_DECLARE(void) blade_handle_autoroute(blade_handle_t *bh, ks_bool_t autoroute, ks_port_t port)
+{
+	ks_assert(bh);
+	ks_assert(bh->peer);
+
+	blade_peer_autoroute(bh->peer, autoroute, port);
+}
+
+KS_DECLARE(ks_status_t) blade_handle_bind(blade_handle_t *bh, const char *ip, ks_port_t port, ks_dht_endpoint_t **endpoint)
+{
+	ks_sockaddr_t addr;
+	int family = AF_INET;
+	
+	ks_assert(bh);
+	ks_assert(ip);
+	ks_assert(port);
+
+	if (ip[1] != '.' && ip[2] != '.' && ip[3] != '.') family = AF_INET6;
+	
+	ks_addr_set(&addr, ip, port, family);
+	return blade_peer_bind(bh->peer, &addr, endpoint);
+}
+
+KS_DECLARE(void) blade_handle_pulse(blade_handle_t *bh, int32_t timeout)
+{
+	ks_assert(bh);
+	ks_assert(timeout >= 0);
+
+	blade_peer_pulse(bh->peer, timeout);
+}
 
 
 /* For Emacs:

libs/libblade/src/include/blade.h

@@ -34,6 +34,7 @@
 #ifndef _BLADE_H_
 #define _BLADE_H_
 #include <ks.h>
+#include <ks_dht.h>
 #include <sodium.h>
 #include "blade_types.h"
 #include "blade_stack.h"

libs/libblade/src/include/blade_peer.h

@@ -35,9 +35,18 @@
 #define _BPCP_H_
 #include <blade.h>
 
+#define BLADE_PEER_TPOOL_MIN 2
+#define BLADE_PEER_TPOOL_MAX 8
+#define BLADE_PEER_TPOOL_STACK (1024 * 256)
+#define BLADE_PEER_TPOOL_IDLE 10
+
 KS_BEGIN_EXTERN_C
-KS_DECLARE(ks_status_t) blade_peer_create(blade_peer_t **bpP, ks_pool_t *pool);
+KS_DECLARE(ks_status_t) blade_peer_create(blade_peer_t **bpP, ks_pool_t *pool, ks_thread_pool_t *tpool, ks_dht_nodeid_t *nodeid);
 KS_DECLARE(ks_status_t) blade_peer_destroy(blade_peer_t **bpP);
+KS_DECLARE(ks_dht_nodeid_t *) blade_peer_myid(blade_peer_t *bp);
+KS_DECLARE(void) blade_peer_autoroute(blade_peer_t *bp, ks_bool_t autoroute, ks_port_t port);
+KS_DECLARE(ks_status_t) blade_peer_bind(blade_peer_t *bp, const ks_sockaddr_t *addr, ks_dht_endpoint_t **endpoint);
+KS_DECLARE(void) blade_peer_pulse(blade_peer_t *bp, int32_t timeout);
 KS_END_EXTERN_C
 
 #endif

libs/libblade/src/include/blade_stack.h

@@ -35,9 +35,18 @@
 #define _BLADE_STACK_H_
 #include <blade.h>
 
+#define BLADE_HANDLE_TPOOL_MIN 2
+#define BLADE_HANDLE_TPOOL_MAX 8
+#define BLADE_HANDLE_TPOOL_STACK (1024 * 256)
+#define BLADE_HANDLE_TPOOL_IDLE 10
+
 KS_BEGIN_EXTERN_C
 KS_DECLARE(ks_status_t) blade_handle_destroy(blade_handle_t **bhP);
-KS_DECLARE(ks_status_t) blade_handle_create(blade_handle_t **bhP, ks_pool_t *pool);
+KS_DECLARE(ks_status_t) blade_handle_create(blade_handle_t **bhP, ks_pool_t *pool, ks_thread_pool_t *tpool, const char *nodeid);
+KS_DECLARE(void) blade_handle_myid(blade_handle_t *bh, char *buffer);
+KS_DECLARE(void) blade_handle_autoroute(blade_handle_t *bh, ks_bool_t autoroute, ks_port_t port);
+KS_DECLARE(ks_status_t) blade_handle_bind(blade_handle_t *bh, const char *ip, ks_port_t port, ks_dht_endpoint_t **endpoint);
+KS_DECLARE(void) blade_handle_pulse(blade_handle_t *bh, int32_t timeout);
 KS_END_EXTERN_C
 
 #endif

libs/libblade/test/Makefile.am

@@ -7,6 +7,11 @@ testbuild_SOURCES = testbuild.c tap.c
 testbuild_CFLAGS = $(AM_CFLAGS)
 testbuild_LDADD = $(TEST_LDADD)
 
+check_PROGRAMS += bladec
+bladec_SOURCES = bladec.c tap.c
+bladec_CFLAGS = $(AM_CFLAGS)
+bladec_LDADD = $(TEST_LDADD)
+
 
 TESTS=$(check_PROGRAMS)
 

libs/libblade/test/bladec.c

+#include "blade.h"
+#include "tap.h"
+
+#ifdef _WIN32
+#define STDIO_FD(_fs) _fileno(_fs)
+#define READ(_fd, _buffer, _count) _read(_fd, _buffer, _count)
+#else
+#define STDIO_FD(_fs) fileno(_fs)
+#define READ(_fd, _buffer, _count) read(_fd, _buffer, _count)
+#endif
+
+#define CONSOLE_INPUT_MAX 512
+
+ks_bool_t g_shutdown = KS_FALSE;
+char g_console_input[CONSOLE_INPUT_MAX];
+size_t g_console_input_length = 0;
+size_t g_console_input_eol = 0;
+
+void loop(blade_handle_t *bh);
+void process_console_input(blade_handle_t *bh, char *line);
+
+typedef void (*command_callback)(blade_handle_t *bh, char *args);
+
+struct command_def_s {
+	const char *cmd;
+	command_callback callback;
+};
+
+void command_test(blade_handle_t *bh, char *args);
+void command_quit(blade_handle_t *bh, char *args);
+void command_myid(blade_handle_t *bh, char *args);
+void command_bind(blade_handle_t *bh, char *args);
+
+static const struct command_def_s command_defs[] = {
+	{ "test", command_test },
+	{ "quit", command_quit },
+	{ "myid", command_myid },
+	{ "bind", command_bind },
+	
+	{ NULL, NULL }
+};
+
+
+int main(int argc, char **argv)
+{
+	blade_handle_t *bh = NULL;
+	const char *nodeid;
+
+	ks_assert(argc >= 2);
+
+	nodeid = argv[1];
+
+	ks_global_set_default_logger(KS_LOG_LEVEL_DEBUG);
+	
+	blade_init();
+
+	blade_handle_create(&bh, NULL, NULL, nodeid);
+
+	blade_handle_autoroute(bh, KS_TRUE, KS_DHT_DEFAULT_PORT);
+
+	loop(bh);
+	
+	blade_handle_destroy(&bh);
+
+	blade_shutdown();
+
+	return 0;
+}
+
+void buffer_console_input(void)
+{
+	ssize_t bytes = 0;
+	struct pollfd poll[1];
+	poll[0].fd = STDIO_FD(stdin);
+	poll[0].events = POLLIN | POLLERR;
+
+	if (ks_poll(poll, 1, 1) > 0) {
+		if (poll[0].revents & POLLIN) {
+			if ((bytes = READ(poll[0].fd, g_console_input + g_console_input_length, CONSOLE_INPUT_MAX - g_console_input_length)) <= 0) {
+				// @todo error
+				return;
+			}
+			g_console_input_length += bytes;
+		}
+	}
+}
+
+void loop(blade_handle_t *bh)
+{
+	while (!g_shutdown) {
+		ks_bool_t eol = KS_FALSE;
+		buffer_console_input();
+
+		for (; g_console_input_eol < g_console_input_length; ++g_console_input_eol) {
+			char c = g_console_input[g_console_input_eol];
+			if (c == '\r' || c == '\n') {
+				eol = KS_TRUE;
+				break;
+			}
+		}
+		if (eol) {
+			g_console_input[g_console_input_eol] = '\0';
+			process_console_input(bh, g_console_input);
+			g_console_input_eol++;
+			for (; g_console_input_eol < g_console_input_length; ++g_console_input_eol) {
+				char c = g_console_input[g_console_input_eol];
+				if (c != '\r' && c != '\n') break;
+			}
+			if (g_console_input_eol == g_console_input_length) g_console_input_eol = g_console_input_length = 0;
+			else {
+				memcpy(g_console_input, g_console_input + g_console_input_eol, g_console_input_length - g_console_input_eol);
+				g_console_input_length -= g_console_input_eol;
+				g_console_input_eol = 0;
+			}
+		}
+		if (g_console_input_length == CONSOLE_INPUT_MAX) {
+			// @todo lines must not exceed 512 bytes, treat as error and ignore buffer until next new line?
+			ks_assert(0);
+		}
+		blade_handle_pulse(bh, 1);
+	}
+}
+
+void parse_argument(char **input, char **arg, char terminator)
+{
+	char *tmp;
+	
+	ks_assert(input);
+	ks_assert(*input);
+	ks_assert(arg);
+
+	tmp = *input;
+	*arg = tmp;
+
+	while (*tmp && *tmp != terminator) ++tmp;
+	if (*tmp == terminator) {
+		*tmp = '\0';
+		++tmp;
+	}
+	*input = tmp;
+}
+
+void process_console_input(blade_handle_t *bh, char *line)
+{
+	char *args = line;
+	char *cmd = NULL;
+	ks_bool_t found = KS_FALSE;
+	
+	ks_log(KS_LOG_DEBUG, "Output: %s\n", line);
+
+	parse_argument(&args, &cmd, ' ');
+	
+	ks_log(KS_LOG_DEBUG, "Command: %s, Args: %s\n", cmd, args);
+
+	for (int32_t index = 0; command_defs[index].cmd; ++index) {
+		if (!strcmp(command_defs[index].cmd, cmd)) {
+			found = KS_TRUE;
+			command_defs[index].callback(bh, args);
+		}
+	}
+	if (!found) ks_log(KS_LOG_INFO, "Command '%s' unknown.\n", cmd);
+}
+
+void command_test(blade_handle_t *bh, char *args)
+{
+	ks_log(KS_LOG_DEBUG, "Hello World!\n");
+}
+
+void command_quit(blade_handle_t *bh, char *args)
+{
+	ks_assert(bh);
+	ks_assert(args);
+	
+	ks_log(KS_LOG_DEBUG, "Shutting down\n");
+	g_shutdown = KS_TRUE;
+}
+
+void command_myid(blade_handle_t *bh, char *args)
+{
+	char buf[KS_DHT_NODEID_SIZE * 2 + 1];
+
+	ks_assert(bh);
+	ks_assert(args);
+
+	blade_handle_myid(bh, buf);
+
+	ks_log(KS_LOG_INFO, "%s\n", buf);
+}
+
+void command_bind(blade_handle_t *bh, char *args)
+{
+	char *ip = NULL;
+	char *port = NULL;
+	ks_port_t p;
+
+	ks_assert(args);
+
+	parse_argument(&args, &ip, ' ');
+	parse_argument(&args, &port, ' ');
+
+	p = atoi(port); // @todo use strtol for error handling
+
+	blade_handle_bind(bh, ip, p, NULL);
+}

libs/libblade/test/testbuild.c

@@ -11,7 +11,7 @@ int main(void)
 
 	plan(1);
 
-	status = blade_handle_create(&bh, NULL);
+	status = blade_handle_create(&bh, NULL, NULL);
 	status = blade_handle_destroy(&bh);
 
 	ok(status == KS_STATUS_SUCCESS);

libs/libks/src/dht/ks_dht.c

@@ -8,13 +8,14 @@ void ks_dht_transaction_destructor(void *ptr) { ks_dht_transaction_destroy((ks_d
 
 void ks_dht_storageitem_destructor(void *ptr) { ks_dht_storageitem_destroy((ks_dht_storageitem_t **)&ptr); }
 
-KS_DECLARE(ks_status_t) ks_dht_create(ks_dht_t **dht, ks_pool_t *pool, ks_thread_pool_t *tpool)
+KS_DECLARE(ks_status_t) ks_dht_create(ks_dht_t **dht, ks_pool_t *pool, ks_thread_pool_t *tpool, ks_dht_nodeid_t *nodeid)
 {
 	ks_bool_t pool_alloc = !pool;
 	ks_dht_t *d = NULL;
 	ks_status_t ret = KS_STATUS_SUCCESS;
 
 	ks_assert(dht);
+	ks_assert(nodeid);
 
 	*dht = NULL;
 
@@ -49,6 +50,8 @@ KS_DECLARE(ks_status_t) ks_dht_create(ks_dht_t **dht, ks_pool_t *pool, ks_thread
 		ks_assert(d->tpool);
 	}
 
+	d->nodeid = *nodeid;
+
 	/**
 	 * Default autorouting to disabled.
 	 */
@@ -366,7 +369,7 @@ KS_DECLARE(ks_status_t) ks_dht_autoroute_check(ks_dht_t *dht, const ks_sockaddr_
 	if (!ep && dht->autoroute) {
 		ks_sockaddr_t addr;
 		if ((ret = ks_addr_set(&addr, ip, dht->autoroute_port, raddr->family)) != KS_STATUS_SUCCESS) return ret;
-		if ((ret = ks_dht_bind(dht, NULL, &addr, &ep)) != KS_STATUS_SUCCESS) return ret;
+		if ((ret = ks_dht_bind(dht, &addr, &ep)) != KS_STATUS_SUCCESS) return ret;
 	}
 
 	/**
@@ -419,7 +422,7 @@ KS_DECLARE(void) ks_dht_register_error(ks_dht_t *dht, const char *value, ks_dht_
 }
 
 
-KS_DECLARE(ks_status_t) ks_dht_bind(ks_dht_t *dht, const ks_dht_nodeid_t *nodeid, const ks_sockaddr_t *addr, ks_dht_endpoint_t **endpoint)
+KS_DECLARE(ks_status_t) ks_dht_bind(ks_dht_t *dht, const ks_sockaddr_t *addr, ks_dht_endpoint_t **endpoint)
 {
 	ks_socket_t sock = KS_SOCK_INVALID;
 	ks_dht_endpoint_t *ep = NULL;
@@ -466,7 +469,7 @@ KS_DECLARE(ks_status_t) ks_dht_bind(ks_dht_t *dht, const ks_dht_nodeid_t *nodeid
 	/**
 	 * Allocate the endpoint to track the local socket.
 	 */
-	ks_dht_endpoint_create(&ep, dht->pool, nodeid, addr, sock);
+	ks_dht_endpoint_create(&ep, dht->pool, addr, sock);
 	ks_assert(ep);
 
 	/**
@@ -506,24 +509,24 @@ KS_DECLARE(ks_status_t) ks_dht_bind(ks_dht_t *dht, const ks_dht_nodeid_t *nodeid
 	if (ep->addr.family == AF_INET) {
 		if (!dht->rt_ipv4 && (ret = ks_dhtrt_initroute(&dht->rt_ipv4, dht, dht->pool)) != KS_STATUS_SUCCESS) goto done;
 		if ((ret = ks_dhtrt_create_node(dht->rt_ipv4,
-										ep->nodeid,
+										dht->nodeid,
 										KS_DHT_LOCAL,
 										ep->addr.host,
 										ep->addr.port,
 										KS_DHTRT_CREATE_DEFAULT,
-										&ep->node)) != KS_STATUS_SUCCESS) goto done;
+										&dht->node)) != KS_STATUS_SUCCESS) goto done;
 	} else {
 		if (!dht->rt_ipv6 && (ret = ks_dhtrt_initroute(&dht->rt_ipv6, dht, dht->pool)) != KS_STATUS_SUCCESS) goto done;
 		if ((ret = ks_dhtrt_create_node(dht->rt_ipv6,
-										ep->nodeid,
+										dht->nodeid,
 										KS_DHT_LOCAL,
 										ep->addr.host,
 										ep->addr.port,
 										KS_DHTRT_CREATE_DEFAULT,
-										&ep->node)) != KS_STATUS_SUCCESS) goto done;
+										&dht->node)) != KS_STATUS_SUCCESS) goto done;
 	}
 	/**
-	 * Do not release the ep->node, keep it alive until cleanup
+	 * Do not release the dht->node, keep it alive until cleanup
 	 */
 
 	/**
@@ -788,6 +791,19 @@ KS_DECLARE(char *) ks_dht_hex(const uint8_t *data, char *buffer, ks_size_t len)
 	return buffer;
 }
 
+KS_DECLARE(uint8_t *) ks_dht_dehex(uint8_t *data, const char *buffer, ks_size_t len)
+{
+	const char *t = buffer;
+	
+	ks_assert(data);
+	ks_assert(buffer);
+	ks_assert(!(len & 1));
+
+	for (int i = 0; i < len; ++i, t += 2) sscanf(t, "%2hhx", &data[i]);
+	
+	return data;
+}
+
 KS_DECLARE(void) ks_dht_utility_nodeid_xor(ks_dht_nodeid_t *dest, ks_dht_nodeid_t *src1, ks_dht_nodeid_t *src2)
 {
 	ks_assert(dest);
@@ -1361,7 +1377,7 @@ KS_DECLARE(ks_status_t) ks_dht_query_setup(ks_dht_t *dht,
 
 	if (args) *args = a;
 
-	ben_dict_set(a, ben_blob("id", 2), ben_blob(ep->nodeid.id, KS_DHT_NODEID_SIZE));
+	ben_dict_set(a, ben_blob("id", 2), ben_blob(dht->nodeid.id, KS_DHT_NODEID_SIZE));
 
 	*message = msg;
 
@@ -1414,7 +1430,7 @@ KS_DECLARE(ks_status_t) ks_dht_response_setup(ks_dht_t *dht,
 
 	if (args) *args = r;
 
-	ben_dict_set(r, ben_blob("id", 2), ben_blob(ep->nodeid.id, KS_DHT_NODEID_SIZE));
+	ben_dict_set(r, ben_blob("id", 2), ben_blob(dht->nodeid.id, KS_DHT_NODEID_SIZE));
 
 	*message = msg;
 
@@ -1546,7 +1562,7 @@ KS_DECLARE(ks_status_t) ks_dht_process_query(ks_dht_t *dht, ks_dht_message_t *me
 	message->args_id = *id;
 
 	ks_log(KS_LOG_DEBUG, "Creating node %s\n", ks_dht_hex(id->id, id_buf, KS_DHT_NODEID_SIZE));
-	if ((ret = ks_dhtrt_create_node(message->endpoint->node->table,
+	if ((ret = ks_dhtrt_create_node(message->endpoint->addr.family == AF_INET ? dht->rt_ipv4 : dht->rt_ipv6,
 									*id,
 									KS_DHT_REMOTE,
 									message->raddr.host,
@@ -1620,7 +1636,7 @@ KS_DECLARE(ks_status_t) ks_dht_process_response(ks_dht_t *dht, ks_dht_message_t
 	message->args_id = *id;
 
 	ks_log(KS_LOG_DEBUG, "Creating node %s\n", ks_dht_hex(id->id, id_buf, KS_DHT_NODEID_SIZE));
-	if ((ret = ks_dhtrt_create_node(message->endpoint->node->table,
+	if ((ret = ks_dhtrt_create_node(message->endpoint->addr.family == AF_INET ? dht->rt_ipv4 : dht->rt_ipv6,
 									*id,
 									KS_DHT_REMOTE,
 									message->raddr.host,
@@ -1629,7 +1645,7 @@ KS_DECLARE(ks_status_t) ks_dht_process_response(ks_dht_t *dht, ks_dht_message_t
 									&node)) != KS_STATUS_SUCCESS) goto done;
 	
 	ks_log(KS_LOG_DEBUG, "Touching node %s\n", ks_dht_hex(id->id, id_buf, KS_DHT_NODEID_SIZE));
-	if ((ret = ks_dhtrt_touch_node(message->endpoint->node->table, *id)) != KS_STATUS_SUCCESS) goto done;
+	if ((ret = ks_dhtrt_touch_node(message->endpoint->addr.family == AF_INET ? dht->rt_ipv4 : dht->rt_ipv6, *id)) != KS_STATUS_SUCCESS) goto done;
 
 	
 	tid = (uint32_t *)message->transactionid;
@@ -2292,7 +2308,7 @@ KS_DECLARE(ks_status_t) ks_dht_query_findnode(ks_dht_t *dht, ks_dht_job_t *job)
 
 	ben_dict_set(a, ben_blob("target", 6), ben_blob(job->query_target.id, KS_DHT_NODEID_SIZE));
 	// Only request both v4 and v6 if we have both interfaces bound and are looking for our own node id, aka bootstrapping
-	if (dht->rt_ipv4 && dht->rt_ipv6 && !memcmp(message->endpoint->nodeid.id, job->query_target.id, KS_DHT_NODEID_SIZE)) {
+	if (dht->rt_ipv4 && dht->rt_ipv6 && !memcmp(dht->nodeid.id, job->query_target.id, KS_DHT_NODEID_SIZE)) {
 		struct bencode *want = ben_list();
 		ben_list_append_str(want, "n4");
 		ben_list_append_str(want, "n6");

libs/libks/src/dht/ks_dht_endpoint.c

@@ -7,7 +7,6 @@
  */
 KS_DECLARE(ks_status_t) ks_dht_endpoint_create(ks_dht_endpoint_t **endpoint,
 											   ks_pool_t *pool,
-											   const ks_dht_nodeid_t *nodeid,
 											   const ks_sockaddr_t *addr,
 											   ks_socket_t sock)
 {
@@ -23,8 +22,6 @@ KS_DECLARE(ks_status_t) ks_dht_endpoint_create(ks_dht_endpoint_t **endpoint,
 	ks_assert(ep);
 
 	ep->pool = pool;
-    if (!nodeid) randombytes_buf(ep->nodeid.id, KS_DHT_NODEID_SIZE);
-	else memcpy(ep->nodeid.id, nodeid->id, KS_DHT_NODEID_SIZE);
 	ep->addr = *addr;
 	ep->sock = sock;
 

libs/libks/src/dht/ks_dht-int.h → libs/libks/src/include/ks_dht-int.h

@@ -319,7 +319,6 @@ KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job);
  */
 KS_DECLARE(ks_status_t) ks_dht_endpoint_create(ks_dht_endpoint_t **endpoint,
 											   ks_pool_t *pool,
-											   const ks_dht_nodeid_t *nodeid,
 											   const ks_sockaddr_t *addr,
 											   ks_socket_t sock);
 KS_DECLARE(void) ks_dht_endpoint_destroy(ks_dht_endpoint_t **endpoint);

libs/libks/test/Makefile.am

@@ -64,6 +64,11 @@ testdht2_SOURCES = testdht2.c tap.c
 testdht2_CFLAGS = $(AM_CFLAGS)
 testdht2_LDADD = $(TEST_LDADD)
 
+check_PROGRAMS += nodeidgen
+nodeidgen_SOURCES = nodeidgen.c tap.c
+nodeidgen_CFLAGS = $(AM_CFLAGS)
+nodeidgen_LDADD = $(TEST_LDADD)
+
 #check_PROGRAMS += testdht_net
 #testdht_net_SOURCES = testdht-net.c tap.c
 #testdht_net_CFLAGS = $(AM_CFLAGS)

libs/libks/test/nodeidgen.c

+#include <ks.h>
+#include <ks_dht.h>
+#include <ks_dht-int.h>
+#include <tap.h>
+
+int main() {
+	uint8_t nodeid[KS_DHT_NODEID_SIZE];
+	char buf[KS_DHT_NODEID_SIZE * 2 + 1];
+	
+	randombytes_buf(nodeid, KS_DHT_NODEID_SIZE);
+
+	ks_dht_hex(nodeid, buf, KS_DHT_NODEID_SIZE);
+
+	printf(buf);
+	return 0;
+}

libs/libks/test/testdht2.c

 #include <ks.h>
-#include <../dht/ks_dht.h>
-#include <../dht/ks_dht-int.h>
+#include <ks_dht.h>
+#include <ks_dht-int.h>
 #include <tap.h>
 
 ks_dht_storageitem_skey_t sk;
@@ -55,6 +55,7 @@ int main() {
 	//ks_size_t buflen;
   ks_status_t err;
   int mask = 0;
+  ks_dht_nodeid_t nodeid;
   ks_dht_t *dht1 = NULL;
   ks_dht_t *dht2 = NULL;
   ks_dht_t *dht3 = NULL;
@@ -112,20 +113,23 @@ int main() {
 	  diag("Binding to %s on ipv6\n", v6);
   }
 
-  err = ks_dht_create(&dht1, NULL, NULL);
+  ks_dht_dehex(nodeid.id, "0000000000000000000000000000000000000001", KS_DHT_NODEID_SIZE);
+  err = ks_dht_create(&dht1, NULL, NULL, &nodeid);
   ok(err == KS_STATUS_SUCCESS);
   
-  err = ks_dht_create(&dht2, NULL, NULL);
+  ks_dht_dehex(nodeid.id, "0000000000000000000000000000000000000002", KS_DHT_NODEID_SIZE);
+  err = ks_dht_create(&dht2, NULL, NULL, &nodeid);
   ok(err == KS_STATUS_SUCCESS);
 
-  err = ks_dht_create(&dht3, NULL, NULL);
+  ks_dht_dehex(nodeid.id, "0000000000000000000000000000000000000003", KS_DHT_NODEID_SIZE);
+  err = ks_dht_create(&dht3, NULL, NULL, &nodeid);
   ok(err == KS_STATUS_SUCCESS);
   
   if (have_v4) {
     err = ks_addr_set(&addr, v4, KS_DHT_DEFAULT_PORT, AF_INET);
 	ok(err == KS_STATUS_SUCCESS);
 	
-    err = ks_dht_bind(dht1, NULL, &addr, &ep1);
+    err = ks_dht_bind(dht1, &addr, &ep1);
     ok(err == KS_STATUS_SUCCESS);
 
 	raddr1 = addr;
@@ -133,7 +137,7 @@ int main() {
 	err = ks_addr_set(&addr, v4, KS_DHT_DEFAULT_PORT + 1, AF_INET);
 	ok(err == KS_STATUS_SUCCESS);
 	
-	err = ks_dht_bind(dht2, NULL, &addr, &ep2);
+	err = ks_dht_bind(dht2, &addr, &ep2);
 	ok(err == KS_STATUS_SUCCESS);
 
 	//raddr2 = addr;
@@ -141,7 +145,7 @@ int main() {
 	err = ks_addr_set(&addr, v4, KS_DHT_DEFAULT_PORT + 2, AF_INET);
 	ok(err == KS_STATUS_SUCCESS);
 	
-	err = ks_dht_bind(dht3, NULL, &addr, &ep3);
+	err = ks_dht_bind(dht3, &addr, &ep3);
 	ok(err == KS_STATUS_SUCCESS);
 
 	//raddr3 = addr;
@@ -151,19 +155,19 @@ int main() {
 	err = ks_addr_set(&addr, v6, KS_DHT_DEFAULT_PORT, AF_INET6);
 	ok(err == KS_STATUS_SUCCESS);
 	  
-    err = ks_dht_bind(dht1, NULL, &addr, NULL);
+    err = ks_dht_bind(dht1, &addr, NULL);
     ok(err == KS_STATUS_SUCCESS);
 
 	err = ks_addr_set(&addr, v6, KS_DHT_DEFAULT_PORT + 1, AF_INET6);
 	ok(err == KS_STATUS_SUCCESS);
 
-	err = ks_dht_bind(dht2, NULL, &addr, NULL);
+	err = ks_dht_bind(dht2, &addr, NULL);
 	ok(err == KS_STATUS_SUCCESS);
 
 	err = ks_addr_set(&addr, v6, KS_DHT_DEFAULT_PORT + 2, AF_INET6);
 	ok(err == KS_STATUS_SUCCESS);
 
-	err = ks_dht_bind(dht3, NULL, &addr, NULL);
+	err = ks_dht_bind(dht3, &addr, NULL);
 	ok(err == KS_STATUS_SUCCESS);
   }
 
@@ -175,11 +179,11 @@ int main() {
 
   ks_dht_pulse(dht1, 100); // Receive and process ping query from dht2, queue and send ping response
 
-  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, dht2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
 
   ks_dht_pulse(dht2, 100); // Receive and process ping response from dht1 (PROCESSING then COMPLETING)
 
-  ok(ks_dhtrt_find_node(dht2->rt_ipv4, ep1->nodeid) != NULL); // The node should be good, and thus be returned as good
+  ok(ks_dhtrt_find_node(dht2->rt_ipv4, dht1->nodeid) != NULL); // The node should be good, and thus be returned as good
 
   ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
 
@@ -189,7 +193,7 @@ int main() {
 	  ks_dht_pulse(dht2, 100);
 	  ks_dht_pulse(dht3, 100);
   }
-  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, dht2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
 
   
   ks_dht_ping(dht3, &raddr1, NULL, NULL); // (QUERYING)
@@ -198,11 +202,11 @@ int main() {
   
   ks_dht_pulse(dht1, 100); // Receive and process ping query from dht3, queue and send ping response
 
-  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, dht3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
 
   ks_dht_pulse(dht3, 100); // Receive and process ping response from dht1 (PROCESSING then COMPLETING)
 
-  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep1->nodeid) != NULL); // The node should be good, and thus be returned as good
+  ok(ks_dhtrt_find_node(dht3->rt_ipv4, dht1->nodeid) != NULL); // The node should be good, and thus be returned as good
 
   ks_dht_pulse(dht3, 100); // Call finish callback and purge the job (COMPLETING)
 
@@ -212,26 +216,26 @@ int main() {
 	  ks_dht_pulse(dht2, 100);
 	  ks_dht_pulse(dht3, 100);
   }
-  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, dht2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
 
   // Test bootstrap find_node from dht3 to dht1 to find dht2 nodeid
 
   /*
   diag("Find_Node test\n");
 
-  ks_dht_findnode(dht3, NULL, &raddr1, NULL, NULL, &ep2->nodeid);
+  ks_dht_findnode(dht3, NULL, &raddr1, NULL, NULL, &dht2->nodeid);
 
   ks_dht_pulse(dht3, 100); // Send queued findnode from dht3 to dht1
 
   ks_dht_pulse(dht1, 100); // Receive and process findnode query from dht3, queue and send findnode response
 
-  ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  ok(ks_dhtrt_find_node(dht1->rt_ipv4, dht3->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
 
   ks_dht_pulse(dht3, 100); // Receive and process findnode response from dht1
   
   ks_dht_pulse(dht3, 100); // Call finish callback and purge the job (COMPLETING)
 
-  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
+  ok(ks_dhtrt_find_node(dht3->rt_ipv4, dht2->nodeid) == NULL); // The node should be dubious, and thus not be returned as good yet
   
   diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
   for (int i = 0; i < 10; ++i) {
@@ -239,12 +243,12 @@ int main() {
 	  ks_dht_pulse(dht2, 100);
 	  ks_dht_pulse(dht3, 100);
   }
-  ok(ks_dhtrt_find_node(dht3->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
+  ok(ks_dhtrt_find_node(dht3->rt_ipv4, dht2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
   */
 
   diag("Search test\n");
   
-  ks_dht_search(dht3, dht2_search_callback, NULL, dht3->rt_ipv4, &ep2->nodeid);
+  ks_dht_search(dht3, dht2_search_callback, NULL, dht3->rt_ipv4, &dht2->nodeid);
   diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
   for (int i = 0; i < 20; ++i) {
 	  ks_dht_pulse(dht1, 100);