[PATCH 0/3] TLS initiator side implementation

[PATCH 0/3] TLS initiator side implementation

[Shradha Shah]

This is the patch series for initiator side TLS implementation.

At alloc_queue time, the kernel will send a netlink message to
the user space process to perform the TLS handshake, which in
turn will reply back with a KTLS socket.

Each queue uses this newly created KTLS socket for its communication.

The TLS handshake uses PSK authorization. I have just chosen a random
value of key and hint but this will be replaced with the correct NQN
once the target implementation is complete.

This is version 1 of this patch series and addresses all the issue
reported on the previous patch series which was a RFC patch series.

Shradha Shah (3):
  nvme-tcp: TLS encryption support-Introduce boolean flag in the header
    file
  nvme-tcp: TLS encrytion support using netlink sockets
  User mode server and client code

 drivers/nvme/host/fabrics.c       |   8 +-
 drivers/nvme/host/fabrics.h       |   3 +
 drivers/nvme/host/nvme.h          |   2 +
 drivers/nvme/host/tcp.c           | 154 +++++++++++++++++--
 include/uapi/linux/nvme_netlink.h |  43 ++++++
 tls/Makefile                      |  12 ++
 tls/client.c                      | 308 ++++++++++++++++++++++++++++++++++++++
 tls/server.c                      | 183 ++++++++++++++++++++++
 8 files changed, 703 insertions(+), 10 deletions(-)
 create mode 100644 include/uapi/linux/nvme_netlink.h
 create mode 100644 tls/Makefile
 create mode 100644 tls/client.c
 create mode 100644 tls/server.c

-- 
2.14.4

[PATCH 1/3] nvme-tcp: TLS encryption support-Introduce boolean flag in the header file

[Shradha Shah]

Signed-off-by: Shradha Shah <sshah at solarflare.com>
---
 drivers/nvme/host/fabrics.c | 8 +++++++-
 drivers/nvme/host/fabrics.h | 3 +++
 drivers/nvme/host/nvme.h    | 1 +
 drivers/nvme/host/tcp.c     | 4 +++-
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 8f0f34d06d46..cc693aeb44f2 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -550,6 +550,7 @@ static const match_table_t opt_tokens = {
 	{ NVMF_OPT_HOST_TRADDR,		"host_traddr=%s"	},
 	{ NVMF_OPT_HOST_ID,		"hostid=%s"		},
 	{ NVMF_OPT_DUP_CONNECT,		"duplicate_connect"	},
+	{ NVMF_OPT_TLS_ENABLE,              "tls_enable"      },
 	{ NVMF_OPT_ERR,			NULL			}
 };
 
@@ -569,6 +570,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 	opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY;
 	opts->kato = NVME_DEFAULT_KATO;
 	opts->duplicate_connect = false;
+	opts->tls_enable = false;
 
 	options = o = kstrdup(buf, GFP_KERNEL);
 	if (!options)
@@ -756,6 +758,9 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 		case NVMF_OPT_DUP_CONNECT:
 			opts->duplicate_connect = true;
 			break;
+		case NVMF_OPT_TLS_ENABLE:
+			opts->tls_enable = true;
+			break;
 		default:
 			pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
 				p);
@@ -837,7 +842,8 @@ EXPORT_SYMBOL_GPL(nvmf_free_options);
 #define NVMF_REQUIRED_OPTS	(NVMF_OPT_TRANSPORT | NVMF_OPT_NQN)
 #define NVMF_ALLOWED_OPTS	(NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \
 				 NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \
-				 NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT)
+				 NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT | \
+				 NVMF_OPT_TLS_ENABLE)
 
 static struct nvme_ctrl *
 nvmf_create_ctrl(struct device *dev, const char *buf, size_t count)
diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h
index a3145d90c1d2..bb9c61db8891 100644
--- a/drivers/nvme/host/fabrics.h
+++ b/drivers/nvme/host/fabrics.h
@@ -58,6 +58,7 @@ enum {
 	NVMF_OPT_CTRL_LOSS_TMO	= 1 << 11,
 	NVMF_OPT_HOST_ID	= 1 << 12,
 	NVMF_OPT_DUP_CONNECT	= 1 << 13,
+	NVMF_OPT_TLS_ENABLE   = 1 << 14,
 };
 
 /**
@@ -80,6 +81,7 @@ enum {
  * @nr_io_queues: Number of controller IO queues that will be established.
  * @reconnect_delay: Time between two consecutive reconnect attempts.
  * @discovery_nqn: indicates if the subsysnqn is the well-known discovery NQN.
+ * @tls_enable: Indicates if connection needs a TLS handshake via AF_KTLS socket
  * @kato:	Keep-alive timeout.
  * @host:	Virtual NVMe host, contains the NQN and Host ID.
  * @max_reconnects: maximum number of allowed reconnect attempts before removing
@@ -98,6 +100,7 @@ struct nvmf_ctrl_options {
 	unsigned int		reconnect_delay;
 	bool			discovery_nqn;
 	bool			duplicate_connect;
+	bool                  tls_enable;
 	unsigned int		kato;
 	struct nvmf_host	*host;
 	int			max_reconnects;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index d0f6008577d9..06ebc7302aed 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -206,6 +206,7 @@ struct nvme_ctrl {
 	u16 icdoff;
 	u16 maxcmd;
 	int nr_reconnects;
+	bool tls_enable;
 	struct nvmf_ctrl_options *opts;
 	struct delayed_work reconnect_work;
 	struct work_struct err_work;
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index ad77c75e286d..a1e16457b74e 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -1908,6 +1908,7 @@ static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
 	ctrl->ctrl.queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
 	ctrl->ctrl.sqsize = opts->queue_size - 1;
 	ctrl->ctrl.kato = opts->kato;
+	ctrl->ctrl.tls_enable = opts->tls_enable;
 
 	INIT_DELAYED_WORK(&ctrl->ctrl.reconnect_work,
 			nvmf_reconnect_ctrl_work);
@@ -1976,7 +1977,8 @@ static struct nvmf_transport_ops nvme_tcp_transport = {
 	.module		= THIS_MODULE,
 	.required_opts	= NVMF_OPT_TRADDR,
 	.allowed_opts	= NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
-			  NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO,
+			  NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
+			  NVMF_OPT_TLS_ENABLE,
 	.create_ctrl	= nvme_tcp_create_ctrl,
 };
 
-- 
2.14.4

[PATCH 2/3] nvme-tcp: TLS encrytion support using netlink sockets

[Shradha Shah]

Signed-off-by: Shradha Shah <sshah at solarflare.com>
---
 drivers/nvme/host/nvme.h          |   1 +
 drivers/nvme/host/tcp.c           | 150 ++++++++++++++++++++++++++++++++++++--
 include/uapi/linux/nvme_netlink.h |  43 +++++++++++
 3 files changed, 186 insertions(+), 8 deletions(-)
 create mode 100644 include/uapi/linux/nvme_netlink.h

diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 06ebc7302aed..1a54f18eb345 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -207,6 +207,7 @@ struct nvme_ctrl {
 	u16 maxcmd;
 	int nr_reconnects;
 	bool tls_enable;
+	struct sock *nl_sk;
 	struct nvmf_ctrl_options *opts;
 	struct delayed_work reconnect_work;
 	struct work_struct err_work;
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index a1e16457b74e..8ac282b7ecf6 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -20,7 +20,7 @@
 #include <net/sock.h>
 #include <net/tcp.h>
 #include <linux/blk-mq.h>
-
+#include <uapi/linux/nvme_netlink.h>
 #include "nvme.h"
 #include "fabrics.h"
 
@@ -108,6 +108,8 @@ struct nvme_tcp_queue {
 	struct nvme_tcp_queue_tx_context tx;
 
 	struct nvme_tcp_request async_req;
+	struct nvme_tcp_uevent tls_handshake_uevent;
+	wait_queue_head_t tls_handshake_wait;
 
 	void (*sc)(struct sock *);
 	void (*dr)(struct sock *);
@@ -996,6 +998,79 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
 	return ret;
 }
 
+static void nl_data_ready(struct sk_buff *skb)
+{
+	struct nlmsghdr *nlh;
+	struct nvme_tcp_uevent *tls_handshake_response_ptr;
+	struct nvme_tcp_queue *queue;
+	int err;
+
+	queue = skb->sk->sk_user_data;
+
+	nlh = (struct nlmsghdr *)skb->data;
+	tls_handshake_response_ptr = (struct nvme_tcp_uevent *)NLMSG_DATA(nlh);
+
+	if (tls_handshake_response_ptr->ev_type == TLS_HANDSHAKE_COMPLETE) {
+		queue->tls_handshake_uevent.ev_type = TLS_HANDSHAKE_COMPLETE;
+		queue->tls_handshake_uevent.qid = tls_handshake_response_ptr->qid;
+		queue->tls_handshake_uevent.tlsrep.sd = tls_handshake_response_ptr->tlsrep.sd;
+		queue->tls_handshake_uevent.tlsrep.pid = tls_handshake_response_ptr->tlsrep.pid;
+		queue->sock = sockfd_lookup(queue->tls_handshake_uevent.tlsrep.sd,
+					    &err);
+		if (queue->sock == NULL) {
+			pr_err("sockfd_lookup error code is %d\n", err);
+			queue->tls_handshake_uevent.ev_type = TLS_HANDSHAKE_ERROR;
+		}
+	} else {
+		queue->tls_handshake_uevent.ev_type = TLS_HANDSHAKE_ERROR;
+		pr_err("TLS handshake failed, KTLS socket not created\n");
+	}
+	wake_up(&queue->tls_handshake_wait);
+}
+
+static int  nvme_netlink_handshake(struct nvme_ctrl *nctrl,
+				   struct nvme_tcp_queue *queue, int qid)
+{
+	struct sk_buff *skb = NULL;
+	struct nlmsghdr *nlh = NULL;
+	int msg_size;
+
+	queue->tls_handshake_uevent.ev_type = TLS_HANDSHAKE_INIT;
+	strncpy(queue->tls_handshake_uevent.tlsreq.hostnqn,
+		nctrl->opts->host->nqn, NVMF_NQN_SIZE);
+	strncpy(queue->tls_handshake_uevent.tlsreq.subsysnqn,
+		nctrl->opts->subsysnqn, NVMF_NQN_SIZE);
+	queue->tls_handshake_uevent.qid = qid;
+
+	msg_size = sizeof(struct nvme_tcp_uevent);
+
+	skb = nlmsg_new(msg_size, GFP_KERNEL);
+	if (skb == NULL) {
+		pr_err("NVMe TCP tls netlink skb NULL\n");
+		return -1;
+	}
+
+	nlh = nlmsg_put(skb, 0, 1, NLMSG_DONE, msg_size, 0);
+	if (!nlh) {
+		kfree_skb(skb);
+		pr_err("NVMe TCP tls nlh NULL\n");
+		return -1;
+	}
+
+	memcpy((nlmsg_data(nlh)),
+	       (const unsigned char *)&queue->tls_handshake_uevent,
+	       sizeof(struct nvme_tcp_uevent));
+
+	NETLINK_CB(skb).portid = 0;
+	NETLINK_CB(skb).dst_group = TLS_HANDSHAKE_GRP;
+
+	nctrl->nl_sk->sk_user_data = queue;
+
+	netlink_broadcast(nctrl->nl_sk, skb, 0, TLS_HANDSHAKE_GRP, GFP_KERNEL);
+
+	return 0;
+}
+
 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
 		int qid, size_t queue_size)
 {
@@ -1009,18 +1084,44 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
 	INIT_WORK(&queue->io_work, nvme_tcp_io_work);
 	queue->queue_size = queue_size;
 
+	init_waitqueue_head(&queue->tls_handshake_wait);
+
 	if (qid > 0)
 		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
 	else
 		queue->cmnd_capsule_len = sizeof(struct nvme_command) +
 						NVME_TCP_ADMIN_CCSZ;
 
-	ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
-			IPPROTO_TCP, &queue->sock);
-	if (ret) {
-		dev_err(ctrl->ctrl.device,
-			"failed to create socket: %d\n", ret);
-		return ret;
+	if (nctrl->tls_enable == true) {
+		ret = nvme_netlink_handshake(nctrl, queue, qid);
+		if (ret) {
+			dev_err(ctrl->ctrl.device,
+				"failed to create KTLS socket: %d\n", ret);
+			return ret;
+		}
+
+		ret = wait_event_interruptible_timeout(queue->tls_handshake_wait,
+						       ((queue->tls_handshake_uevent.ev_type == TLS_HANDSHAKE_COMPLETE) |
+							(queue->tls_handshake_uevent.ev_type == TLS_HANDSHAKE_ERROR)),
+						       msecs_to_jiffies(5000));
+		if (ret == 0)
+			pr_err("TLS handshake failed, timeout elapsed\n");
+		else if (ret < 0)
+			pr_err("Interrupted by signal\n");
+
+		if (queue->sock == NULL) {
+			dev_err(ctrl->ctrl.device,
+				"failed to create socket\n");
+			return -ENOTSOCK;
+		}
+	} else {
+		ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
+				  IPPROTO_TCP, &queue->sock);
+		if (ret) {
+			dev_err(ctrl->ctrl.device,
+				"failed to create socket: %d\n", ret);
+			return ret;
+		}
 	}
 
 	/* Single syn retry */
@@ -1363,6 +1464,14 @@ int nvme_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
 }
 EXPORT_SYMBOL_GPL(nvme_configure_io_queues);
 
+void nvme_release_netlink_socket(struct nvme_ctrl *ctrl)
+{
+	if ((ctrl->tls_enable == true) && (ctrl->nl_sk != NULL)) {
+		netlink_kernel_release(ctrl->nl_sk);
+		ctrl->nl_sk = NULL;
+	}
+}
+
 void nvme_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
 {
 	nvme_tcp_stop_admin_queue(ctrl);
@@ -1372,6 +1481,7 @@ void nvme_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
 		nvme_tcp_free_tagset(ctrl, ctrl->admin_tagset);
 	}
 	nvme_tcp_free_admin_queue(ctrl);
+	nvme_release_netlink_socket(ctrl);
 }
 EXPORT_SYMBOL_GPL(nvme_destroy_admin_queue);
 
@@ -1648,6 +1758,22 @@ static int nvme_tcp_post_configure(struct nvme_ctrl *ctrl)
 	return 0;
 }
 
+static int open_netlink_socket(struct nvme_ctrl *ctrl)
+{
+	struct netlink_kernel_cfg cfg = {
+		.groups = TLS_HANDSHAKE_GRP,
+		.input =  nl_data_ready,
+	};
+
+	ctrl->nl_sk = netlink_kernel_create(&init_net, NETLINK_USERSOCK, &cfg);
+	if (!ctrl->nl_sk) {
+		pr_err("Error creating netlink socket\n");
+		return -ENOTSOCK;
+	}
+
+	return 0;
+}
+
 int nvme_probe_ctrl(struct nvme_ctrl *ctrl)
 {
 	bool changed;
@@ -1655,9 +1781,15 @@ int nvme_probe_ctrl(struct nvme_ctrl *ctrl)
 
 	INIT_WORK(&ctrl->reset_work, nvme_reset_ctrl_work);
 
+	if (ctrl->tls_enable == true) {
+		ret = open_netlink_socket(ctrl);
+		if (ret)
+			goto err;
+	}
+
 	ret = nvme_configure_admin_queue(ctrl, true);
 	if (ret)
-		goto err;
+		goto out_remove_netlink_socket;
 
 	ret = nvme_tcp_post_configure(ctrl);
 	if (ret)
@@ -1677,6 +1809,8 @@ int nvme_probe_ctrl(struct nvme_ctrl *ctrl)
 	return 0;
 out_remove_admin_queue:
 	nvme_destroy_admin_queue(ctrl, true);
+out_remove_netlink_socket:
+	nvme_release_netlink_socket(ctrl);
 err:
 	if (ret > 0)
 		ret = -EIO;
diff --git a/include/uapi/linux/nvme_netlink.h b/include/uapi/linux/nvme_netlink.h
new file mode 100644
index 000000000000..65801fa24aae
--- /dev/null
+++ b/include/uapi/linux/nvme_netlink.h
@@ -0,0 +1,43 @@
+/*
+ * Definitions for the NVM Express over fabrics netlink interface
+ * Copyright (c) 2011-2014, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _UAPI_LINUX_NVME_NETLINK_H
+#define _UAPI_LINUX_NVME_NETLINK_H
+
+#include <linux/types.h>
+
+#define TLS_HANDSHAKE_INIT 1
+#define TLS_HANDSHAKE_COMPLETE 2
+#define TLS_HANDSHAKE_ERROR 3
+
+#define TLS_HANDSHAKE_GRP 21
+
+#define NVMF_NQN_SIZE		223
+
+struct nvme_tcp_uevent {
+	__u32	ev_type;
+	__u32 qid;
+	struct tls_uconnect_request {
+		char			hostnqn[NVMF_NQN_SIZE];
+		char			subsysnqn[NVMF_NQN_SIZE];
+	} tlsreq;
+	struct tls_uconnect_reply {
+		__u32			 sd;
+		__u32			pid;
+	} tlsrep;
+};
+
+#endif
-- 
2.14.4

[PATCH 3/3] User mode server and client code

[Shradha Shah]

Signed-off-by: Shradha Shah <sshah at solarflare.com>
---
 tls/Makefile |  12 +++
 tls/client.c | 308 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tls/server.c | 183 +++++++++++++++++++++++++++++++++++
 3 files changed, 503 insertions(+)
 create mode 100644 tls/Makefile
 create mode 100644 tls/client.c
 create mode 100644 tls/server.c

diff --git a/tls/Makefile b/tls/Makefile
new file mode 100644
index 000000000000..3f387eae31b4
--- /dev/null
+++ b/tls/Makefile
@@ -0,0 +1,12 @@
+CC = gcc
+CFLAGS = -Wall -pedantic -O2 -std=gnu99
+LDFLAGS = -lgnutls -lpthread
+
+client: client.c
+	${CC} ${CLFLAGS} ${LDFLAGS} $^ -o $@
+
+server: server.c
+	${CC} ${CFLAGS} ${LDFLAGS} $^ -o $@
+
+clean:
+	rm -f *.o client server
diff --git a/tls/client.c b/tls/client.c
new file mode 100644
index 000000000000..fd3582be3ee9
--- /dev/null
+++ b/tls/client.c
@@ -0,0 +1,308 @@
+/*
+ * TLS support for NVMe over TCP.
+ * Copyright (c) 2017 Shradha Shah <sshah at solarflare.com>.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <string.h>
+#include <assert.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <gnutls/gnutls.h>
+#include <malloc.h>
+#include <alloca.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <linux/netlink.h>
+#include <linux/tls.h>
+#include "../include/uapi/linux/nvme_netlink.h"
+
+#define PORT 5557
+#define EXIT_SUCCESS 0
+#define EXIT_FAILURE 1
+
+gnutls_psk_client_credentials_t pskcred;
+gnutls_session_t session[100];
+
+int ktls_socket_init(gnutls_session_t session)
+{
+	int err, ksd;
+	gnutls_datum_t mac_key;
+	gnutls_datum_t iv_read;
+	gnutls_datum_t iv_write;
+	gnutls_datum_t cipher_key_read;
+	gnutls_datum_t cipher_key_write;
+	unsigned char seq_number_read[8];
+	unsigned char seq_number_write[8];
+	struct tls12_crypto_info_aes_gcm_128 crypto_info;
+
+	/* now we need to initialize state after the handshake in the kernel*/
+	err = gnutls_record_get_state(session, 1, &mac_key, &iv_read,
+				      &cipher_key_read, seq_number_read);
+	if (err < 0) {
+		printf("failed to get receiving state from GnuTLS session/n");
+		goto end;
+	}
+
+	err = gnutls_record_get_state(session, 0, &mac_key, &iv_write,
+				      &cipher_key_write, seq_number_write);
+	if (err < 0) {
+		printf("failed to get sending state from Gnu TLS session/n");
+		goto end;
+	}
+
+	ksd = socket(AF_INET, SOCK_STREAM, 0);
+	if (ksd == -1) {
+		printf("socket error:/n");
+		return -1;
+	}
+
+	crypto_info.info.version = TLS_1_2_VERSION;
+	crypto_info.info.cipher_type = TLS_CIPHER_AES_GCM_128;
+	memcpy(crypto_info.iv, iv_write.data,  TLS_CIPHER_AES_GCM_128_IV_SIZE);
+	memcpy(crypto_info.rec_seq, seq_number_write,
+	       TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
+	memcpy(crypto_info.key, cipher_key_write.data,
+	       TLS_CIPHER_AES_GCM_128_KEY_SIZE);
+	memcpy(crypto_info.salt, iv_write.data,
+	       TLS_CIPHER_AES_GCM_128_SALT_SIZE);
+
+	err = setsockopt(ksd, SOL_TCP, TLS_TX, &crypto_info,
+			 sizeof(crypto_info));
+	if (err) {
+		printf("failed to set TLS sock opt %d\n", err);
+		goto end;
+	} else {
+		printf("created TLS socket fd=%d\n", ksd);
+	}
+	return ksd;
+ end:
+	return err;
+}
+
+void tls_close_session(gnutls_session_t session)
+{
+	gnutls_bye(session, GNUTLS_SHUT_RDWR);
+	gnutls_deinit(session);
+	gnutls_psk_free_client_credentials(pskcred);
+	gnutls_global_deinit();
+}
+
+int tls_handshake(int session_id, char *host)
+{
+	int ret, sd;
+	struct sockaddr_in sa;
+	char *desc;
+	const char *hint;
+	const gnutls_datum_t key = { (void *) "DEADBEEF", 8 };
+
+	/* for backwards compatibility with gnutls < 3.3.0 */
+	gnutls_global_init();
+	gnutls_psk_allocate_client_credentials(&pskcred);
+	gnutls_psk_set_client_credentials(pskcred, "test", &key,
+					  GNUTLS_PSK_KEY_HEX);
+
+	/* Initialize TLS session */
+	gnutls_init(&session[session_id], GNUTLS_CLIENT);
+	gnutls_priority_set_direct(session[session_id], "NORMAL:-KX-ALL:+PSK",
+				   NULL);
+	gnutls_credentials_set(session[session_id], GNUTLS_CRD_PSK, pskcred);
+
+	sd = socket(AF_INET, SOCK_STREAM, 0);
+	memset(&sa, '\0', sizeof(sa));
+	sa.sin_family = AF_INET;
+	sa.sin_port = htons(PORT);
+	inet_pton(AF_INET, host, &sa.sin_addr);
+
+	ret = connect(sd, (struct sockaddr *) &sa, sizeof(sa));
+	if (ret) {
+		fprintf(stderr, "Connect error%d\n", errno);
+		goto end;
+	}
+
+	gnutls_transport_set_int(session[session_id], sd);
+	gnutls_handshake_set_timeout(session[session_id],
+				     GNUTLS_DEFAULT_HANDSHAKE_TIMEOUT);
+
+	/* Perform the TLS handshake */
+	do {
+		ret = gnutls_handshake(session[session_id]);
+	} while (ret < 0 && gnutls_error_is_fatal(ret) == 0);
+	if (ret < 0) {
+		fprintf(stderr, "*** Handshake failed: %s\n",
+			gnutls_strerror(ret));
+		goto close;
+	}
+	printf("- Handshake was completed");
+	desc = gnutls_session_get_desc(session[session_id]);
+	printf("- Session info: %s\n", desc);
+
+	hint = gnutls_psk_client_get_hint(session[session_id]);
+	//replace the string "hint" with subsystem NQN
+	if (hint == NULL || strcmp(hint, "hint") != 0) {
+		printf("client: hint is not the expected: %s\n",
+		       gnutls_psk_client_get_hint(session[session_id]));
+		goto close;
+	}
+
+	gnutls_free(desc);
+
+	ret = ktls_socket_init(session[session_id]);
+	if (ret < 0) {
+		printf("failed to get KTLS socket");
+		goto close;
+	} else
+		goto success;
+close:
+	shutdown(sd, SHUT_RDWR); //no more receptions
+end:
+	tls_close_session(session[session_id]);
+success:
+	close(sd);
+	return ret;
+}
+
+static void daemonize(void)
+{
+	pid_t pid, sid;
+	int fd;
+
+	if (getppid() == 1)
+		return;
+	pid = fork();
+	if (pid < 0)
+		exit(EXIT_FAILURE);
+	if (pid > 0)
+		exit(EXIT_SUCCESS);
+
+	sid = setsid();
+	if (sid < 0)
+		exit(EXIT_FAILURE);
+
+	if ((chdir("/")) < 0)
+		exit(EXIT_FAILURE);
+
+	fd = open("/dev/null", O_RDWR, 0);
+	if (fd != -1) {
+		dup2(fd, STDIN_FILENO);
+		dup2(fd, STDOUT_FILENO);
+		dup2(fd, STDERR_FILENO);
+		if (fd > 2)
+			close(fd);
+	}
+	umask(027);
+}
+
+int main(int argc, char **argv)
+{
+	struct sockaddr_nl src_addr, dest_addr;
+	struct nlmsghdr *nlh_recv = NULL;
+	struct nlmsghdr *nlh_send = NULL;
+	struct msghdr msg;
+	struct iovec iov;
+	char buffer[4096];
+	int sock_fd, ktls_socket, i = 0, max_session_count = 0;
+	int group = TLS_HANDSHAKE_GRP;
+	struct nvme_tcp_uevent *tls_handshake_response_ptr;
+	int rc;
+
+	if (argc != 2) {
+		printf("TOO FEW ARGUMENTS\n");
+		exit(-1);
+	}
+
+	daemonize();
+
+	sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_USERSOCK);
+
+	memset(&src_addr, 0, sizeof(src_addr));
+	src_addr.nl_family = AF_NETLINK;
+	src_addr.nl_pid = getpid();
+
+	bind(sock_fd, (struct sockaddr *)&src_addr,
+	     sizeof(src_addr));
+
+	if (setsockopt(sock_fd, SOL_NETLINK, NETLINK_ADD_MEMBERSHIP,
+		       &group, sizeof(group)) < 0)
+		printf("setsockopt < 0\n");
+
+	memset(&dest_addr, 0, sizeof(dest_addr));
+	dest_addr.nl_family = AF_NETLINK;
+	dest_addr.nl_pid = 0;
+	dest_addr.nl_groups = 0;
+
+	while (1) {
+		iov.iov_base = (void *)buffer;
+		iov.iov_len = sizeof(buffer);
+		msg.msg_name = (void *)&dest_addr;
+		msg.msg_namelen = sizeof(dest_addr);
+		msg.msg_iov = &iov;
+		msg.msg_iovlen = 1;
+
+		printf("Waiting for message from kernel\n");
+
+		rc = recvmsg(sock_fd, &msg, 0);
+		if (rc > 0) {
+			nlh_recv = (struct nlmsghdr *)buffer;
+
+			tls_handshake_response_ptr = (struct nvme_tcp_uevent *)NLMSG_DATA(nlh_recv);
+
+			if (tls_handshake_response_ptr->ev_type == TLS_HANDSHAKE_INIT) {
+				ktls_socket = tls_handshake(i, argv[1]);
+				if (ktls_socket < 0) {
+					tls_handshake_response_ptr->ev_type = TLS_HANDSHAKE_ERROR;
+				} else {
+					i++;
+					max_session_count++;
+					tls_handshake_response_ptr->ev_type = TLS_HANDSHAKE_COMPLETE;
+					tls_handshake_response_ptr->tlsrep.sd = ktls_socket;
+					tls_handshake_response_ptr->tlsrep.pid = getpid();
+				}
+
+				nlh_send = (struct nlmsghdr *)malloc(NLMSG_SPACE(sizeof(struct nvme_tcp_uevent)));
+				memset(nlh_send, 0,
+				       NLMSG_SPACE(sizeof(struct nvme_tcp_uevent)));
+				nlh_send->nlmsg_len = NLMSG_SPACE(sizeof(struct nvme_tcp_uevent));
+				nlh_send->nlmsg_pid = getpid();
+				nlh_send->nlmsg_flags = 0;
+
+				memcpy((NLMSG_DATA(nlh_send)), (const unsigned char *)(NLMSG_DATA(nlh_recv)), sizeof(struct nvme_tcp_uevent));
+
+				iov.iov_base = (void *)nlh_send;
+				iov.iov_len = nlh_send->nlmsg_len;
+				msg.msg_name = (void *)&dest_addr;
+				msg.msg_namelen = sizeof(dest_addr);
+				msg.msg_iov = &iov;
+				msg.msg_iovlen = 1;
+
+				printf("Sending message to kernel\n");
+				sendmsg(sock_fd, &msg, 0);
+				free(nlh_send);
+			}
+			memset(&msg, 0, sizeof(msg));
+			memset(&iov, 0, sizeof(iov));
+			memset(buffer, 0, 4096);
+		}
+	}
+	close(sock_fd);
+	for (i = 0; i < max_session_count; i++)
+		tls_close_session(session[i]);
+	return ktls_socket;
+}
+
diff --git a/tls/server.c b/tls/server.c
new file mode 100644
index 000000000000..86f3c959077b
--- /dev/null
+++ b/tls/server.c
@@ -0,0 +1,183 @@
+/*
+ * TLS support for NVMe over TCP.
+ * Copyright (c) 2017 Shradha Shah <sshah at solarflare.com>.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <assert.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <malloc.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <sys/select.h>
+#include <netdb.h>
+
+#include <gnutls/gnutls.h>
+#include <gnutls/compat.h>
+#include <gnutls/crypto.h>
+
+#define PORT 5557 /* listen to 5556 port */
+#define KEYFILE "key.pem"
+#define CERTFILE "cert.pem"
+#define CAFILE "/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt"
+#define CRLFILE "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"
+#define EXIT_SUCCESS 0
+#define EXIT_FAILURE 1
+
+static gnutls_dh_params_t dh_params;
+
+static int pskfunc(gnutls_session_t session, const char *username,
+		   gnutls_datum_t *key)
+{
+	printf("psk: username %s\n", username);
+	key->data = gnutls_malloc(4);
+	key->data[0] = 0xDE;
+	key->data[1] = 0xAD;
+	key->data[2] = 0xBE;
+	key->data[3] = 0xEF;
+	key->size = 4;
+	return 0;
+}
+
+static int generate_dh_params(void)
+{
+	int bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
+					       GNUTLS_SEC_PARAM_LEGACY);
+
+	/* Generate Diffie-Hellman parameters - for use with DHE
+	 * kx algorithms. When short bit length is used, it might
+	 * be wise to regenerate parameters often.
+	 */
+	gnutls_dh_params_init(&dh_params);
+	gnutls_dh_params_generate2(dh_params, bits);
+	return EXIT_SUCCESS;
+}
+
+static void daemonize(void)
+{
+	pid_t pid, sid;
+	int fd;
+
+	if (getppid() == 1)
+		return;
+
+	pid = fork();
+	if (pid < 0)
+		exit(EXIT_FAILURE);
+	if (pid > 0)
+		exit(EXIT_SUCCESS);
+
+	sid = setsid();
+	if (sid < 0)
+		exit(EXIT_FAILURE);
+
+	if ((chdir("/")) < 0)
+		exit(EXIT_FAILURE);
+
+	fd = open("/dev/null", O_RDWR, 0);
+	if (fd != -1) {
+		dup2(fd, STDIN_FILENO);
+		dup2(fd, STDOUT_FILENO);
+		dup2(fd, STDERR_FILENO);
+		if (fd > 2)
+			close(fd);
+	}
+	umask(027);
+}
+
+int main(void)
+{
+	int listen_sd;
+	int sd, ret;
+	gnutls_psk_server_credentials_t server_pskcred;
+	struct sockaddr_in sa_serv;
+	struct sockaddr_in sa_cli;
+	socklen_t client_len;
+	char topbuf[512];
+	gnutls_session_t session;
+	int optval = 1;
+	char *desc;
+
+	daemonize();
+	/* for backwards compatibility with gnutls < 3.3.0 */
+	gnutls_global_init();
+	gnutls_psk_allocate_server_credentials(&server_pskcred);
+	gnutls_psk_set_server_credentials_hint(server_pskcred, "hint");
+	gnutls_psk_set_server_credentials_function(server_pskcred, pskfunc);
+	generate_dh_params();
+	gnutls_psk_set_server_dh_params(server_pskcred, dh_params);
+
+	/* Socket operations */
+	listen_sd = socket(AF_INET, SOCK_STREAM, 0);
+
+	memset(&sa_serv, '\0', sizeof(sa_serv));
+	sa_serv.sin_family = AF_INET;
+	sa_serv.sin_addr.s_addr = INADDR_ANY;
+	sa_serv.sin_port = htons(PORT); /* Server Port number */
+
+	setsockopt(listen_sd, SOL_SOCKET, SO_REUSEADDR,
+		   (void *) &optval, sizeof(int));
+	bind(listen_sd, (struct sockaddr *) &sa_serv, sizeof(sa_serv));
+	listen(listen_sd, 1024);
+	printf("Server ready. Listening to port '%d'.", PORT);
+
+	client_len = sizeof(sa_cli);
+	for (;;) {
+		gnutls_init(&session, GNUTLS_SERVER);
+		gnutls_priority_set_direct(session, "NORMAL:-KX-ALL:+PSK",
+					   NULL);
+		gnutls_credentials_set(session, GNUTLS_CRD_PSK,
+				       server_pskcred);
+
+		sd = accept(listen_sd, (struct sockaddr *) &sa_cli,
+			    &client_len);
+		printf("- connection from %s, port %d",
+		       inet_ntop(AF_INET, &sa_cli.sin_addr, topbuf,
+				 sizeof(topbuf)), ntohs(sa_cli.sin_port));
+		gnutls_transport_set_int(session, sd);
+		do {
+			ret = gnutls_handshake(session);
+		} while (ret < 0 && gnutls_error_is_fatal(ret) == 0);
+		if (ret < 0) {
+			close(sd);
+			gnutls_deinit(session);
+			fprintf(stderr, "*** Handshake has failed (%s)",
+				gnutls_strerror(ret));
+			continue;
+		}
+		printf("- Handshake was completed");
+		desc = gnutls_session_get_desc(session);
+		printf("- Session info: %s\n", desc);
+		gnutls_free(desc);
+
+		//KTLS socket
+		//pass the KTLS socket to kernel
+
+		close(sd);
+	}
+	close(listen_sd);
+	/* do not wait for the peer to close the connection. */
+	gnutls_bye(session, GNUTLS_SHUT_WR);
+	gnutls_deinit(session);
+	gnutls_dh_params_deinit(dh_params);
+	gnutls_psk_free_server_credentials(server_pskcred);
+	gnutls_global_deinit();
+
+	return EXIT_SUCCESS;
+}
+
-- 
2.14.4

[PATCH 0/3] TLS initiator side implementation

[Sagi Grimberg]

Wrong mailing list...

You can safely ignore...