[patch 0/9] [rfc] synchronisation of persistence engines

[patch 0/9] [rfc] synchronisation of persistence engines

[Simon Horman]

Hi,

This is an extension of the synchronisation protocol
to support the synchronisation of persistence engine data.

Although it is called v2, it is only WIP and does not meet
many of the stated goals of v2 - notably no IPv6, fwmark
or timeout synchronisation.

It is not intended to compete with other work in this area,
rather to add to the discussion.

[patch 1/9] IPVS: Make the cp argument to ip_vs_sync_conn() static

[Simon Horman]

[-- Attachment #1: sync-ip_vs_sync_conn-const.patch --]
[-- Type: text/plain, Size: 1274 bytes --]

Signed-off-by: Simon Horman <horms@verge.net.au>

Index: lvs-test-2.6/include/net/ip_vs.h
===================================================================
--- lvs-test-2.6.orig/include/net/ip_vs.h	2010-10-10 23:29:32.000000000 +0900
+++ lvs-test-2.6/include/net/ip_vs.h	2010-10-10 23:29:40.000000000 +0900
@@ -889,7 +889,7 @@ extern char ip_vs_master_mcast_ifn[IP_VS
 extern char ip_vs_backup_mcast_ifn[IP_VS_IFNAME_MAXLEN];
 extern int start_sync_thread(int state, char *mcast_ifn, __u8 syncid);
 extern int stop_sync_thread(int state);
-extern void ip_vs_sync_conn(struct ip_vs_conn *cp);
+extern void ip_vs_sync_conn(const struct ip_vs_conn *cp);
 
 
 /*
Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 23:29:49.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 23:29:52.000000000 +0900
@@ -236,7 +236,7 @@ get_curr_sync_buff(unsigned long time)
  *      Add an ip_vs_conn information into the current sync_buff.
  *      Called by ip_vs_in.
  */
-void ip_vs_sync_conn(struct ip_vs_conn *cp)
+void ip_vs_sync_conn(const struct ip_vs_conn *cp)
 {
 	struct ip_vs_sync_mesg *m;
 	struct ip_vs_sync_conn *s;

[patch 2/9] IPVS: Remove useless { } block from ip_vs_process_message()

[Simon Horman]

[-- Attachment #1: ip_vs_process_message-indentation.patch --]
[-- Type: text/plain, Size: 1291 bytes --]

Signed-off-by: Simon Horman <horms@verge.net.au>

Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 11:14:04.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 11:14:11.000000000 +0900
@@ -381,20 +381,18 @@ static void ip_vs_process_message(const
 			}
 		}
 
-		{
-			if (ip_vs_conn_fill_param_sync(AF_INET, s->protocol,
-					      (union nf_inet_addr *)&s->caddr,
-					      s->cport,
-					      (union nf_inet_addr *)&s->vaddr,
-					      s->vport, &param)) {
-				pr_err("ip_vs_conn_fill_param_sync failed");
-				return;
-			}
-			if (!(flags & IP_VS_CONN_F_TEMPLATE))
-				cp = ip_vs_conn_in_get(&param);
-			else
-				cp = ip_vs_ct_in_get(&param);
+		if (ip_vs_conn_fill_param_sync(AF_INET, s->protocol,
+					       (union nf_inet_addr *)&s->caddr,
+					       s->cport,
+					       (union nf_inet_addr *)&s->vaddr,
+					       s->vport, &param)) {
+			pr_err("ip_vs_conn_fill_param_sync failed");
+			return;
 		}
+		if (!(flags & IP_VS_CONN_F_TEMPLATE))
+			cp = ip_vs_conn_in_get(&param);
+		else
+			cp = ip_vs_ct_in_get(&param);
 		if (!cp) {
 			/*
 			 * Find the appropriate destination for the connection.

[patch 3/9] IPVS: buffer argument to ip_vs_process_message() should not be const

[Simon Horman]

[-- Attachment #1: sync-ip_vs_process_message-no-const-buffer.patch --]
[-- Type: text/plain, Size: 827 bytes --]

It is assigned to a non-const variable and its contents are modified.

Signed-off-by: Simon Horman <horms@verge.net.au>

Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 10:39:13.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 10:41:54.000000000 +0900
@@ -303,7 +303,7 @@ ip_vs_conn_fill_param_sync(int af, int p
  *      Process received multicast message and create the corresponding
  *      ip_vs_conn entries.
  */
-static void ip_vs_process_message(const char *buffer, const size_t buflen)
+static void ip_vs_process_message(char *buffer, const size_t buflen)
 {
 	struct ip_vs_sync_mesg *m = (struct ip_vs_sync_mesg *)buffer;
 	struct ip_vs_sync_conn *s;

[patch 4/9] IPVS: Add persistence engine to connection entry

[Simon Horman]

[-- Attachment #1: sync-conn-pe-2.patch --]
[-- Type: text/plain, Size: 5586 bytes --]

The dest of a connection may not exist if it has been created as the result
of connection synchronisation. But in order for connection entries for
templates with persistence engine data created through connection
synchronisation to be valid access to the persistence engine pointer is
required.  So add the persistence engine to the connection itself.

Signed-off-by: Simon Horman <horms@verge.net.au>

Index: lvs-test-2.6/include/net/ip_vs.h
===================================================================
--- lvs-test-2.6.orig/include/net/ip_vs.h	2010-10-15 08:12:46.000000000 +0900
+++ lvs-test-2.6/include/net/ip_vs.h	2010-10-15 08:14:35.000000000 +0900
@@ -421,6 +421,7 @@ struct ip_vs_conn {
 	struct ip_vs_seq        in_seq;         /* incoming seq. struct */
 	struct ip_vs_seq        out_seq;        /* outgoing seq. struct */
 
+	const struct ip_vs_pe	*pe;
 	char			*pe_data;
 	__u8			pe_data_len;
 };
@@ -801,8 +802,19 @@ void ip_vs_bind_pe(struct ip_vs_service
 void ip_vs_unbind_pe(struct ip_vs_service *svc);
 int register_ip_vs_pe(struct ip_vs_pe *pe);
 int unregister_ip_vs_pe(struct ip_vs_pe *pe);
-extern struct ip_vs_pe *ip_vs_pe_get(const char *name);
-extern void ip_vs_pe_put(struct ip_vs_pe *pe);
+struct ip_vs_pe *ip_vs_pe_getbyname(const char *name);
+
+static inline void ip_vs_pe_get(const struct ip_vs_pe *pe)
+{
+	if (pe && pe->module)
+		__module_get(pe->module);
+}
+
+static inline void ip_vs_pe_put(const struct ip_vs_pe *pe)
+{
+	if (pe && pe->module)
+		module_put(pe->module);
+}
 
 /*
  *	IPVS protocol functions (from ip_vs_proto.c)
Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_conn.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_conn.c	2010-10-15 08:12:46.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_conn.c	2010-10-15 08:12:51.000000000 +0900
@@ -176,8 +176,8 @@ static unsigned int ip_vs_conn_hashkey_c
 	ip_vs_conn_fill_param(cp->af, cp->protocol, &cp->caddr, cp->cport,
 			      NULL, 0, &p);
 
-	if (cp->dest && cp->dest->svc->pe) {
-		p.pe = cp->dest->svc->pe;
+	if (cp->pe) {
+		p.pe = cp->pe;
 		p.pe_data = cp->pe_data;
 		p.pe_data_len = cp->pe_data_len;
 	}
@@ -763,6 +763,7 @@ static void ip_vs_conn_expire(unsigned l
 		if (cp->flags & IP_VS_CONN_F_NFCT)
 			ip_vs_conn_drop_conntrack(cp);
 
+		ip_vs_pe_put(cp->pe);
 		kfree(cp->pe_data);
 		if (unlikely(cp->app != NULL))
 			ip_vs_unbind_app(cp);
@@ -824,7 +825,9 @@ ip_vs_conn_new(const struct ip_vs_conn_p
 			&cp->daddr, daddr);
 	cp->dport          = dport;
 	cp->flags	   = flags;
-	if (flags & IP_VS_CONN_F_TEMPLATE && p->pe_data) {
+	if (flags & IP_VS_CONN_F_TEMPLATE && p->pe) {
+		ip_vs_pe_get(p->pe);
+		cp->pe = p->pe;
 		cp->pe_data = p->pe_data;
 		cp->pe_data_len = p->pe_data_len;
 	}
@@ -956,15 +959,13 @@ static int ip_vs_conn_seq_show(struct se
 		char pe_data[IP_VS_PENAME_MAXLEN + IP_VS_PEDATA_MAXLEN + 3];
 		size_t len = 0;
 
-		if (cp->dest && cp->pe_data &&
-		    cp->dest->svc->pe->show_pe_data) {
+		if (cp->pe_data) {
 			pe_data[0] = ' ';
-			len = strlen(cp->dest->svc->pe->name);
-			memcpy(pe_data + 1, cp->dest->svc->pe->name, len);
+			len = strlen(cp->pe->name);
+			memcpy(pe_data + 1, cp->pe->name, len);
 			pe_data[len + 1] = ' ';
 			len += 2;
-			len += cp->dest->svc->pe->show_pe_data(cp,
-							       pe_data + len);
+			len += cp->pe->show_pe_data(cp, pe_data + len);
 		}
 		pe_data[len] = '\0';
 
Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_pe.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_pe.c	2010-10-15 08:12:46.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_pe.c	2010-10-15 08:12:51.000000000 +0900
@@ -30,7 +30,7 @@ void ip_vs_unbind_pe(struct ip_vs_servic
 
 /* Get pe in the pe list by name */
 static struct ip_vs_pe *
-ip_vs_pe_getbyname(const char *pe_name)
+__ip_vs_pe_getbyname(const char *pe_name)
 {
 	struct ip_vs_pe *pe;
 
@@ -60,28 +60,22 @@ ip_vs_pe_getbyname(const char *pe_name)
 }
 
 /* Lookup pe and try to load it if it doesn't exist */
-struct ip_vs_pe *ip_vs_pe_get(const char *name)
+struct ip_vs_pe *ip_vs_pe_getbyname(const char *name)
 {
 	struct ip_vs_pe *pe;
 
 	/* Search for the pe by name */
-	pe = ip_vs_pe_getbyname(name);
+	pe = __ip_vs_pe_getbyname(name);
 
 	/* If pe not found, load the module and search again */
 	if (!pe) {
 		request_module("ip_vs_pe_%s", name);
-		pe = ip_vs_pe_getbyname(name);
+		pe = __ip_vs_pe_getbyname(name);
 	}
 
 	return pe;
 }
 
-void ip_vs_pe_put(struct ip_vs_pe *pe)
-{
-	if (pe && pe->module)
-		module_put(pe->module);
-}
-
 /* Register a pe in the pe list */
 int register_ip_vs_pe(struct ip_vs_pe *pe)
 {
Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_ctl.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_ctl.c	2010-10-15 08:12:46.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_ctl.c	2010-10-15 08:12:51.000000000 +0900
@@ -1149,7 +1149,7 @@ ip_vs_add_service(struct ip_vs_service_u
 	}
 
 	if (u->pe_name && *u->pe_name) {
-		pe = ip_vs_pe_get(u->pe_name);
+		pe = ip_vs_pe_getbyname(u->pe_name);
 		if (pe == NULL) {
 			pr_info("persistence engine module ip_vs_pe_%s "
 				"not found\n", u->pe_name);
@@ -1260,7 +1260,7 @@ ip_vs_edit_service(struct ip_vs_service
 	old_sched = sched;
 
 	if (u->pe_name && *u->pe_name) {
-		pe = ip_vs_pe_get(u->pe_name);
+		pe = ip_vs_pe_getbyname(u->pe_name);
 		if (pe == NULL) {
 			pr_info("persistence engine module ip_vs_pe_%s "
 				"not found\n", u->pe_name);

[patch 5/9] IPVS: rename ip_vs_sync_mesg as ip_vs_sync_mesg_v1

[Simon Horman]

[-- Attachment #1: sync-ip_vs_sync_mesg_v1.patch --]
[-- Type: text/plain, Size: 3948 bytes --]

This is in preparation for adding a new version of the
synchronisation protocol.

* Change name of struct ip_vs_sync_mesg to struct ip_vs_sync_mesg_v1
* Change the name of SYNC_MESG_HEADER_LEN to SYNC_MESG_V1_HEADER_LEN
* Define SYNC_MESG_V1_HEADER_LEN as sizeof(struct ip_vs_sync_mesg_v1)
  instead of 4
* Always use SYNC_MESG_V1_HEADER_LEN instead of 4

Signed-off-by: Simon Horman <horms@verge.net.au>

Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-15 07:50:16.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-15 07:50:24.000000000 +0900
@@ -81,7 +81,8 @@ struct ip_vs_sync_thread_data {
 (sizeof(struct ip_vs_sync_conn) + sizeof(struct ip_vs_sync_conn_options))
 
 
-/*
+/* IPVS Connection Synchronisation v1
+
   The master mulitcasts messages to the backup load balancers in the
   following format.
 
@@ -102,10 +103,10 @@ struct ip_vs_sync_thread_data {
       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 */
 
-#define SYNC_MESG_HEADER_LEN	4
+#define SYNC_MESG_V1_HEADER_LEN	sizeof(struct ip_vs_sync_mesg_v1)
 #define MAX_CONNS_PER_SYNCBUFF	255 /* nr_conns in ip_vs_sync_mesg is 8 bit */
 
-struct ip_vs_sync_mesg {
+struct ip_vs_sync_mesg_v1 {
 	__u8                    nr_conns;
 	__u8                    syncid;
 	__u16                   size;
@@ -122,7 +123,7 @@ struct ip_vs_sync_buff {
 	unsigned long           firstuse;
 
 	/* pointers for the message data */
-	struct ip_vs_sync_mesg  *mesg;
+	struct ip_vs_sync_mesg_v1 *mesg;
 	unsigned char           *head;
 	unsigned char           *end;
 };
@@ -188,8 +189,8 @@ static inline struct ip_vs_sync_buff * i
 	}
 	sb->mesg->nr_conns = 0;
 	sb->mesg->syncid = ip_vs_master_syncid;
-	sb->mesg->size = 4;
-	sb->head = (unsigned char *)sb->mesg + 4;
+	sb->mesg->size = SYNC_MESG_V1_HEADER_LEN;
+	sb->head = (unsigned char *)sb->mesg + SYNC_MESG_V1_HEADER_LEN;
 	sb->end = (unsigned char *)sb->mesg + sync_send_mesg_maxlen;
 	sb->firstuse = jiffies;
 	return sb;
@@ -238,7 +239,7 @@ get_curr_sync_buff(unsigned long time)
  */
 void ip_vs_sync_conn(const struct ip_vs_conn *cp)
 {
-	struct ip_vs_sync_mesg *m;
+	struct ip_vs_sync_mesg_v1 *m;
 	struct ip_vs_sync_conn *s;
 	int len;
 
@@ -305,7 +306,7 @@ ip_vs_conn_fill_param_sync(int af, int p
  */
 static void ip_vs_process_message(char *buffer, const size_t buflen)
 {
-	struct ip_vs_sync_mesg *m = (struct ip_vs_sync_mesg *)buffer;
+	struct ip_vs_sync_mesg_v1 *m = (struct ip_vs_sync_mesg_v1 *)buffer;
 	struct ip_vs_sync_conn *s;
 	struct ip_vs_sync_conn_options *opt;
 	struct ip_vs_conn *cp;
@@ -315,7 +316,7 @@ static void ip_vs_process_message(char *
 	char *p;
 	int i;
 
-	if (buflen < sizeof(struct ip_vs_sync_mesg)) {
+	if (buflen < SYNC_MESG_V1_HEADER_LEN) {
 		IP_VS_ERR_RL("sync message header too short\n");
 		return;
 	}
@@ -335,7 +336,7 @@ static void ip_vs_process_message(char *
 		return;
 	}
 
-	p = (char *)buffer + sizeof(struct ip_vs_sync_mesg);
+	p = (char *)buffer + SYNC_MESG_V1_HEADER_LEN;
 	for (i=0; i<m->nr_conns; i++) {
 		unsigned flags, state;
 
@@ -540,8 +541,8 @@ static int set_sync_mesg_maxlen(int sync
 
 		num = (dev->mtu - sizeof(struct iphdr) -
 		       sizeof(struct udphdr) -
-		       SYNC_MESG_HEADER_LEN - 20) / SIMPLE_CONN_SIZE;
-		sync_send_mesg_maxlen = SYNC_MESG_HEADER_LEN +
+		       SYNC_MESG_V1_HEADER_LEN - 20) / SIMPLE_CONN_SIZE;
+		sync_send_mesg_maxlen = SYNC_MESG_V1_HEADER_LEN +
 			SIMPLE_CONN_SIZE * min(num, MAX_CONNS_PER_SYNCBUFF);
 		IP_VS_DBG(7, "setting the maximum length of sync sending "
 			  "message %d.\n", sync_send_mesg_maxlen);
@@ -719,7 +720,7 @@ ip_vs_send_async(struct socket *sock, co
 }
 
 static void
-ip_vs_send_sync_msg(struct socket *sock, struct ip_vs_sync_mesg *msg)
+ip_vs_send_sync_msg(struct socket *sock, struct ip_vs_sync_mesg_v1 *msg)
 {
 	int msize;
 

[patch 6/9] IPVS: Add ip_vs_sync_mesg_v2

[Simon Horman]

[-- Attachment #1: sync-ip_vs_sync_mesg_v2.patch --]
[-- Type: text/plain, Size: 2609 bytes --]

Signed-off-by: Simon Horman <horms@verge.net.au>

Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 11:31:00.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 11:46:51.000000000 +0900
@@ -104,7 +104,6 @@ struct ip_vs_sync_thread_data {
 */
 
 #define SYNC_MESG_V1_HEADER_LEN	sizeof(struct ip_vs_sync_mesg_v1)
-#define MAX_CONNS_PER_SYNCBUFF	255 /* nr_conns in ip_vs_sync_mesg is 8 bit */
 
 struct ip_vs_sync_mesg_v1 {
 	__u8                    nr_conns;
@@ -114,6 +113,46 @@ struct ip_vs_sync_mesg_v1 {
 	/* ip_vs_sync_conn entries start here */
 };
 
+/* IPVS Connection Synchronisation v2
+
+  The master mulitcasts messages to the backup load balancers in the
+  following format.
+
+       0                   1                   2                   3
+       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+      |  Count Conns  |    SyncID     |            Zero               |
+      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+      |    Version    |     Zero      |            Size               |
+      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+      |                                                               |
+      |                    IPVS Sync Connection (1)                   |
+      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+      |                            .                                  |
+      |                            .                                  |
+      |                            .                                  |
+      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+      |                                                               |
+      |                    IPVS Sync Connection (n)                   |
+      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*/
+
+#define SYNC_MESG_V2_HEADER_LEN	sizeof(struct ip_vs_sync_mesg_v2)
+
+struct ip_vs_sync_mesg_v2 {
+	__u8                    nr_conns;
+	__u8                    syncid;
+	__u16                   zero1;
+
+	__u8			version;
+	__u8			zero2;
+	__u16			size;
+
+	/* ip_vs_sync_conn entries start here */
+};
+
+#define MAX_CONNS_PER_SYNCBUFF	255 /* nr_conns in ip_vs_sync_mesg is 8 bit */
+
 /* the maximum length of sync (sending/receiving) message */
 static int sync_send_mesg_maxlen;
 static int sync_recv_mesg_maxlen;

[patch 7/9] IPVS: Allow parsing of ip_vs_sync_mesg_v2 messages

[Simon Horman]

[-- Attachment #1: sync-ip_vs_process_message-v2.patch --]
[-- Type: text/plain, Size: 6675 bytes --]

Signed-off-by: Simon Horman <horms@verge.net.au>

--- 

This is a prototype of the v2 protocol that includes persistence engine
data synchronisation but not other features such as IPv6 service
synchronisation that are planned for inclusion in the full version of v2.

Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-15 08:17:20.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-15 08:28:32.000000000 +0900
@@ -151,6 +151,10 @@ struct ip_vs_sync_mesg_v2 {
 	/* ip_vs_sync_conn entries start here */
 };
 
+#define SYNC_MSG_STR_ALIGN_BYTES 4
+#define SYNC_MSG_STR_ALIGN(_n_) \
+	(((_n_)+(SYNC_MSG_STR_ALIGN_BYTES-1))&~(SYNC_MSG_STR_ALIGN_BYTES-1))
+
 #define MAX_CONNS_PER_SYNCBUFF	255 /* nr_conns in ip_vs_sync_mesg is 8 bit */
 
 /* the maximum length of sync (sending/receiving) message */
@@ -332,10 +336,28 @@ static inline int
 ip_vs_conn_fill_param_sync(int af, int protocol,
 			   const union nf_inet_addr *caddr, __be16 cport,
 			   const union nf_inet_addr *vaddr, __be16 vport,
+			   const char *pe_name, const char *pe_data,
 			   struct ip_vs_conn_param *p)
 {
 	/* XXX: Need to take into account persistence engine */
 	ip_vs_conn_fill_param(af, protocol, caddr, cport, vaddr, vport, p);
+
+	if (pe_data && pe_name) {
+		p->pe_data = kstrdup(pe_data, GFP_KERNEL);
+		if (!p->pe_data) {
+			IP_VS_ERR_RL("can't allocate memory for persistence engine data\n");
+			return -ENOMEM;
+		}
+
+		p->pe = ip_vs_pe_getbyname(pe_name);
+		if (p->pe == NULL) {
+			IP_VS_ERR_RL("persistence engine module ip_vs_pe_%s "
+				     "not found\n", pe_name);
+			kfree(p->pe_data);
+			return -ENOENT;
+		}
+	}
+
 	return 0;
 }
 
@@ -345,13 +367,16 @@ ip_vs_conn_fill_param_sync(int af, int p
  */
 static void ip_vs_process_message(char *buffer, const size_t buflen)
 {
-	struct ip_vs_sync_mesg_v1 *m = (struct ip_vs_sync_mesg_v1 *)buffer;
+	struct ip_vs_sync_mesg_v1 *m_v1 = (struct ip_vs_sync_mesg_v1 *)buffer;
+	struct ip_vs_sync_mesg_v2 *m_v2 = (struct ip_vs_sync_mesg_v2 *)buffer;
 	struct ip_vs_sync_conn *s;
 	struct ip_vs_sync_conn_options *opt;
 	struct ip_vs_conn *cp;
 	struct ip_vs_protocol *pp;
 	struct ip_vs_dest *dest;
 	struct ip_vs_conn_param param;
+	__u8 version = 1;
+	size_t size;
 	char *p;
 	int i;
 
@@ -361,23 +386,36 @@ static void ip_vs_process_message(char *
 	}
 
 	/* Convert size back to host byte order */
-	m->size = ntohs(m->size);
-
-	if (buflen != m->size) {
+	size = m_v1->size = ntohs(m_v1->size);
+	if (m_v1->size == 0) {
+		if (buflen < sizeof(struct ip_vs_sync_mesg_v2)) {
+			IP_VS_ERR_RL("sync message v2 header too short\n");
+			return;
+		}
+		size = m_v2->size = ntohs(m_v2->size);
+		version = m_v2->version;
+		if (version != 2) {
+			IP_VS_DBG(7, "Ignoring incoming msg with version = %d\n",
+				  version);
+			return;
+		}
+	}
+	if (buflen != size) {
 		IP_VS_ERR_RL("bogus sync message size\n");
 		return;
 	}
 
 	/* SyncID sanity check */
-	if (ip_vs_backup_syncid != 0 && m->syncid != ip_vs_backup_syncid) {
+	if (ip_vs_backup_syncid != 0 && m_v1->syncid != ip_vs_backup_syncid) {
 		IP_VS_DBG(7, "Ignoring incoming msg with syncid = %d\n",
-			  m->syncid);
+			  m_v1->syncid);
 		return;
 	}
 
 	p = (char *)buffer + SYNC_MESG_V1_HEADER_LEN;
-	for (i=0; i<m->nr_conns; i++) {
+	for (i = 0; i < m_v1->nr_conns; i++) {
 		unsigned flags, state;
+		char *pe_name = NULL, *pe_data = NULL;
 
 		if (p + SIMPLE_CONN_SIZE > buffer+buflen) {
 			IP_VS_ERR_RL("bogus conn in sync message\n");
@@ -421,13 +459,58 @@ static void ip_vs_process_message(char *
 			}
 		}
 
+		if (flags & IP_VS_CONN_PE && version == 2) {
+			__u16 pe_name_len, pe_data_len;
+
+			flags &= ~IP_VS_CONN_PE;
+
+			if (p + (sizeof(__u16) + SYNC_MSG_STR_ALIGN_BYTES) * 2 >
+			    buffer + buflen) {
+				IP_VS_ERR_RL("bogus persistence engine in sync message\n");
+				return;
+			}
+
+			memcpy(&pe_name_len, p, sizeof(pe_name_len));
+			pe_name_len = ntohs(pe_name_len);
+			p += sizeof(pe_name_len);
+			if (pe_name_len > IP_VS_PENAME_MAXLEN) {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name length > IP_VS_PENAME_MAXLEN (%u > %u)\n",
+					     pe_name_len, IP_VS_PENAME_MAXLEN);
+				return;
+			}
+
+			memcpy(&pe_data_len, p, sizeof(pe_data_len));
+			pe_data_len = ntohs(pe_data_len);
+			p += sizeof(pe_data_len);
+			if (pe_data_len > IP_VS_PENAME_MAXLEN) {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name length > IP_VS_PENAME_MAXLEN (%u > %u)\n",
+					     pe_data_len, IP_VS_PENAME_MAXLEN);
+				return;
+			}
+
+			pe_name = p;
+			if (pe_name[pe_name_len + 1] != '\0') {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name is not nul terminated\n");
+				return;
+			}
+			p += SYNC_MSG_STR_ALIGN(pe_name_len);
+
+			pe_data = p;
+			if (pe_data[pe_data_len + 1] != '\0') {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_data is not nul terminated\n");
+				return;
+			}
+			p += SYNC_MSG_STR_ALIGN(pe_data_len);
+		}
+
 		if (ip_vs_conn_fill_param_sync(AF_INET, s->protocol,
 					       (union nf_inet_addr *)&s->caddr,
 					       s->cport,
 					       (union nf_inet_addr *)&s->vaddr,
-					       s->vport, &param)) {
-			pr_err("ip_vs_conn_fill_param_sync failed");
-			return;
+					       s->vport, pe_name, pe_data,
+					       &param)) {
+			IP_VS_ERR_RL("Unsupported persistence engine %s in sync msg\n", pe_name);
+			continue;
 		}
 		if (!(flags & IP_VS_CONN_F_TEMPLATE))
 			cp = ip_vs_conn_in_get(&param);
@@ -464,6 +547,7 @@ static void ip_vs_process_message(char *
 				atomic_dec(&dest->refcnt);
 			if (!cp) {
 				pr_err("ip_vs_conn_new failed\n");
+				ip_vs_pe_put(param.pe);
 				return;
 			}
 		} else if (!cp->dest) {
@@ -512,6 +596,7 @@ static void ip_vs_process_message(char *
 		else
 			cp->timeout = (3*60*HZ);
 		ip_vs_conn_put(cp);
+		ip_vs_pe_put(param.pe);
 	}
 }
 
Index: lvs-test-2.6/include/linux/ip_vs.h
===================================================================
--- lvs-test-2.6.orig/include/linux/ip_vs.h	2010-10-15 08:17:11.000000000 +0900
+++ lvs-test-2.6/include/linux/ip_vs.h	2010-10-15 08:17:23.000000000 +0900
@@ -88,6 +88,7 @@
 #define IP_VS_CONN_F_NO_CPORT	0x0800		/* no client port set yet */
 #define IP_VS_CONN_F_TEMPLATE	0x1000		/* template, not connection */
 #define IP_VS_CONN_F_ONE_PACKET	0x2000		/* forward only one packet */
+#define IP_VS_CONN_PE		0x4000		/* persistene engine in use */
 
 /* Flags that are not sent to backup server start from bit 16 */
 #define IP_VS_CONN_F_NFCT	(1 << 16)	/* use netfilter conntrack */

[patch 8/9] IPVS: Calculate remaining synchronisation buffer space earlier

[Simon Horman]

[-- Attachment #1: sync-ip_vs_sync_conn-space.patch --]
[-- Type: text/plain, Size: 1944 bytes --]

Up until now there have only been two possible buffer entry sizes.
So it has been easy enough to check there is enough space for the
larger version left after some space is used and flush the buffer
accordingly.

v2 of the synchronisation protocol will add a variety of different
entry lengths, including dynamically sized entries. So move
this calculation before an entry is added to the buffer but after
its size is known.

This may cause a delay in synchronisation as the calculation
and thus flushing of a full buffer is effectively delayed by one entry
more than before.

Signed-off-by: Simon Horman <horms@verge.net.au>

Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 13:27:05.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 13:30:36.000000000 +0900
@@ -286,7 +286,16 @@ void ip_vs_sync_conn(struct ip_vs_conn *
 	struct ip_vs_sync_conn *s;
 	int len;
 
+	len = (cp->flags & IP_VS_CONN_F_SEQ_MASK) ? FULL_CONN_SIZE :
+		SIMPLE_CONN_SIZE;
+
 	spin_lock(&curr_sb_lock);
+
+	/* check if there is a enough space */
+	if (curr_sb && curr_sb->head + len > curr_sb->end) {
+		sb_queue_tail(curr_sb);
+		curr_sb = NULL;
+	}
 	if (!curr_sb) {
 		if (!(curr_sb=ip_vs_sync_buff_create())) {
 			spin_unlock(&curr_sb_lock);
@@ -295,8 +304,6 @@ void ip_vs_sync_conn(struct ip_vs_conn *
 		}
 	}
 
-	len = (cp->flags & IP_VS_CONN_F_SEQ_MASK) ? FULL_CONN_SIZE :
-		SIMPLE_CONN_SIZE;
 	m = curr_sb->mesg;
 	s = (struct ip_vs_sync_conn *)curr_sb->head;
 
@@ -320,11 +327,6 @@ void ip_vs_sync_conn(struct ip_vs_conn *
 	m->size += len;
 	curr_sb->head += len;
 
-	/* check if there is a space for next one */
-	if (curr_sb->head+FULL_CONN_SIZE > curr_sb->end) {
-		sb_queue_tail(curr_sb);
-		curr_sb = NULL;
-	}
 	spin_unlock(&curr_sb_lock);
 
 	/* synchronize its controller if it has */

[patch 9/9] IPVS: Send v2 synchronisation packets

[Simon Horman]

[-- Attachment #1: sync-ip_vs_sync_conn-v2.patch --]
[-- Type: text/plain, Size: 8018 bytes --]

Signed-off-by: Simon Horman <horms@verge.net.au>

--- 

This is a prototype of the v2 protocol that includes persistence engine
data synchronisation but not other features such as IPv6 service
synchronisation that are planned for inclusion in the full version of v2.

Index: lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c
===================================================================
--- lvs-test-2.6.orig/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 23:35:26.000000000 +0900
+++ lvs-test-2.6/net/netfilter/ipvs/ip_vs_sync.c	2010-10-10 23:36:48.000000000 +0900
@@ -166,7 +166,7 @@ struct ip_vs_sync_buff {
 	unsigned long           firstuse;
 
 	/* pointers for the message data */
-	struct ip_vs_sync_mesg_v1 *mesg;
+	struct ip_vs_sync_mesg_v2 *mesg;
 	unsigned char           *head;
 	unsigned char           *end;
 };
@@ -230,12 +230,16 @@ static inline struct ip_vs_sync_buff * i
 		kfree(sb);
 		return NULL;
 	}
+
+	memset(sb->mesg, 0, SYNC_MESG_V2_HEADER_LEN);
 	sb->mesg->nr_conns = 0;
 	sb->mesg->syncid = ip_vs_master_syncid;
-	sb->mesg->size = SYNC_MESG_V1_HEADER_LEN;
-	sb->head = (unsigned char *)sb->mesg + SYNC_MESG_V1_HEADER_LEN;
+	sb->mesg->size = SYNC_MESG_V2_HEADER_LEN;
+	sb->mesg->version = 2;
+	sb->head = (unsigned char *)sb->mesg + SYNC_MESG_V2_HEADER_LEN;
 	sb->end = (unsigned char *)sb->mesg + sync_send_mesg_maxlen;
 	sb->firstuse = jiffies;
+
 	return sb;
 }
 
@@ -282,17 +286,35 @@ get_curr_sync_buff(unsigned long time)
  */
 void ip_vs_sync_conn(const struct ip_vs_conn *cp)
 {
-	struct ip_vs_sync_mesg_v1 *m;
+	struct ip_vs_sync_mesg_v2 *m;
 	struct ip_vs_sync_conn *s;
-	int len;
+	__u16 flags = cp->flags;
+	int len, pe_len = 0;
+	size_t pe_name_len = 0;
 
 	len = (cp->flags & IP_VS_CONN_F_SEQ_MASK) ? FULL_CONN_SIZE :
 		SIMPLE_CONN_SIZE;
 
+	if (cp->pe_data_len) {
+		if (!cp->dest) {
+			IP_VS_ERR_RL("connection has pe_data_len but no dest, can't sychronise\n");
+			return;
+		}
+		if (!cp->pe_data) {
+			IP_VS_ERR_RL("connection has pe_data_len but no pe_data, can't sychronise\n");
+			return;
+		}
+		flags |= IP_VS_CONN_PE;
+		pe_name_len = strlen(cp->dest->svc->pe->name) + 1;
+		pe_len = sizeof(__u16) * 2 +
+			SYNC_MSG_STR_ALIGN(pe_name_len) +
+			SYNC_MSG_STR_ALIGN(cp->pe_data_len);
+	}
+
 	spin_lock(&curr_sb_lock);
 
 	/* check if there is a enough space */
-	if (curr_sb && curr_sb->head + len > curr_sb->end) {
+	if (curr_sb && curr_sb->head + len + pe_len) {
 		sb_queue_tail(curr_sb);
 		curr_sb = NULL;
 	}
@@ -315,7 +337,7 @@ void ip_vs_sync_conn(const struct ip_vs_
 	s->caddr = cp->caddr.ip;
 	s->vaddr = cp->vaddr.ip;
 	s->daddr = cp->daddr.ip;
-	s->flags = htons(cp->flags & ~IP_VS_CONN_F_HASHED);
+	s->flags = htons(flags & ~IP_VS_CONN_F_HASHED);
 	s->state = htons(cp->state);
 	if (cp->flags & IP_VS_CONN_F_SEQ_MASK) {
 		struct ip_vs_sync_conn_options *opt =
@@ -327,6 +349,32 @@ void ip_vs_sync_conn(const struct ip_vs_
 	m->size += len;
 	curr_sb->head += len;
 
+	if (cp->pe_data_len) {
+		__u16 l;
+
+		l = pe_name_len;
+		l = htons(l);
+		memcpy(curr_sb->head, &l, sizeof(l));
+		m->size += sizeof(l);
+		curr_sb->head += sizeof(l);
+
+		l = cp->pe_data_len;
+		l = htons(l);
+		memcpy(curr_sb->head, &l, sizeof(l));
+		m->size += sizeof(l);
+		curr_sb->head += sizeof(l);
+
+		memset(curr_sb->head, 0, SYNC_MSG_STR_ALIGN(pe_name_len));
+		memcpy(curr_sb->head, cp->dest->svc->pe->name, pe_name_len);
+		m->size += SYNC_MSG_STR_ALIGN(pe_name_len);
+		curr_sb->head += SYNC_MSG_STR_ALIGN(pe_name_len);
+
+		memset(curr_sb->head, 0, SYNC_MSG_STR_ALIGN(cp->pe_data_len));
+		memcpy(curr_sb->head, cp->pe_data, cp->pe_data_len);
+		m->size += SYNC_MSG_STR_ALIGN(cp->pe_data_len);
+		curr_sb->head += SYNC_MSG_STR_ALIGN(cp->pe_data_len);
+	}
+
 	spin_unlock(&curr_sb_lock);
 
 	/* synchronize its controller if it has */
@@ -339,7 +387,7 @@ ip_vs_conn_fill_param_sync(int af, int p
 			   const union nf_inet_addr *caddr, __be16 cport,
 			   const union nf_inet_addr *vaddr, __be16 vport,
 			   const char *pe_name, const char *pe_data,
-			   struct ip_vs_conn_param *p)
+			   __u8 pe_data_len, struct ip_vs_conn_param *p)
 {
 	/* XXX: Need to take into account persistence engine */
 	ip_vs_conn_fill_param(af, protocol, caddr, cport, vaddr, vport, p);
@@ -350,6 +398,7 @@ ip_vs_conn_fill_param_sync(int af, int p
 			IP_VS_ERR_RL("can't allocate memory for persistence engine data\n");
 			return -ENOMEM;
 		}
+		p->pe_data_len = pe_data_len;
 
 		p->pe = ip_vs_pe_get(pe_name);
 		if (p->pe == NULL) {
@@ -377,6 +426,7 @@ static void ip_vs_process_message(char *
 	struct ip_vs_protocol *pp;
 	struct ip_vs_dest *dest;
 	struct ip_vs_conn_param param;
+	__u16 pe_name_len, pe_data_len;
 	__u8 version = 1;
 	size_t size;
 	char *p;
@@ -414,7 +464,11 @@ static void ip_vs_process_message(char *
 		return;
 	}
 
-	p = (char *)buffer + SYNC_MESG_V1_HEADER_LEN;
+	p = (char *)buffer;
+	if (version == 1)
+		p += SYNC_MESG_V1_HEADER_LEN;
+	else
+		p += SYNC_MESG_V2_HEADER_LEN;
 	for (i = 0; i < m_v1->nr_conns; i++) {
 		unsigned flags, state;
 		char *pe_name = NULL, *pe_data = NULL;
@@ -462,46 +516,50 @@ static void ip_vs_process_message(char *
 		}
 
 		if (flags & IP_VS_CONN_PE && version == 2) {
-			__u16 pe_name_len, pe_data_len;
-
 			flags &= ~IP_VS_CONN_PE;
 
 			if (p + (sizeof(__u16) + SYNC_MSG_STR_ALIGN_BYTES) * 2 >
 			    buffer + buflen) {
-				IP_VS_ERR_RL("bogus persistence engine in sync message\n");
+				IP_VS_ERR_RL("bogus persistence engine in sync message: data is less than the minimum length\n");
 				return;
 			}
 
 			memcpy(&pe_name_len, p, sizeof(pe_name_len));
 			pe_name_len = ntohs(pe_name_len);
 			p += sizeof(pe_name_len);
-			if (pe_name_len > IP_VS_PENAME_MAXLEN) {
-				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name length > IP_VS_PENAME_MAXLEN (%u > %u)\n",
-					     pe_name_len, IP_VS_PENAME_MAXLEN);
+			if (pe_name_len > IP_VS_PENAME_MAXLEN + 1) {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name length > IP_VS_PENAME_MAXLEN + 1 (%u > %u)\n",
+					     pe_name_len,
+					     IP_VS_PENAME_MAXLEN + 1);
+				return;
+			}
+			if (pe_name_len < 1) {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name length < 1 (%u < 1)\n", pe_name_len);
 				return;
 			}
 
 			memcpy(&pe_data_len, p, sizeof(pe_data_len));
 			pe_data_len = ntohs(pe_data_len);
 			p += sizeof(pe_data_len);
-			if (pe_data_len > IP_VS_PENAME_MAXLEN) {
-				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name length > IP_VS_PENAME_MAXLEN (%u > %u)\n",
-					     pe_data_len, IP_VS_PENAME_MAXLEN);
+			if (pe_data_len > IP_VS_PEDATA_MAXLEN) {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_data length > IP_VS_PEDATA_MAXLEN (%u > %u)\n",
+					     pe_data_len, IP_VS_PEDATA_MAXLEN);
+				return;
+			}
+
+			if (p + pe_name_len + pe_data_len > buffer + buflen) {
+				IP_VS_ERR_RL("bogus persistence engine in sync message: length missmatch\n");
 				return;
 			}
 
 			pe_name = p;
-			if (pe_name[pe_name_len + 1] != '\0') {
+			if (pe_name[pe_name_len - 1] != '\0') {
 				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_name is not nul terminated\n");
 				return;
 			}
 			p += SYNC_MSG_STR_ALIGN(pe_name_len);
 
 			pe_data = p;
-			if (pe_data[pe_data_len + 1] != '\0') {
-				IP_VS_ERR_RL("bogus persistence engine in sync message: pe_data is not nul terminated\n");
-				return;
-			}
 			p += SYNC_MSG_STR_ALIGN(pe_data_len);
 		}
 
@@ -510,7 +568,7 @@ static void ip_vs_process_message(char *
 					       s->cport,
 					       (union nf_inet_addr *)&s->vaddr,
 					       s->vport, pe_name, pe_data,
-					       &param)) {
+					       pe_data_len, &param)) {
 			IP_VS_ERR_RL("Unsupported persistence engine %s in sync msg\n", pe_name);
 			continue;
 		}
@@ -846,7 +904,7 @@ ip_vs_send_async(struct socket *sock, co
 }
 
 static void
-ip_vs_send_sync_msg(struct socket *sock, struct ip_vs_sync_mesg_v1 *msg)
+ip_vs_send_sync_msg(struct socket *sock, struct ip_vs_sync_mesg_v2 *msg)
 {
 	int msize;
 

Re: [patch 0/9] [rfc] synchronisation of persistence engines

[Simon Horman]

On Sun, Oct 24, 2010 at 12:26:26AM +0900, Simon Horman wrote:
> Hi,
> 
> This is an extension of the synchronisation protocol
> to support the synchronisation of persistence engine data.
> 
> Although it is called v2, it is only WIP and does not meet
> many of the stated goals of v2 - notably no IPv6, fwmark
> or timeout synchronisation.
> 
> It is not intended to compete with other work in this area,
> rather to add to the discussion.

The series is available in the pe-sync-0.1 branch of
git://git.kernel.org/pub/scm/linux/kernel/git/horms/lvs-test-2.6.git