[PATCH v3 0/2] tools/xenstore: simplify xenstored main loop

[PATCH v3 0/2] tools/xenstore: simplify xenstored main loop

[Juergen Gross]

Small series to make the main loop of xenstored more readable.

Changes in V2:
- split into two patches
- use const
- NO_SOCKETS handling

Changes in V3:
- guard extern declaration with NO_SOCKETS

Juergen Gross (2):
  tools/xenstore: move per connection read and write func hooks into a
    struct
  tools/xenstore: simplify xenstored main loop

 tools/xenstore/xenstored_core.c   | 121 ++++++++++++++----------------
 tools/xenstore/xenstored_core.h   |  23 +++---
 tools/xenstore/xenstored_domain.c |  15 +++-
 3 files changed, 81 insertions(+), 78 deletions(-)

-- 
2.26.2

[PATCH v3 1/2] tools/xenstore: move per connection read and write func hooks into a struct

[Juergen Gross]

Put the interface type specific functions into an own structure and let
struct connection contain only a pointer to that new function vector.

Don't even define the socket based functions in case of NO_SOCKETS
(Mini-OS).

Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Julien Grall <jgrall@amazon.com>
---
V2:
- split off from V1 patch (Julien Grall)
- use const qualifier (Julien Grall)
- drop socket specific case for Mini-OS (Julien Grall)
V3:
- guard extern declaration with #ifndef NO_SOCKETS (Julien Grall)
---
 tools/xenstore/xenstored_core.c   | 44 +++++++++++++------------------
 tools/xenstore/xenstored_core.h   | 21 ++++++++-------
 tools/xenstore/xenstored_domain.c | 13 +++++++--
 3 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/tools/xenstore/xenstored_core.c b/tools/xenstore/xenstored_core.c
index 4b7b71cfb3..856f518075 100644
--- a/tools/xenstore/xenstored_core.c
+++ b/tools/xenstore/xenstored_core.c
@@ -226,8 +226,8 @@ static bool write_messages(struct connection *conn)
 				sockmsg_string(out->hdr.msg.type),
 				out->hdr.msg.len,
 				out->buffer, conn);
-		ret = conn->write(conn, out->hdr.raw + out->used,
-				  sizeof(out->hdr) - out->used);
+		ret = conn->funcs->write(conn, out->hdr.raw + out->used,
+					 sizeof(out->hdr) - out->used);
 		if (ret < 0)
 			return false;
 
@@ -243,8 +243,8 @@ static bool write_messages(struct connection *conn)
 			return true;
 	}
 
-	ret = conn->write(conn, out->buffer + out->used,
-			  out->hdr.msg.len - out->used);
+	ret = conn->funcs->write(conn, out->buffer + out->used,
+				 out->hdr.msg.len - out->used);
 	if (ret < 0)
 		return false;
 
@@ -1531,8 +1531,8 @@ static void handle_input(struct connection *conn)
 	/* Not finished header yet? */
 	if (in->inhdr) {
 		if (in->used != sizeof(in->hdr)) {
-			bytes = conn->read(conn, in->hdr.raw + in->used,
-					   sizeof(in->hdr) - in->used);
+			bytes = conn->funcs->read(conn, in->hdr.raw + in->used,
+						  sizeof(in->hdr) - in->used);
 			if (bytes < 0)
 				goto bad_client;
 			in->used += bytes;
@@ -1557,8 +1557,8 @@ static void handle_input(struct connection *conn)
 		in->inhdr = false;
 	}
 
-	bytes = conn->read(conn, in->buffer + in->used,
-			   in->hdr.msg.len - in->used);
+	bytes = conn->funcs->read(conn, in->buffer + in->used,
+				  in->hdr.msg.len - in->used);
 	if (bytes < 0)
 		goto bad_client;
 
@@ -1581,7 +1581,7 @@ static void handle_output(struct connection *conn)
 		ignore_connection(conn);
 }
 
-struct connection *new_connection(connwritefn_t *write, connreadfn_t *read)
+struct connection *new_connection(const struct interface_funcs *funcs)
 {
 	struct connection *new;
 
@@ -1591,8 +1591,7 @@ struct connection *new_connection(connwritefn_t *write, connreadfn_t *read)
 
 	new->fd = -1;
 	new->pollfd_idx = -1;
-	new->write = write;
-	new->read = read;
+	new->funcs = funcs;
 	new->is_ignored = false;
 	new->transaction_started = 0;
 	INIT_LIST_HEAD(&new->out_list);
@@ -1621,20 +1620,8 @@ struct connection *get_connection_by_id(unsigned int conn_id)
 static void accept_connection(int sock)
 {
 }
-
-int writefd(struct connection *conn, const void *data, unsigned int len)
-{
-	errno = EBADF;
-	return -1;
-}
-
-int readfd(struct connection *conn, void *data, unsigned int len)
-{
-	errno = EBADF;
-	return -1;
-}
 #else
-int writefd(struct connection *conn, const void *data, unsigned int len)
+static int writefd(struct connection *conn, const void *data, unsigned int len)
 {
 	int rc;
 
@@ -1650,7 +1637,7 @@ int writefd(struct connection *conn, const void *data, unsigned int len)
 	return rc;
 }
 
-int readfd(struct connection *conn, void *data, unsigned int len)
+static int readfd(struct connection *conn, void *data, unsigned int len)
 {
 	int rc;
 
@@ -1672,6 +1659,11 @@ int readfd(struct connection *conn, void *data, unsigned int len)
 	return rc;
 }
 
+const struct interface_funcs socket_funcs = {
+	.write = writefd,
+	.read = readfd,
+};
+
 static void accept_connection(int sock)
 {
 	int fd;
@@ -1681,7 +1673,7 @@ static void accept_connection(int sock)
 	if (fd < 0)
 		return;
 
-	conn = new_connection(writefd, readfd);
+	conn = new_connection(&socket_funcs);
 	if (conn)
 		conn->fd = fd;
 	else
diff --git a/tools/xenstore/xenstored_core.h b/tools/xenstore/xenstored_core.h
index 6a6d0448e8..6c4845c196 100644
--- a/tools/xenstore/xenstored_core.h
+++ b/tools/xenstore/xenstored_core.h
@@ -86,8 +86,11 @@ struct delayed_request {
 };
 
 struct connection;
-typedef int connwritefn_t(struct connection *, const void *, unsigned int);
-typedef int connreadfn_t(struct connection *, void *, unsigned int);
+
+struct interface_funcs {
+	int (*write)(struct connection *, const void *, unsigned int);
+	int (*read)(struct connection *, void *, unsigned int);
+};
 
 struct connection
 {
@@ -131,9 +134,8 @@ struct connection
 	/* My watches. */
 	struct list_head watches;
 
-	/* Methods for communicating over this connection: write can be NULL */
-	connwritefn_t *write;
-	connreadfn_t *read;
+	/* Methods for communicating over this connection. */
+	const struct interface_funcs *funcs;
 
 	/* Support for live update: connection id. */
 	unsigned int conn_id;
@@ -196,7 +198,7 @@ int write_node_raw(struct connection *conn, TDB_DATA *key, struct node *node,
 struct node *read_node(struct connection *conn, const void *ctx,
 		       const char *name);
 
-struct connection *new_connection(connwritefn_t *write, connreadfn_t *read);
+struct connection *new_connection(const struct interface_funcs *funcs);
 struct connection *get_connection_by_id(unsigned int conn_id);
 void check_store(void);
 void corrupt(struct connection *conn, const char *fmt, ...);
@@ -254,10 +256,9 @@ void finish_daemonize(void);
 /* Open a pipe for signal handling */
 void init_pipe(int reopen_log_pipe[2]);
 
-int writefd(struct connection *conn, const void *data, unsigned int len);
-int readfd(struct connection *conn, void *data, unsigned int len);
-
-extern struct interface_funcs socket_funcs;
+#ifndef NO_SOCKETS
+extern const struct interface_funcs socket_funcs;
+#endif
 extern xengnttab_handle **xgt_handle;
 
 int remember_string(struct hashtable *hash, const char *str);
diff --git a/tools/xenstore/xenstored_domain.c b/tools/xenstore/xenstored_domain.c
index 0c17937c0f..f3cd56050e 100644
--- a/tools/xenstore/xenstored_domain.c
+++ b/tools/xenstore/xenstored_domain.c
@@ -172,6 +172,11 @@ static int readchn(struct connection *conn, void *data, unsigned int len)
 	return len;
 }
 
+static const struct interface_funcs domain_funcs = {
+	.write = writechn,
+	.read = readchn,
+};
+
 static void *map_interface(domid_t domid)
 {
 	return xengnttab_map_grant_ref(*xgt_handle, domid,
@@ -389,7 +394,7 @@ static int new_domain(struct domain *domain, int port, bool restore)
 
 	domain->introduced = true;
 
-	domain->conn = new_connection(writechn, readchn);
+	domain->conn = new_connection(&domain_funcs);
 	if (!domain->conn)  {
 		errno = ENOMEM;
 		return errno;
@@ -1288,10 +1293,14 @@ void read_state_connection(const void *ctx, const void *state)
 	struct domain *domain, *tdomain;
 
 	if (sc->conn_type == XS_STATE_CONN_TYPE_SOCKET) {
-		conn = new_connection(writefd, readfd);
+#ifdef NO_SOCKETS
+		barf("socket based connection without sockets");
+#else
+		conn = new_connection(&socket_funcs);
 		if (!conn)
 			barf("error restoring connection");
 		conn->fd = sc->spec.socket_fd;
+#endif
 	} else {
 		domain = introduce_domain(ctx, sc->spec.ring.domid,
 					  sc->spec.ring.evtchn, true);
-- 
2.26.2

[PATCH v3 2/2] tools/xenstore: simplify xenstored main loop

[Juergen Gross]

The main loop of xenstored is rather complicated due to different
handling of socket and ring-page interfaces. Unify that handling by
introducing interface type specific functions can_read() and
can_write().

Take the opportunity to remove the empty list check before calling
write_messages() because the function is already able to cope with an
empty list.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
V2:
- split off function vector introduction (Julien Grall)
V3:
- expand commit message (Julien Grall)
---
 tools/xenstore/xenstored_core.c   | 77 +++++++++++++++----------------
 tools/xenstore/xenstored_core.h   |  2 +
 tools/xenstore/xenstored_domain.c |  2 +
 3 files changed, 41 insertions(+), 40 deletions(-)

diff --git a/tools/xenstore/xenstored_core.c b/tools/xenstore/xenstored_core.c
index 856f518075..883a1a582a 100644
--- a/tools/xenstore/xenstored_core.c
+++ b/tools/xenstore/xenstored_core.c
@@ -1659,9 +1659,34 @@ static int readfd(struct connection *conn, void *data, unsigned int len)
 	return rc;
 }
 
+static bool socket_can_process(struct connection *conn, int mask)
+{
+	if (conn->pollfd_idx == -1)
+		return false;
+
+	if (fds[conn->pollfd_idx].revents & ~(POLLIN | POLLOUT)) {
+		talloc_free(conn);
+		return false;
+	}
+
+	return (fds[conn->pollfd_idx].revents & mask) && !conn->is_ignored;
+}
+
+static bool socket_can_write(struct connection *conn)
+{
+	return socket_can_process(conn, POLLOUT);
+}
+
+static bool socket_can_read(struct connection *conn)
+{
+	return socket_can_process(conn, POLLIN);
+}
+
 const struct interface_funcs socket_funcs = {
 	.write = writefd,
 	.read = readfd,
+	.can_write = socket_can_write,
+	.can_read = socket_can_read,
 };
 
 static void accept_connection(int sock)
@@ -2296,47 +2321,19 @@ int main(int argc, char *argv[])
 			if (&next->list != &connections)
 				talloc_increase_ref_count(next);
 
-			if (conn->domain) {
-				if (domain_can_read(conn))
-					handle_input(conn);
-				if (talloc_free(conn) == 0)
-					continue;
-
-				talloc_increase_ref_count(conn);
-				if (domain_can_write(conn) &&
-				    !list_empty(&conn->out_list))
-					handle_output(conn);
-				if (talloc_free(conn) == 0)
-					continue;
-			} else {
-				if (conn->pollfd_idx != -1) {
-					if (fds[conn->pollfd_idx].revents
-					    & ~(POLLIN|POLLOUT))
-						talloc_free(conn);
-					else if ((fds[conn->pollfd_idx].revents
-						  & POLLIN) &&
-						 !conn->is_ignored)
-						handle_input(conn);
-				}
-				if (talloc_free(conn) == 0)
-					continue;
-
-				talloc_increase_ref_count(conn);
-
-				if (conn->pollfd_idx != -1) {
-					if (fds[conn->pollfd_idx].revents
-					    & ~(POLLIN|POLLOUT))
-						talloc_free(conn);
-					else if ((fds[conn->pollfd_idx].revents
-						  & POLLOUT) &&
-						 !conn->is_ignored)
-						handle_output(conn);
-				}
-				if (talloc_free(conn) == 0)
-					continue;
+			if (conn->funcs->can_read(conn))
+				handle_input(conn);
+			if (talloc_free(conn) == 0)
+				continue;
 
-				conn->pollfd_idx = -1;
-			}
+			talloc_increase_ref_count(conn);
+
+			if (conn->funcs->can_write(conn))
+				handle_output(conn);
+			if (talloc_free(conn) == 0)
+				continue;
+
+			conn->pollfd_idx = -1;
 		}
 
 		if (delayed_requests) {
diff --git a/tools/xenstore/xenstored_core.h b/tools/xenstore/xenstored_core.h
index 6c4845c196..bb36111ecc 100644
--- a/tools/xenstore/xenstored_core.h
+++ b/tools/xenstore/xenstored_core.h
@@ -90,6 +90,8 @@ struct connection;
 struct interface_funcs {
 	int (*write)(struct connection *, const void *, unsigned int);
 	int (*read)(struct connection *, void *, unsigned int);
+	bool (*can_write)(struct connection *);
+	bool (*can_read)(struct connection *);
 };
 
 struct connection
diff --git a/tools/xenstore/xenstored_domain.c b/tools/xenstore/xenstored_domain.c
index f3cd56050e..708bf68af0 100644
--- a/tools/xenstore/xenstored_domain.c
+++ b/tools/xenstore/xenstored_domain.c
@@ -175,6 +175,8 @@ static int readchn(struct connection *conn, void *data, unsigned int len)
 static const struct interface_funcs domain_funcs = {
 	.write = writechn,
 	.read = readchn,
+	.can_write = domain_can_write,
+	.can_read = domain_can_read,
 };
 
 static void *map_interface(domid_t domid)
-- 
2.26.2

Re: [PATCH v3 2/2] tools/xenstore: simplify xenstored main loop

[Julien Grall]

Hi Juergen,

On 18/05/2021 07:19, Juergen Gross wrote:
> The main loop of xenstored is rather complicated due to different
> handling of socket and ring-page interfaces. Unify that handling by
> introducing interface type specific functions can_read() and
> can_write().
> 
> Take the opportunity to remove the empty list check before calling
> write_messages() because the function is already able to cope with an
> empty list.
> 
> Signed-off-by: Juergen Gross <jgross@suse.com>

Reviewed-by: Julien Grall <jgrall@amazon.com>

I have also committed the series.

Cheers,

> ---
> V2:
> - split off function vector introduction (Julien Grall)
> V3:
> - expand commit message (Julien Grall)
> ---
>   tools/xenstore/xenstored_core.c   | 77 +++++++++++++++----------------
>   tools/xenstore/xenstored_core.h   |  2 +
>   tools/xenstore/xenstored_domain.c |  2 +
>   3 files changed, 41 insertions(+), 40 deletions(-)
> 
> diff --git a/tools/xenstore/xenstored_core.c b/tools/xenstore/xenstored_core.c
> index 856f518075..883a1a582a 100644
> --- a/tools/xenstore/xenstored_core.c
> +++ b/tools/xenstore/xenstored_core.c
> @@ -1659,9 +1659,34 @@ static int readfd(struct connection *conn, void *data, unsigned int len)
>   	return rc;
>   }
>   
> +static bool socket_can_process(struct connection *conn, int mask)
> +{
> +	if (conn->pollfd_idx == -1)
> +		return false;
> +
> +	if (fds[conn->pollfd_idx].revents & ~(POLLIN | POLLOUT)) {
> +		talloc_free(conn);
> +		return false;
> +	}
> +
> +	return (fds[conn->pollfd_idx].revents & mask) && !conn->is_ignored;
> +}
> +
> +static bool socket_can_write(struct connection *conn)
> +{
> +	return socket_can_process(conn, POLLOUT);
> +}
> +
> +static bool socket_can_read(struct connection *conn)
> +{
> +	return socket_can_process(conn, POLLIN);
> +}
> +
>   const struct interface_funcs socket_funcs = {
>   	.write = writefd,
>   	.read = readfd,
> +	.can_write = socket_can_write,
> +	.can_read = socket_can_read,
>   };
>   
>   static void accept_connection(int sock)
> @@ -2296,47 +2321,19 @@ int main(int argc, char *argv[])
>   			if (&next->list != &connections)
>   				talloc_increase_ref_count(next);
>   
> -			if (conn->domain) {
> -				if (domain_can_read(conn))
> -					handle_input(conn);
> -				if (talloc_free(conn) == 0)
> -					continue;
> -
> -				talloc_increase_ref_count(conn);
> -				if (domain_can_write(conn) &&
> -				    !list_empty(&conn->out_list))
> -					handle_output(conn);
> -				if (talloc_free(conn) == 0)
> -					continue;
> -			} else {
> -				if (conn->pollfd_idx != -1) {
> -					if (fds[conn->pollfd_idx].revents
> -					    & ~(POLLIN|POLLOUT))
> -						talloc_free(conn);
> -					else if ((fds[conn->pollfd_idx].revents
> -						  & POLLIN) &&
> -						 !conn->is_ignored)
> -						handle_input(conn);
> -				}
> -				if (talloc_free(conn) == 0)
> -					continue;
> -
> -				talloc_increase_ref_count(conn);
> -
> -				if (conn->pollfd_idx != -1) {
> -					if (fds[conn->pollfd_idx].revents
> -					    & ~(POLLIN|POLLOUT))
> -						talloc_free(conn);
> -					else if ((fds[conn->pollfd_idx].revents
> -						  & POLLOUT) &&
> -						 !conn->is_ignored)
> -						handle_output(conn);
> -				}
> -				if (talloc_free(conn) == 0)
> -					continue;
> +			if (conn->funcs->can_read(conn))
> +				handle_input(conn);
> +			if (talloc_free(conn) == 0)
> +				continue;
>   
> -				conn->pollfd_idx = -1;
> -			}
> +			talloc_increase_ref_count(conn);
> +
> +			if (conn->funcs->can_write(conn))
> +				handle_output(conn);
> +			if (talloc_free(conn) == 0)
> +				continue;
> +
> +			conn->pollfd_idx = -1;
>   		}
>   
>   		if (delayed_requests) {
> diff --git a/tools/xenstore/xenstored_core.h b/tools/xenstore/xenstored_core.h
> index 6c4845c196..bb36111ecc 100644
> --- a/tools/xenstore/xenstored_core.h
> +++ b/tools/xenstore/xenstored_core.h
> @@ -90,6 +90,8 @@ struct connection;
>   struct interface_funcs {
>   	int (*write)(struct connection *, const void *, unsigned int);
>   	int (*read)(struct connection *, void *, unsigned int);
> +	bool (*can_write)(struct connection *);
> +	bool (*can_read)(struct connection *);
>   };
>   
>   struct connection
> diff --git a/tools/xenstore/xenstored_domain.c b/tools/xenstore/xenstored_domain.c
> index f3cd56050e..708bf68af0 100644
> --- a/tools/xenstore/xenstored_domain.c
> +++ b/tools/xenstore/xenstored_domain.c
> @@ -175,6 +175,8 @@ static int readchn(struct connection *conn, void *data, unsigned int len)
>   static const struct interface_funcs domain_funcs = {
>   	.write = writechn,
>   	.read = readchn,
> +	.can_write = domain_can_write,
> +	.can_read = domain_can_read,
>   };
>   
>   static void *map_interface(domid_t domid)
> 

-- 
Julien Grall