[PATCH V5 08/17] pipeline: add API for shared library-based pipeline build

[Cristian Dumitrescu <cristian.dumitrescu@intel.com>]

Previously, the pipeline build operation was done based on the
specification file (typically produced by the P4 compiler), then the C
code with optimized functions for the pipeline actions and
instructions was generated, built into a shared object library, loaded
and installed into the pipeline in a completely hardcoded and
non-customizable way.

Now, this process is split into three explicit stages:
i) code generation (specification file -> C file);
ii) code build (C file -> shared object library);
iii) code installation (library load into the pipeline).

Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com>
Signed-off-by: Kamalakannan R. <kamalakannan.r@intel.com>
---
 examples/pipeline/cli.c              |  85 ++++---
 lib/pipeline/rte_swx_pipeline.c      | 319 +++++++++++----------------
 lib/pipeline/rte_swx_pipeline.h      |  38 ++--
 lib/pipeline/rte_swx_pipeline_spec.c |  51 -----
 lib/pipeline/version.map             |   2 +-
 5 files changed, 208 insertions(+), 287 deletions(-)

diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c
index ad553f19ab..f0285675b3 100644
--- a/examples/pipeline/cli.c
+++ b/examples/pipeline/cli.c
@@ -984,55 +984,88 @@ cmd_pipeline_port_out(char **tokens,
 }
 
 static const char cmd_pipeline_build_help[] =
-"pipeline <pipeline_name> build <spec_file>\n";
+"pipeline <pipeline_name> build lib <lib_file> io <iospec_file> numa <numa_node>\n";
 
 static void
 cmd_pipeline_build(char **tokens,
 	uint32_t n_tokens,
 	char *out,
 	size_t out_size,
-	void *obj)
+	void *obj __rte_unused)
 {
-	struct pipeline *p = NULL;
-	FILE *spec = NULL;
-	uint32_t err_line;
-	const char *err_msg;
-	int status;
+	struct rte_swx_pipeline *p = NULL;
+	struct rte_swx_ctl_pipeline *ctl = NULL;
+	char *pipeline_name, *lib_file_name, *iospec_file_name;
+	FILE *iospec_file = NULL;
+	uint32_t numa_node = 0;
+	int status = 0;
 
-	if (n_tokens != 4) {
+	/* Parsing. */
+	if (n_tokens != 9) {
 		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
 		return;
 	}
 
-	p = pipeline_find(obj, tokens[1]);
-	if (!p || p->ctl) {
-		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "build")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "build");
 		return;
 	}
 
-	spec = fopen(tokens[3], "r");
-	if (!spec) {
-		snprintf(out, out_size, "Cannot open file %s.\n", tokens[3]);
+	if (strcmp(tokens[3], "lib")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "lib");
 		return;
 	}
 
-	status = rte_swx_pipeline_build_from_spec(p->p,
-		spec,
-		&err_line,
-		&err_msg);
-	fclose(spec);
-	if (status) {
-		snprintf(out, out_size, "Error %d at line %u: %s\n.",
-			status, err_line, err_msg);
+	lib_file_name = tokens[4];
+
+	if (strcmp(tokens[5], "io")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "io");
 		return;
 	}
 
-	p->ctl = rte_swx_ctl_pipeline_create(p->p);
-	if (!p->ctl) {
-		snprintf(out, out_size, "Pipeline control create failed.");
-		rte_swx_pipeline_free(p->p);
+	iospec_file_name = tokens[6];
+
+	if (strcmp(tokens[7], "numa")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "numa");
 		return;
 	}
+
+	if (parser_read_uint32(&numa_node, tokens[8])) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "numa_node");
+		return;
+	}
+
+	/* I/O spec file open. */
+	iospec_file = fopen(iospec_file_name, "r");
+	if (!iospec_file) {
+		snprintf(out, out_size, "Cannot open file \"%s\".\n", iospec_file_name);
+		return;
+	}
+
+	status = rte_swx_pipeline_build_from_lib(&p,
+						 pipeline_name,
+						 lib_file_name,
+						 iospec_file,
+						 (int)numa_node);
+	if (status) {
+		snprintf(out, out_size, "Pipeline build failed (%d).", status);
+		goto free;
+	}
+
+	ctl = rte_swx_ctl_pipeline_create(p);
+	if (!ctl) {
+		snprintf(out, out_size, "Pipeline control create failed.");
+		goto free;
+	}
+
+free:
+	if (status)
+		rte_swx_pipeline_free(p);
+
+	if (iospec_file)
+		fclose(iospec_file);
 }
 
 static void
diff --git a/lib/pipeline/rte_swx_pipeline.c b/lib/pipeline/rte_swx_pipeline.c
index dd5f7107fa..12e156b00b 100644
--- a/lib/pipeline/rte_swx_pipeline.c
+++ b/lib/pipeline/rte_swx_pipeline.c
@@ -9911,9 +9911,6 @@ rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
 	return 0;
 }
 
-static int
-pipeline_compile(struct rte_swx_pipeline *p);
-
 int
 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
 {
@@ -10003,8 +10000,6 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p)
 
 	p->build_done = 1;
 
-	pipeline_compile(p);
-
 	return 0;
 
 error:
@@ -13327,160 +13322,6 @@ instruction_group_list_custom_instructions_count(struct instruction_group_list *
 	return n_custom_instr;
 }
 
-static int
-pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
-{
-	struct action *a;
-	FILE *f = NULL;
-
-	/* Create the .c file. */
-	f = fopen("/tmp/pipeline.c", "w");
-	if (!f)
-		return -EIO;
-
-	/* Include the .h file. */
-	fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
-
-	/* Add the code for each action. */
-	TAILQ_FOREACH(a, &p->actions, node) {
-		fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
-
-		action_data_codegen(a, f);
-
-		fprintf(f, "\n");
-
-		action_instr_codegen(a, f);
-
-		fprintf(f, "\n");
-	}
-
-	/* Add the pipeline code. */
-	instruction_group_list_codegen(igl, p, f);
-
-	/* Close the .c file. */
-	fclose(f);
-
-	return 0;
-}
-
-#ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
-#define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
-#endif
-
-static int
-pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
-{
-	struct action *a;
-	struct instruction_group *g;
-	char *dir_in, *buffer = NULL;
-	const char *dir_out;
-	int status = 0;
-
-	/* Get the environment variables. */
-	dir_in = getenv("RTE_INSTALL_DIR");
-	if (!dir_in) {
-		status = -EINVAL;
-		goto free;
-	}
-
-	dir_out = "/tmp";
-
-	/* Memory allocation for the command buffer. */
-	buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
-	if (!buffer) {
-		status = -ENOMEM;
-		goto free;
-	}
-
-	snprintf(buffer,
-		 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
-		 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
-		 "-I %s/lib/pipeline "
-		 "-I %s/lib/eal/include "
-		 "-I %s/lib/eal/x86/include "
-		 "-I %s/lib/eal/include/generic "
-		 "-I %s/lib/meter "
-		 "-I %s/lib/port "
-		 "-I %s/lib/table "
-		 "-I %s/lib/pipeline "
-		 "-I %s/config "
-		 "-I %s/build "
-		 "-I %s/lib/eal/linux/include "
-		 ">%s/pipeline.log 2>&1 "
-		 "&& "
-		 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
-		 ">>%s/pipeline.log 2>&1",
-		 dir_out,
-		 dir_out,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_in,
-		 dir_out,
-		 dir_out,
-		 dir_out,
-		 dir_out);
-
-	/* Build the shared object library. */
-	status = system(buffer);
-	if (status)
-		goto free;
-
-	/* Open library. */
-	snprintf(buffer,
-		 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
-		 "%s/libpipeline.so",
-		 dir_out);
-
-	p->lib = dlopen(buffer, RTLD_LAZY);
-	if (!p->lib) {
-		status = -EIO;
-		goto free;
-	}
-
-	/* Get the action function symbols. */
-	TAILQ_FOREACH(a, &p->actions, node) {
-		snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
-
-		p->action_funcs[a->id] = dlsym(p->lib, buffer);
-		if (!p->action_funcs[a->id]) {
-			status = -EINVAL;
-			goto free;
-		}
-	}
-
-	/* Get the pipeline function symbols. */
-	TAILQ_FOREACH(g, igl, node) {
-		if (g->first_instr_id == g->last_instr_id)
-			continue;
-
-		snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
-
-		g->func = dlsym(p->lib, buffer);
-		if (!g->func) {
-			status = -EINVAL;
-			goto free;
-		}
-	}
-
-free:
-	if (status && p->lib) {
-		dlclose(p->lib);
-		p->lib = NULL;
-	}
-
-	free(buffer);
-
-	return status;
-}
-
 static int
 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
 		      struct instruction_group_list *igl)
@@ -13548,41 +13389,6 @@ pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
 	instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
 }
 
-static int
-pipeline_compile(struct rte_swx_pipeline *p)
-{
-	struct instruction_group_list *igl = NULL;
-	int status = 0;
-
-	igl = instruction_group_list_create(p);
-	if (!igl) {
-		status = -ENOMEM;
-		goto free;
-	}
-
-	/* Code generation. */
-	status = pipeline_codegen(p, igl);
-	if (status)
-		goto free;
-
-	/* Build and load the shared object library. */
-	status = pipeline_libload(p, igl);
-	if (status)
-		goto free;
-
-	/* Adjust instructions. */
-	status = pipeline_adjust_check(p, igl);
-	if (status)
-		goto free;
-
-	pipeline_adjust(p, igl);
-
-free:
-	instruction_group_list_free(igl);
-
-	return status;
-}
-
 int
 rte_swx_pipeline_codegen(FILE *spec_file,
 			 FILE *code_file,
@@ -13675,3 +13481,128 @@ rte_swx_pipeline_codegen(FILE *spec_file,
 
 	return status;
 }
+
+int
+rte_swx_pipeline_build_from_lib(struct rte_swx_pipeline **pipeline,
+				const char *name,
+				const char *lib_file_name,
+				FILE *iospec_file,
+				int numa_node)
+{
+	struct rte_swx_pipeline *p = NULL;
+	void *lib = NULL;
+	struct pipeline_iospec *sio = NULL;
+	struct pipeline_spec *s = NULL;
+	struct instruction_group_list *igl = NULL;
+	struct action *a;
+	struct instruction_group *g;
+	int status = 0;
+
+	/* Check input arguments. */
+	if (!pipeline ||
+	    !name ||
+	    !name[0] ||
+	    !lib_file_name ||
+	    !lib_file_name[0] ||
+	    !iospec_file) {
+		status = -EINVAL;
+		goto free;
+	}
+
+	/* Open the library. */
+	lib = dlopen(lib_file_name, RTLD_LAZY);
+	if (!lib) {
+		status = -EIO;
+		goto free;
+	}
+
+	/* Get the pipeline specification structures. */
+	s = dlsym(lib, "pipeline_spec");
+	if (!s) {
+		status = -EINVAL;
+		goto free;
+	}
+
+	sio = pipeline_iospec_parse(iospec_file, NULL, NULL);
+	if (!sio) {
+		status = -EINVAL;
+		goto free;
+	}
+
+	/* Pipeline configuration based on the specification structures. */
+	status = rte_swx_pipeline_config(&p, name, numa_node);
+	if (status)
+		goto free;
+
+	status = pipeline_iospec_configure(p, sio, NULL);
+	if (status)
+		goto free;
+
+	status = pipeline_spec_configure(p, s, NULL);
+	if (status)
+		goto free;
+
+	/* Pipeline build. */
+	status = rte_swx_pipeline_build(p);
+	if (status)
+		goto free;
+
+	/* Action instructions. */
+	TAILQ_FOREACH(a, &p->actions, node) {
+		char name[RTE_SWX_NAME_SIZE * 2];
+
+		snprintf(name, sizeof(name), "action_%s_run", a->name);
+
+		p->action_funcs[a->id] = dlsym(lib, name);
+		if (!p->action_funcs[a->id]) {
+			status = -EINVAL;
+			goto free;
+		}
+	}
+
+	/* Pipeline instructions. */
+	igl = instruction_group_list_create(p);
+	if (!igl) {
+		status = -ENOMEM;
+		goto free;
+	}
+
+	TAILQ_FOREACH(g, igl, node) {
+		char name[RTE_SWX_NAME_SIZE * 2];
+
+		if (g->first_instr_id == g->last_instr_id)
+			continue;
+
+		snprintf(name, sizeof(name), "pipeline_func_%u", g->group_id);
+
+		g->func = dlsym(lib, name);
+		if (!g->func) {
+			status = -EINVAL;
+			goto free;
+		}
+	}
+
+	status = pipeline_adjust_check(p, igl);
+	if (status)
+		goto free;
+
+	pipeline_adjust(p, igl);
+
+	p->lib = lib;
+
+	*pipeline = p;
+
+free:
+	instruction_group_list_free(igl);
+
+	pipeline_iospec_free(sio);
+
+	if (status) {
+		rte_swx_pipeline_free(p);
+
+		if (lib)
+			dlclose(lib);
+	}
+
+	return status;
+}
diff --git a/lib/pipeline/rte_swx_pipeline.h b/lib/pipeline/rte_swx_pipeline.h
index 724607b87c..9c629d4118 100644
--- a/lib/pipeline/rte_swx_pipeline.h
+++ b/lib/pipeline/rte_swx_pipeline.h
@@ -983,30 +983,38 @@ rte_swx_pipeline_codegen(FILE *spec_file,
 			 const char **err_msg);
 
 /**
- * Pipeline build from specification file
+ * Pipeline build from shared object library
  *
- * @param[in] p
- *   Pipeline handle.
- * @param[in] spec
- *   Pipeline specification file.
- * @param[out] err_line
- *   In case of error and non-NULL, the line number within the *spec* file where
- *   the error occurred. The first line number in the file is 1.
- * @param[out] err_msg
- *   In case of error and non-NULL, the error message.
+ * The shared object library must be built from the C language source code file
+ * previously generated by the rte_swx_pipeline_codegen() API function.
+ *
+ * The pipeline I/O specification file defines the I/O ports of the pipeline.
+ *
+ * @param[out] p
+ *   Pipeline handle. Must point to valid memory. Contains valid pipeline handle
+ *   when the function returns successfully.
+ * @param[in] name
+ *   Pipeline unique name.
+ * @param[in] lib_file_name
+ *   Shared object library file name.
+ * @param[in] iospec_file
+ *   Pipeline I/O specification file.
+ * @param[in] numa_node
+ *   Non-Uniform Memory Access (NUMA) node.
  * @return
  *   0 on success or the following error codes otherwise:
  *   -EINVAL: Invalid argument;
  *   -ENOMEM: Not enough space/cannot allocate memory;
- *   -EEXIST: Resource with the same name already exists;
+ *   -EEXIST: Pipeline with this name already exists;
  *   -ENODEV: Extern object or table creation error.
  */
 __rte_experimental
 int
-rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p,
-				 FILE *spec,
-				 uint32_t *err_line,
-				 const char **err_msg);
+rte_swx_pipeline_build_from_lib(struct rte_swx_pipeline **p,
+				const char *name,
+				const char *lib_file_name,
+				FILE *iospec_file,
+				int numa_node);
 
 /**
  * Pipeline run
diff --git a/lib/pipeline/rte_swx_pipeline_spec.c b/lib/pipeline/rte_swx_pipeline_spec.c
index f34803793d..3c16daf7de 100644
--- a/lib/pipeline/rte_swx_pipeline_spec.c
+++ b/lib/pipeline/rte_swx_pipeline_spec.c
@@ -3522,57 +3522,6 @@ pipeline_spec_configure(struct rte_swx_pipeline *p,
 	return 0;
 }
 
-int
-rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p,
-				 FILE *spec_file,
-				 uint32_t *err_line,
-				 const char **err_msg)
-{
-	struct pipeline_spec *s = NULL;
-	int status = 0;
-
-	/* Check the input arguments. */
-	if (!p || !spec_file) {
-		if (err_line)
-			*err_line = 0;
-		if (err_msg)
-			*err_msg = "Invalid input argument.";
-		status = -EINVAL;
-		goto error;
-	}
-
-	/* Spec file parse. */
-	s = pipeline_spec_parse(spec_file, err_line, err_msg);
-	if (!s) {
-		status = -EINVAL;
-		goto error;
-	}
-
-	/* Pipeline configure. */
-	status = pipeline_spec_configure(p, s, err_msg);
-	if (status) {
-		if (err_line)
-			*err_line = 0;
-		goto error;
-	}
-
-	/* Pipeline build. */
-	status = rte_swx_pipeline_build(p);
-	if (status) {
-		if (err_line)
-			*err_line = 0;
-		if (err_msg)
-			*err_msg = "Pipeline build error.";
-		goto error;
-	}
-
-	return 0;
-
-error:
-	pipeline_spec_free(s);
-	return status;
-}
-
 static void
 port_in_params_free(void *params, const char *port_type)
 {
diff --git a/lib/pipeline/version.map b/lib/pipeline/version.map
index 8d95005a5b..16806e6802 100644
--- a/lib/pipeline/version.map
+++ b/lib/pipeline/version.map
@@ -82,7 +82,6 @@ EXPERIMENTAL {
 	rte_swx_ctl_table_ops_get;
 	rte_swx_pipeline_action_config;
 	rte_swx_pipeline_build;
-	rte_swx_pipeline_build_from_spec;
 	rte_swx_pipeline_config;
 	rte_swx_pipeline_extern_func_register;
 	rte_swx_pipeline_extern_object_config;
@@ -148,6 +147,7 @@ EXPERIMENTAL {
 
 	#added in 22.11
 	rte_swx_ctl_pipeline_find;
+	rte_swx_pipeline_build_from_lib;
 	rte_swx_pipeline_codegen;
 	rte_swx_pipeline_find;
 };
-- 
2.34.1