[dpdk-dev] [PATCH v3 5/6] net/ark: cleanup and document ark dynamic extansion

[Ed Czeck <ed.czeck@atomicrules.com>]

- Rename extension functions with rte_pmd_ark prefix
- Move extension prototype to rte_pmd_ark.h
- Update documentation for extension

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
v3:
- split function rename from previous commit
---
 doc/guides/nics/ark.rst       | 139 ++++++++++++++++-
 drivers/net/ark/ark_ethdev.c  |  32 ++--
 drivers/net/ark/ark_ext.h     |  90 -----------
 drivers/net/ark/rte_pmd_ark.h | 284 +++++++++++++++++++++++++++++++++-
 4 files changed, 437 insertions(+), 108 deletions(-)
 delete mode 100644 drivers/net/ark/ark_ext.h

diff --git a/doc/guides/nics/ark.rst b/doc/guides/nics/ark.rst
index dcccfee26..bcc3babd5 100644
--- a/doc/guides/nics/ark.rst
+++ b/doc/guides/nics/ark.rst
@@ -1,5 +1,5 @@
 .. SPDX-License-Identifier: BSD-3-Clause
-    Copyright (c) 2015-2017 Atomic Rules LLC
+    Copyright (c) 2015-2021 Atomic Rules LLC
     All rights reserved.
 
 ARK Poll Mode Driver
@@ -130,6 +130,141 @@ Configuration Information
      be offloaded or remain in host software.
 
 
+Dynamic PMD Extension
+---------------------
+
+Dynamic PMD extensions allow users to customize net/ark functionality
+using their own code. Arkville RTL and this PMD support high-throughput data
+movement, and these extensions allow PMD support for users' FPGA
+features.
+Dynamic PMD extensions operate by having users supply a shared object
+file which is loaded by Arkville PMD during initialization.  The
+object file contains extension (or hook) functions that are registered
+and then called during PMD operations.
+
+The allowable set of extension functions are defined and documented in
+``rte_pmd_ark.h``, only the initialization function,
+``rte_pmd_ark_dev_init()``, is required; all others are optional. The
+following sections give a small extension example along with
+instructions for compiling and using the extension.
+
+
+Extension Example
+^^^^^^^^^^^^^^^^^
+
+The following example shows an extension which populates mbuf fields
+during RX from user meta data coming from FPGA hardware.
+
+.. code-block:: c
+
+   #include <rte_pmd_ark.h>
+   #include <rte_mbuf.h>
+   #include <rte_ethdev.h>
+   #include <rte_malloc.h>
+
+   /* Global structure passed to extension/hook functions */
+   struct ark_user_extension {
+       int timestamp_dynfield_offset;
+   };
+
+   /* RX tuser field based on user's hardware */
+   struct user_rx_meta {
+      uint64_t timestamp;
+      uint32_t rss;
+   } __rte_packed;
+
+   /* Create ark_user_extension object for use in other hook functions */
+   void *rte_pmd_ark_dev_init(struct rte_eth_dev * dev,
+                              void * abar, int port_id )
+   {
+      RTE_SET_USED(dev);
+      RTE_SET_USED(abar);
+      fprintf(stderr, "Called Arkville user extension for port %u\n",
+              port_id);
+
+      struct ark_user_extension *xdata = rte_zmalloc("macExtS",
+             sizeof(struct ark_user_extension), 64);
+      if (!xdata)
+         return NULL;
+
+      /* register dynfield for rx timestamp */
+      rte_mbuf_dyn_rx_timestamp_register(&xdata->timestamp_dynfield_offset,
+                                         NULL);
+
+      fprintf(stderr, "timestamp fields offset in extension is %d\n",
+              xdata->timestamp_dynfield_offset);
+      return xdata;
+   }
+
+   /* uninitialization */
+   void rte_pmd_ark_dev_uninit(struct rte_eth_dev * dev, void *user_data)
+   {
+      rte_free(user_data);
+   }
+
+   /* Hook function -- called for each RX packet
+    * Extract RX timestamp and RSS from meta and place in mbuf
+    */
+   void rte_pmd_ark_rx_user_meta_hook(struct rte_mbuf *mbuf,
+                                      const uint32_t *meta,
+                                      void *user_data)
+   {
+      struct ark_user_extension *xdata = user_data;
+      struct user_rx_meta *user_rx = (struct user_rx_meta*)meta;
+      *RTE_MBUF_DYNFIELD(mbuf, xdata->timestamp_dynfield_offset, uint64_t*) =
+                         user_rx->timestamp;
+      mbuf->hash.rss = user_rx->rss;
+   }
+
+
+Compiling Extension
+^^^^^^^^^^^^^^^^^^^
+
+It is recommended to the compile the extension code with
+``-Wmissing-prototypes`` flag to insure correct function types. Typical
+DPDK options will also be needed.
+
+
+An example command line is give below
+
+.. code-block:: console
+
+    cc `pkg-config --cflags libdpdk` \
+    -O3 -DALLOW_EXPERIMENTAL_API -fPIC -Wall -Wmissing-prototypes -c \
+    -o pmd_net_ark_ext.o pmd_net_ark_ext.c
+    # Linking
+    cc -o libfx1_100g_ext.so.1 -shared \
+    `pkg-config --libs libdpdk` \
+    -Wl,--unresolved-symbols=ignore-all \
+    -Wl,-soname,libpmd_net_ark_ext.so.1 pmd_net_ark_ext.o
+
+In a ``Makefile`` this would be
+
+.. code-block:: Makefile
+
+   CFLAGS += $(shell pkg-config --cflags libdpdk)
+   CFLAGS += -O3 -DALLOW_EXPERIMENTAL_API -fPIC -Wall -Wmissing-prototypes
+   # Linking
+   LDFLAGS += $(shell pkg-config --libs libdpdk)
+   LDFLAGS += -Wl,--unresolved-symbols=ignore-all -Wl,-soname,libpmd_net_ark_ext.so.1
+
+The application must be linked with the ``-export-dynamic`` flags if any
+DPDK or application specific code will called from the extension.
+
+
+Enabling Extension
+^^^^^^^^^^^^^^^^^^
+
+The extensions are enabled in the application through the use of an
+environment variable ``ARK_EXT_PATH`` This variable points to the lib
+extension file generated above.  For example:
+
+.. code-block:: console
+
+   export ARK_EXT_PATH=$(PWD)/libpmd_net_ark_ext.so.1
+   testpmd ...
+
+
 Building DPDK
 -------------
 
@@ -144,6 +279,8 @@ documentation that comes with DPDK suite <linux_gsg>`.
 To build with a non-zero minimum tx packet length, set the above macro in your
 CFLAGS environment prior to the meson build step. I.e.,
 
+.. code-block:: console
+
     export CFLAGS="-DRTE_LIBRTE_ARK_MIN_TX_PKTLEN=60"
     meson build
 
diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 15ec83893..f8a4b8711 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -185,58 +185,58 @@ check_for_ext(struct ark_adapter *ark)
 	/* Get the entry points */
 	ark->user_ext.dev_init =
 		(void *(*)(struct rte_eth_dev *, void *, int))
-		dlsym(ark->d_handle, "dev_init");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_init");
 	ARK_PMD_LOG(DEBUG, "device ext init pointer = %p\n",
 		      ark->user_ext.dev_init);
 	ark->user_ext.dev_get_port_count =
 		(int (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_get_port_count");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_get_port_count");
 	ark->user_ext.dev_uninit =
 		(void (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_uninit");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_uninit");
 	ark->user_ext.dev_configure =
 		(int (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_configure");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_configure");
 	ark->user_ext.dev_start =
 		(int (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_start");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_start");
 	ark->user_ext.dev_stop =
 		(void (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_stop");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_stop");
 	ark->user_ext.dev_close =
 		(void (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_close");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_close");
 	ark->user_ext.link_update =
 		(int (*)(struct rte_eth_dev *, int, void *))
-		dlsym(ark->d_handle, "link_update");
+		dlsym(ark->d_handle, "rte_pmd_ark_link_update");
 	ark->user_ext.dev_set_link_up =
 		(int (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_set_link_up");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_set_link_up");
 	ark->user_ext.dev_set_link_down =
 		(int (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "dev_set_link_down");
+		dlsym(ark->d_handle, "rte_pmd_ark_dev_set_link_down");
 	ark->user_ext.stats_get =
 		(int (*)(struct rte_eth_dev *, struct rte_eth_stats *,
 			  void *))
-		dlsym(ark->d_handle, "stats_get");
+		dlsym(ark->d_handle, "rte_pmd_ark_stats_get");
 	ark->user_ext.stats_reset =
 		(void (*)(struct rte_eth_dev *, void *))
-		dlsym(ark->d_handle, "stats_reset");
+		dlsym(ark->d_handle, "rte_pmd_ark_stats_reset");
 	ark->user_ext.mac_addr_add =
 		(void (*)(struct rte_eth_dev *, struct rte_ether_addr *,
 			uint32_t, uint32_t, void *))
-		dlsym(ark->d_handle, "mac_addr_add");
+		dlsym(ark->d_handle, "rte_pmd_ark_mac_addr_add");
 	ark->user_ext.mac_addr_remove =
 		(void (*)(struct rte_eth_dev *, uint32_t, void *))
-		dlsym(ark->d_handle, "mac_addr_remove");
+		dlsym(ark->d_handle, "rte_pmd_ark_mac_addr_remove");
 	ark->user_ext.mac_addr_set =
 		(void (*)(struct rte_eth_dev *, struct rte_ether_addr *,
 			  void *))
-		dlsym(ark->d_handle, "mac_addr_set");
+		dlsym(ark->d_handle, "rte_pmd_ark_mac_addr_set");
 	ark->user_ext.set_mtu =
 		(int (*)(struct rte_eth_dev *, uint16_t,
 			  void *))
-		dlsym(ark->d_handle, "set_mtu");
+		dlsym(ark->d_handle, "rte_pmd_ark_set_mtu");
 	ark->user_ext.rx_user_meta_hook =
 		(rx_user_meta_hook_fn)dlsym(ark->d_handle,
 					    "rte_pmd_ark_rx_user_meta_hook");
diff --git a/drivers/net/ark/ark_ext.h b/drivers/net/ark/ark_ext.h
deleted file mode 100644
index 821fb55bb..000000000
--- a/drivers/net/ark/ark_ext.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
- */
-
-#ifndef _ARK_EXT_H_
-#define _ARK_EXT_H_
-
-#include <ethdev_driver.h>
-
-/*
- * This is the template file for users who which to define a dynamic
- * extension to the Arkville PMD.   User's who create an extension
- * should include this file and define the necessary and desired
- * functions.
- * Only 1 function is required for an extension, dev_init(); all other
- * functions prototyped in this file are optional.
- */
-
-/*
- * Called post PMD init.
- * The implementation returns its private data that gets passed into
- * all other functions as user_data
- * The ARK extension implementation MUST implement this function
- */
-void *dev_init(struct rte_eth_dev *dev, void *a_bar, int port_id);
-
-/* Called during device shutdown */
-void dev_uninit(struct rte_eth_dev *dev, void *user_data);
-
-/* This call is optional and allows the
- * extension to specify the number of supported ports.
- */
-uint8_t dev_get_port_count(struct rte_eth_dev *dev,
-			   void *user_data);
-
-/*
- * The following functions are optional and are directly mapped
- * from the DPDK PMD ops structure.
- * Each function if implemented is called after the ARK PMD
- * implementation executes.
- */
-
-int dev_configure(struct rte_eth_dev *dev,
-		  void *user_data);
-
-int dev_start(struct rte_eth_dev *dev,
-	      void *user_data);
-
-void dev_stop(struct rte_eth_dev *dev,
-	      void *user_data);
-
-void dev_close(struct rte_eth_dev *dev,
-	       void *user_data);
-
-int link_update(struct rte_eth_dev *dev,
-		int wait_to_complete,
-		void *user_data);
-
-int dev_set_link_up(struct rte_eth_dev *dev,
-		    void *user_data);
-
-int dev_set_link_down(struct rte_eth_dev *dev,
-		      void *user_data);
-
-int stats_get(struct rte_eth_dev *dev,
-	       struct rte_eth_stats *stats,
-	       void *user_data);
-
-void stats_reset(struct rte_eth_dev *dev,
-		 void *user_data);
-
-void mac_addr_add(struct rte_eth_dev *dev,
-		  struct rte_ether_addr *macadr,
-		  uint32_t index,
-		  uint32_t pool,
-		  void *user_data);
-
-void mac_addr_remove(struct rte_eth_dev *dev,
-		     uint32_t index,
-		     void *user_data);
-
-void mac_addr_set(struct rte_eth_dev *dev,
-		  struct rte_ether_addr *mac_addr,
-		  void *user_data);
-
-int set_mtu(struct rte_eth_dev *dev,
-	    uint16_t size,
-	    void *user_data);
-
-#endif
diff --git a/drivers/net/ark/rte_pmd_ark.h b/drivers/net/ark/rte_pmd_ark.h
index a77dd311f..7c093711f 100644
--- a/drivers/net/ark/rte_pmd_ark.h
+++ b/drivers/net/ark/rte_pmd_ark.h
@@ -1,13 +1,295 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2020 Atomic Rules LLC
+ * Copyright (c) 2020-2021 Atomic Rules LLC
  */
 
 #ifndef RTE_PMD_ARK_H
 #define RTE_PMD_ARK_H
 
+#include <stdint.h>
+struct rte_eth_dev;
+struct rte_mbuf;
+struct rte_ether_addr;
+struct rte_eth_stats;
+
 /**
  * @file
  * ARK driver-specific API
  */
 
+/* The following section lists function prototypes for Arkville's
+ * dynamic PMD extension. User's who create an extension
+ * must include this file and define the necessary and desired
+ * functions. Only 1 function is required for an extension,
+ * rte_pmd_ark_dev_init(); all other functions prototypes in this
+ * section are optional.
+ * See documentation for compiling and use of extensions.
+ */
+
+/**
+ * Extension prototype, required implementation if extensions are used.
+ * Called during device probe to initialize the user structure
+ * passed to other extension functions.  This is called once for each
+ * port of the device.
+ *
+ * @param dev
+ *   current device.
+ * @param a_bar
+ *   access to PCIe device bar (application bar) and hence access to
+ *   user's portion of FPGA.
+ * @param port_id
+ *   port identifier.
+ * @return user_data
+ *   which will be passed to other extension functions.
+ */
+void *rte_pmd_ark_dev_init(struct rte_eth_dev *dev, void *a_bar, int port_id);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during device uninit.
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ */
+void rte_pmd_ark_dev_uninit(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during device probe to change the port count from 1.
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+uint8_t dev_get_port_count(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_configure().
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+int rte_pmd_ark_dev_configure(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_start().
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+int rte_pmd_ark_dev_start(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during  rte_eth_dev_stop().
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+void rte_pmd_ark_dev_stop(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_close().
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+void rte_pmd_ark_dev_close(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during link_update status event.
+ *
+ * @param dev
+ *   current device.
+ * @param wait_to_complete
+ *    argument from update event.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+int rte_pmd_ark_link_update(struct rte_eth_dev *dev,
+			    int wait_to_complete,
+			    void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_set_link_up().
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+int rte_pmd_ark_dev_set_link_up(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_set_link_down().
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+int rte_pmd_ark_dev_set_link_down(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_stats_get(); allows updates to the stats
+ * struct in addition Ark's PMD operations.
+ *
+ * @param dev
+ *   current device.
+ * @param stats
+ *   statistics struct already populated by Ark PMD.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+int rte_pmd_ark_stats_get(struct rte_eth_dev *dev,
+			  struct rte_eth_stats *stats,
+			  void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_stats_reset().
+ *
+ * @param dev
+ *   current device.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+void rte_pmd_ark_stats_reset(struct rte_eth_dev *dev, void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_mac_addr_add().
+ *
+ * @param dev
+ *   current device.
+ * @param macaddr
+ *   The MAC address to add
+ * @param index
+ *   The index into the MAC address array.
+ * @param pool
+ *   VMDq pool index from caller
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+void rte_pmd_ark_mac_addr_add(struct rte_eth_dev *dev,
+			      struct rte_ether_addr *macaddr,
+			      uint32_t index,
+			      uint32_t pool,
+			      void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_mac_addr_remove().
+ *
+ * @param dev
+ *   current device.
+ * @param index
+ *   The index into the MAC address array.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+void rte_pmd_ark_mac_addr_remove(struct rte_eth_dev *dev,
+				 uint32_t index,
+				 void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_default_mac_addr_set().
+ *
+ * @param dev
+ *   current device.
+ * @param mac_addr
+ *   The new default MAC address.
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+void rte_pmd_ark_mac_addr_set(struct rte_eth_dev *dev,
+			      struct rte_ether_addr *mac_addr,
+			      void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_dev_set_mtu().
+ *
+ * @param dev
+ *   current device.
+ * @param size
+ *   The MTU to be applied
+ * @param user_data
+ *   user argument from dev_init() call.
+ * @return (0) if successful.
+ */
+int rte_pmd_ark_set_mtu(struct rte_eth_dev *dev,
+			uint16_t size,
+			void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_rx_burst() for each packet. This extension
+ * function allows the transfer of meta data from the user's FPGA to
+ * mbuf fields.
+ *
+ * @param mbuf
+ *   The newly received mbuf
+ * @param meta
+ *   The meta data from the user, up to 20 bytes. The
+ *   underlying data in the PMD is of type uint32_t meta[5];
+ * @param user_data
+ *   user argument from dev_init() call.
+ */
+void rte_pmd_ark_rx_user_meta_hook(struct rte_mbuf *mbuf,
+				   const uint32_t *meta,
+				   void *user_data);
+
+/**
+ * Extension prototype, optional implementation.
+ * Called during rte_eth_tx_burst() for each packet. This extension
+ * function allows the transfer of data from the mbuf to the user's
+ * FPGA.  Up to 20 bytes (5 32-bit words) are transferable
+ *
+ * @param mbuf
+ *   The mbuf about to be transmitted.
+ * @param meta
+ *   The meta data to be populate by this call. The
+ *   underlying in the PMD is of type uint32_t meta[5];
+ * @param meta_cnt
+ *   The count in 32-bit words of the meta data populated, 0 to 5.
+ * @param user_data
+ *   user argument from dev_init() call.
+ */
+void rte_pmd_ark_tx_user_meta_hook(const struct rte_mbuf *mbuf,
+				   uint32_t *meta,
+				   uint8_t *meta_cnt,
+				   void *user_data);
+
 #endif /* RTE_PMD_ARK_H */
-- 
2.17.1