[dpdk-dev] [PATCH 0/2] Support compressed firmwares

[dpdk-dev] [PATCH 0/2] Support compressed firmwares

[David Marchand]

Fedora 34 only provides compressed firmwares.

Introduce an internal driver helper to handle transparently compression.

I chose libarchive for decompressing as it seems widely available and
DPDK had used it in the past.

Windows support only matters for net/ice and firmware loading was skipped
in this driver before this series. Since I don't know if/how we want to
load firmwares on Windows, I let an empty stub for this OS.

This series has been compile tested on Linux (I'll trust the CI for
others OSes).
I only tested basic init with a net/ice device (no DCF test).

So please drivers maintainers, check nothing is broken.


-- 
David Marchand

David Marchand (2):
  net/ice: factorize firmware loading
  eal: handle compressed firmwares

 .github/workflows/build.yml      |   1 +
 .travis.yml                      |   1 +
 config/meson.build               |   9 ++
 drivers/net/bnx2x/bnx2x.c        |  35 +++----
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++---------------
 drivers/net/ice/ice_ethdev.c     | 175 ++++++++++++-------------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 drivers/net/nfp/nfp_net.c        |  57 +++-------
 drivers/net/qede/qede_main.c     |  44 +++-----
 lib/eal/include/rte_firmware.h   |  35 +++++++
 lib/eal/unix/eal_firmware.c      | 106 +++++++++++++++++++
 lib/eal/unix/meson.build         |   1 +
 lib/eal/version.map              |   2 +
 lib/eal/windows/eal.c            |   9 ++
 15 files changed, 283 insertions(+), 298 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

-- 
2.23.0

[dpdk-dev] [PATCH 1/2] net/ice: factorize firmware loading

[David Marchand]

Both "normal" and "dcf" inits have their copy of some firmware loading
code.

The DSN query is moved in specific parts for the "normal" and "dcf" init.

A common helper ice_load_pkg is then introduced and takes an adapter
pointer as its main input.

This helper takes care of finding the right firmware file and loading
it.
The adapter active_pkg_type field is set by this helper.

The ice_access macro is removed from the osdep.h header: osdep.h should
only hosts wrappers for base driver code.

Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
 drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 4 files changed, 88 insertions(+), 179 deletions(-)

diff --git a/drivers/net/ice/base/ice_osdep.h b/drivers/net/ice/base/ice_osdep.h
index 878c5597d4..78093adb00 100644
--- a/drivers/net/ice/base/ice_osdep.h
+++ b/drivers/net/ice/base/ice_osdep.h
@@ -74,12 +74,6 @@ typedef uint64_t        s64;
 #define min(a, b) RTE_MIN(a, b)
 #define max(a, b) RTE_MAX(a, b)
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 #define FIELD_SIZEOF(t, f) RTE_SIZEOF_FIELD(t, f)
 #define ARRAY_SIZE(arr) RTE_DIM(arr)
 
diff --git a/drivers/net/ice/ice_dcf_parent.c b/drivers/net/ice/ice_dcf_parent.c
index 1d7aa8bc87..a30cfefaaf 100644
--- a/drivers/net/ice/ice_dcf_parent.c
+++ b/drivers/net/ice/ice_dcf_parent.c
@@ -298,13 +298,14 @@ static void ice_dcf_uninit_parent_hw(struct ice_hw *hw)
 }
 
 static int
-ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
+ice_dcf_load_pkg(struct ice_adapter *adapter)
 {
 	struct ice_dcf_adapter *dcf_adapter =
-			container_of(hw, struct ice_dcf_adapter, parent.hw);
+			container_of(&adapter->hw, struct ice_dcf_adapter, parent.hw);
 	struct virtchnl_pkg_info pkg_info;
 	struct dcf_virtchnl_cmd vc_cmd;
-	uint64_t dsn;
+	bool use_dsn;
+	uint64_t dsn = 0;
 
 	vc_cmd.v_op = VIRTCHNL_OP_DCF_GET_PKG_INFO;
 	vc_cmd.req_msglen = 0;
@@ -312,90 +313,11 @@ ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
 	vc_cmd.rsp_buflen = sizeof(pkg_info);
 	vc_cmd.rsp_msgbuf = (uint8_t *)&pkg_info;
 
-	if (ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd))
-		goto pkg_file_direct;
+	use_dsn = ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd) == 0;
+	if (use_dsn)
+		rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
 
-	rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_UPDATES "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_DEFAULT "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-pkg_file_direct:
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_UPDATES);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_DEFAULT);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	return -1;
-}
-
-static int
-ice_dcf_load_pkg(struct ice_hw *hw)
-{
-	char pkg_name[ICE_MAX_PKG_FILENAME_SIZE];
-	uint8_t *pkg_buf;
-	uint32_t buf_len;
-	struct stat st;
-	FILE *fp;
-	int err;
-
-	if (ice_dcf_request_pkg_name(hw, pkg_name)) {
-		PMD_INIT_LOG(ERR, "Failed to locate the package file");
-		return -ENOENT;
-	}
-
-	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_name);
-
-	err = stat(pkg_name, &st);
-	if (err) {
-		PMD_INIT_LOG(ERR, "Failed to get file status");
-		return err;
-	}
-
-	buf_len = st.st_size;
-	pkg_buf = rte_malloc(NULL, buf_len, 0);
-	if (!pkg_buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buffer of size %u for package",
-			     buf_len);
-		return -1;
-	}
-
-	fp = fopen(pkg_name, "rb");
-	if (!fp)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s", pkg_name);
-		err = -1;
-		goto ret;
-	}
-
-	err = fread(pkg_buf, buf_len, 1, fp);
-	fclose(fp);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data");
-		err = -1;
-		goto ret;
-	}
-
-	err = ice_copy_and_init_pkg(hw, pkg_buf, buf_len);
-	if (err)
-		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d", err);
-
-ret:
-	rte_free(pkg_buf);
-	return err;
+	return ice_load_pkg(adapter, use_dsn, dsn);
 }
 
 int
@@ -436,13 +358,12 @@ ice_dcf_init_parent_adapter(struct rte_eth_dev *eth_dev)
 		return err;
 	}
 
-	err = ice_dcf_load_pkg(parent_hw);
+	err = ice_dcf_load_pkg(parent_adapter);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to load package with error %d",
 			     err);
 		goto uninit_hw;
 	}
-	parent_adapter->active_pkg_type = ice_load_pkg_type(parent_hw);
 
 	parent_adapter->pf.main_vsi->idx = hw->num_vfs;
 	ice_dcf_update_pf_vsi_map(parent_hw,
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 5fd5f99b6f..05ccc41d95 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -1649,57 +1649,7 @@ ice_pf_setup(struct ice_pf *pf)
 	return 0;
 }
 
-/*
- * Extract device serial number from PCIe Configuration Space and
- * determine the pkg file path according to the DSN.
- */
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int
-ice_pkg_file_search_path(struct rte_pci_device *pci_dev, char *pkg_file)
-{
-	off_t pos;
-	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
-	uint32_t dsn_low, dsn_high;
-	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
-
-	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
-
-	if (pos) {
-		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		if (rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
-			 "ice-%08x%08x.pkg", dsn_high, dsn_low);
-	} else {
-		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
-		goto fail_dsn;
-	}
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-fail_dsn:
-	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
-		return 0;
-	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	return 0;
-}
-#endif
-
-enum ice_pkg_type
+static enum ice_pkg_type
 ice_load_pkg_type(struct ice_hw *hw)
 {
 	enum ice_pkg_type package_type;
@@ -1723,37 +1673,62 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int ice_load_pkg(struct rte_eth_dev *dev)
+#ifdef RTE_EXEC_ENV_WINDOWS
+#define ice_access _access
+#else
+#define ice_access access
+#endif
+
+int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
-	struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
+	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
 	int err;
-	uint8_t *buf;
+	uint8_t *buf = NULL;
 	int buf_len;
 	FILE *file;
 	struct stat fstat;
-	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
-	struct ice_adapter *ad =
-		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 
-	err = ice_pkg_file_search_path(pci_dev, pkg_file);
-	if (err) {
+	if (!use_dsn)
+		goto no_dsn;
+
+	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
+	snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
+		"ice-%016" PRIx64 ".pkg", dsn);
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+no_dsn:
+	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(pkg_file, 0))
+		goto load_fw;
+	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
+	if (ice_access(pkg_file, 0)) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
-		return err;
+		return -1;
 	}
 
+load_fw:
 	file = fopen(pkg_file, "rb");
 	if (!file)  {
 		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
 		return -1;
 	}
 
+	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
+
 	err = stat(pkg_file, &fstat);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		fclose(file);
-		return err;
+		goto out;
 	}
 
 	buf_len = fstat.st_size;
@@ -1762,44 +1737,33 @@ static int ice_load_pkg(struct rte_eth_dev *dev)
 	if (!buf) {
 		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
 				buf_len);
-		fclose(file);
-		return -1;
+		err = -1;
+		goto out;
 	}
 
 	err = fread(buf, buf_len, 1, file);
 	if (err != 1) {
 		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		fclose(file);
 		err = -1;
-		goto fail_exit;
+		goto out;
 	}
 
-	fclose(file);
-
 	err = ice_copy_and_init_pkg(hw, buf, buf_len);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
-		goto fail_exit;
+		goto out;
 	}
 
 	/* store the loaded pkg type info */
-	ad->active_pkg_type = ice_load_pkg_type(hw);
+	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
-	err = ice_init_hw_tbls(hw);
-	if (err) {
-		PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", err);
-		goto fail_init_tbls;
-	}
-
-	return 0;
-
-fail_init_tbls:
-	rte_free(hw->pkg_copy);
-fail_exit:
+out:
+	fclose(file);
 	rte_free(buf);
 	return err;
 }
-#endif
+
+#undef ice_access
 
 static void
 ice_base_queue_get(struct ice_pf *pf)
@@ -2030,6 +1994,12 @@ ice_dev_init(struct rte_eth_dev *dev)
 		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_vsi *vsi;
 	int ret;
+#ifndef RTE_EXEC_ENV_WINDOWS
+	off_t pos;
+	uint32_t dsn_low, dsn_high;
+	uint64_t dsn;
+	bool use_dsn;
+#endif
 
 	dev->dev_ops = &ice_eth_dev_ops;
 	dev->rx_queue_count = ice_rx_queue_count;
@@ -2081,7 +2051,30 @@ ice_dev_init(struct rte_eth_dev *dev)
 	}
 
 #ifndef RTE_EXEC_ENV_WINDOWS
-	ret = ice_load_pkg(dev);
+	use_dsn = false;
+	dsn = 0;
+	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
+	if (pos) {
+		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0 ||
+				rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
+			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
+		} else {
+			use_dsn = true;
+			dsn = (uint64_t)dsn_high << 32 | dsn_low;
+		}
+	} else {
+		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
+	}
+
+	ret = ice_load_pkg(pf->adapter, use_dsn, dsn);
+	if (ret == 0) {
+		ret = ice_init_hw_tbls(hw);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", ret);
+			rte_free(hw->pkg_copy);
+		}
+	}
+
 	if (ret) {
 		if (ad->devargs.safe_mode_support == 0) {
 			PMD_INIT_LOG(ERR, "Failed to load the DDP package,"
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index 2a8a8169d5..1997f9296f 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -531,7 +531,8 @@ struct ice_vsi_vlan_pvid_info {
 #define ICE_PF_TO_ETH_DEV(pf) \
 	(((struct ice_pf *)pf)->adapter->eth_dev)
 
-enum ice_pkg_type ice_load_pkg_type(struct ice_hw *hw);
+int
+ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn);
 struct ice_vsi *
 ice_setup_vsi(struct ice_pf *pf, enum ice_vsi_type type);
 int
-- 
2.23.0

[dpdk-dev] [PATCH 2/2] eal: handle compressed firmwares

[David Marchand]

Introduce an internal firmware loading helper to remove code duplication
in our drivers and handle xz compressed firmwares by calling libarchive.

This helper tries to look for .xz suffixes so that drivers are not aware
the firmwares have been compressed.

libarchive is set as an optional dependency: without libarchive, a
runtime warning is emitted so that users know there is a compressed
firmware.

Windows implementation is left as an empty stub.

Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 .github/workflows/build.yml    |   1 +
 .travis.yml                    |   1 +
 config/meson.build             |   9 +++
 drivers/net/bnx2x/bnx2x.c      |  35 ++++-------
 drivers/net/ice/ice_ethdev.c   |  60 ++++---------------
 drivers/net/nfp/nfp_net.c      |  57 ++++--------------
 drivers/net/qede/qede_main.c   |  44 ++++++--------
 lib/eal/include/rte_firmware.h |  35 +++++++++++
 lib/eal/unix/eal_firmware.c    | 106 +++++++++++++++++++++++++++++++++
 lib/eal/unix/meson.build       |   1 +
 lib/eal/version.map            |   2 +
 lib/eal/windows/eal.c          |   9 +++
 12 files changed, 218 insertions(+), 142 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c4d6dcdbf..7dac20ddeb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -93,6 +93,7 @@ jobs:
       run: sudo apt install -y ccache libnuma-dev python3-setuptools
         python3-wheel python3-pip python3-pyelftools ninja-build libbsd-dev
         libpcap-dev libibverbs-dev libcrypto++-dev libfdt-dev libjansson-dev
+        libarchive-dev
     - name: Install libabigail build dependencies if no cache is available
       if: env.ABI_CHECKS == 'true' && steps.libabigail-cache.outputs.cache-hit != 'true'
       run: sudo apt install -y autoconf automake libtool pkg-config libxml2-dev
diff --git a/.travis.yml b/.travis.yml
index 5b702cc9bb..23067d9e3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,7 @@ addons:
     packages: &required_packages
       - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
       - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
+      - [libarchive-dev]
 
 _aarch64_packages: &aarch64_packages
   - *required_packages
diff --git a/config/meson.build b/config/meson.build
index 017bb2efbb..337daa2719 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -166,6 +166,15 @@ if fdt_dep.found() and cc.has_header('fdt.h')
     dpdk_extra_ldflags += '-lfdt'
 endif
 
+has_libarchive = 0
+archive_dep = cc.find_library('libarchive', required: false)
+if archive_dep.found() and cc.has_header('archive.h')
+    dpdk_conf.set10('RTE_HAS_LIBARCHIVE', true)
+    has_libarchive = 1
+    add_project_link_arguments('-larchive', language: 'c')
+    dpdk_extra_ldflags += '-larchive'
+endif
+
 libexecinfo = cc.find_library('libexecinfo', required: false)
 if libexecinfo.found() and cc.has_header('execinfo.h')
     add_project_link_arguments('-lexecinfo', language: 'c')
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
index 654878d9de..60292753e2 100644
--- a/drivers/net/bnx2x/bnx2x.c
+++ b/drivers/net/bnx2x/bnx2x.c
@@ -26,7 +26,9 @@
 #include <arpa/inet.h>
 #include <fcntl.h>
 #include <zlib.h>
+
 #include <rte_bitops.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
@@ -9655,44 +9657,33 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc)
 void bnx2x_load_firmware(struct bnx2x_softc *sc)
 {
 	const char *fwname;
-	int f;
-	struct stat st;
+	void *buf;
+	size_t bufsz;
 
 	fwname = sc->devinfo.device_id == CHIP_NUM_57711
 		? FW_NAME_57711 : FW_NAME_57810;
-	f = open(fwname, O_RDONLY);
-	if (f < 0) {
+	if (rte_firmware_read(fwname, &buf, &bufsz) != 0) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
 		return;
 	}
 
-	if (fstat(f, &st) < 0) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
-		close(f);
-		return;
-	}
-
-	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
+	sc->firmware = rte_zmalloc("bnx2x_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!sc->firmware) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
-		close(f);
-		return;
+		goto out;
 	}
 
-	if (read(f, sc->firmware, st.st_size) != st.st_size) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
-		close(f);
-		return;
-	}
-	close(f);
-
-	sc->fw_len = st.st_size;
+	sc->fw_len = bufsz;
 	if (sc->fw_len < FW_HEADER_LEN) {
 		PMD_DRV_LOG(NOTICE, sc,
 			    "Invalid fw size: %" PRIu64, sc->fw_len);
-		return;
+		goto out;
 	}
+
+	memcpy(sc->firmware, buf, sc->fw_len);
 	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
+out:
+	free(buf);
 }
 
 static void
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 05ccc41d95..6bb8c76ab3 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -10,6 +10,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include <rte_firmware.h>
 #include <rte_tailq.h>
 
 #include "base/ice_sched.h"
@@ -1673,22 +1674,14 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
 	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
 	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
+	void *buf;
+	size_t bufsz;
 	int err;
-	uint8_t *buf = NULL;
-	int buf_len;
-	FILE *file;
-	struct stat fstat;
 
 	if (!use_dsn)
 		goto no_dsn;
@@ -1698,57 +1691,31 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 		"ice-%016" PRIx64 ".pkg", dsn);
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 no_dsn:
 	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
+
 	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	if (ice_access(pkg_file, 0)) {
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
 		return -1;
 	}
 
 load_fw:
-	file = fopen(pkg_file, "rb");
-	if (!file)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
-		return -1;
-	}
-
 	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
 
-	err = stat(pkg_file, &fstat);
-	if (err) {
-		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		goto out;
-	}
-
-	buf_len = fstat.st_size;
-	buf = rte_malloc(NULL, buf_len, 0);
-
-	if (!buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
-				buf_len);
-		err = -1;
-		goto out;
-	}
-
-	err = fread(buf, buf_len, 1, file);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		err = -1;
-		goto out;
-	}
-
-	err = ice_copy_and_init_pkg(hw, buf, buf_len);
+	err = ice_copy_and_init_pkg(hw, buf, bufsz);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
 		goto out;
@@ -1758,13 +1725,10 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
 out:
-	fclose(file);
-	rte_free(buf);
+	free(buf);
 	return err;
 }
 
-#undef ice_access
-
 static void
 ice_base_queue_get(struct ice_pf *pf)
 {
diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
index 2ee88fbfc7..879da7ef59 100644
--- a/drivers/net/nfp/nfp_net.c
+++ b/drivers/net/nfp/nfp_net.c
@@ -29,6 +29,7 @@
 #include <rte_alarm.h>
 #include <rte_spinlock.h>
 #include <rte_service_component.h>
+#include <rte_firmware.h>
 
 #include "nfpcore/nfp_cpp.h"
 #include "nfpcore/nfp_nffw.h"
@@ -3366,12 +3367,10 @@ static int
 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 {
 	struct nfp_cpp *cpp = nsp->cpp;
-	int fw_f;
-	char *fw_buf;
+	void *fw_buf;
 	char fw_name[125];
 	char serial[40];
-	struct stat file_stat;
-	off_t fsize, bytes;
+	size_t fsize;
 
 	/* Looking for firmware file in order of priority */
 
@@ -3384,66 +3383,34 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 
 	snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 			serial);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Then try the PCI name */
 	snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 			dev->device.name);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Finally try the card type and media */
 	snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f < 0) {
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
 		PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 		return -ENOENT;
 	}
 
-read_fw:
-	if (fstat(fw_f, &file_stat) < 0) {
-		PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
-		close(fw_f);
-		return -ENOENT;
-	}
-
-	fsize = file_stat.st_size;
-	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
-			    fw_name, (uint64_t)fsize);
-
-	fw_buf = malloc((size_t)fsize);
-	if (!fw_buf) {
-		PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
-		close(fw_f);
-		return -ENOMEM;
-	}
-	memset(fw_buf, 0, fsize);
-
-	bytes = read(fw_f, fw_buf, fsize);
-	if (bytes != fsize) {
-		PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
-				   "Just %" PRIu64 " of %" PRIu64 " bytes read",
-				   (uint64_t)bytes, (uint64_t)fsize);
-		free(fw_buf);
-		close(fw_f);
-		return -EIO;
-	}
+load_fw:
+	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
+		fw_name, fsize);
 
 	PMD_DRV_LOG(INFO, "Uploading the firmware ...");
-	nfp_nsp_load_fw(nsp, fw_buf, bytes);
+	nfp_nsp_load_fw(nsp, fw_buf, fsize);
 	PMD_DRV_LOG(INFO, "Done");
 
 	free(fw_buf);
-	close(fw_f);
-
 	return 0;
 }
 
diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
index caa9d1d4f6..347e407b1b 100644
--- a/drivers/net/qede/qede_main.c
+++ b/drivers/net/qede/qede_main.c
@@ -6,6 +6,7 @@
 
 #include <limits.h>
 #include <rte_alarm.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #include "qede_ethdev.h"
@@ -127,51 +128,40 @@ static void qed_free_stream_mem(struct ecore_dev *edev)
 #ifdef CONFIG_ECORE_BINARY_FW
 static int qed_load_firmware_data(struct ecore_dev *edev)
 {
-	int fd;
-	struct stat st;
 	const char *fw = RTE_LIBRTE_QEDE_FW;
+	void *buf;
+	size_t bufsz;
+	int ret;
 
 	if (strcmp(fw, "") == 0)
 		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
 	else
 		strcpy(qede_fw_file, fw);
 
-	fd = open(qede_fw_file, O_RDONLY);
-	if (fd < 0) {
-		DP_ERR(edev, "Can't open firmware file\n");
-		return -ENOENT;
-	}
-
-	if (fstat(fd, &st) < 0) {
-		DP_ERR(edev, "Can't stat firmware file\n");
-		close(fd);
+	if (rte_firmware_read(qede_fw_file, &buf, &bufsz) < 0) {
+		DP_ERR(edev, "Can't read firmware data: %s\n", qede_fw_file);
 		return -1;
 	}
 
-	edev->firmware = rte_zmalloc("qede_fw", st.st_size,
-				    RTE_CACHE_LINE_SIZE);
+	edev->firmware = rte_zmalloc("qede_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!edev->firmware) {
 		DP_ERR(edev, "Can't allocate memory for firmware\n");
-		close(fd);
-		return -ENOMEM;
-	}
-
-	if (read(fd, edev->firmware, st.st_size) != st.st_size) {
-		DP_ERR(edev, "Can't read firmware data\n");
-		close(fd);
-		return -1;
+		ret = -ENOMEM;
+		goto out;
 	}
 
-	edev->fw_len = st.st_size;
+	memcpy(edev->firmware, buf, bufsz);
+	edev->fw_len = bufsz;
 	if (edev->fw_len < 104) {
 		DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n",
 			  edev->fw_len);
-		close(fd);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
-
-	close(fd);
-	return 0;
+	ret = 0;
+out:
+	free(buf);
+	return ret;
 }
 #endif
 
diff --git a/lib/eal/include/rte_firmware.h b/lib/eal/include/rte_firmware.h
new file mode 100644
index 0000000000..d2a7bc8083
--- /dev/null
+++ b/lib/eal/include/rte_firmware.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifndef __RTE_FIRMWARE_H__
+#define __RTE_FIRMWARE_H__
+
+#include <sys/types.h>
+
+#include <rte_compat.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Load a firmware in a dynamically allocated buffer, dealing with compressed
+ * files if libarchive is available.
+ *
+ * @param name
+ *      Firmware filename to load.
+ * @param buf
+ *      Buffer allocated by this function. If this function succeeds, the
+ *      caller is responsible for freeing the buffer.
+ * @param bufsz
+ *      Size of the data in the buffer.
+ *
+ * @return
+ *      0 if successful.
+ *      Negative otherwise, buf and bufsize contents are invalid.
+ */
+__rte_internal
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz);
+
+#endif
diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
new file mode 100644
index 0000000000..ea66fecfe9
--- /dev/null
+++ b/lib/eal/unix/eal_firmware.c
@@ -0,0 +1,106 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifdef RTE_HAS_LIBARCHIVE
+#include <archive.h>
+#endif
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_firmware.h>
+#include <rte_log.h>
+
+static int
+firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	const size_t blocksize = 4096;
+	int ret = -1;
+	int err;
+#ifdef RTE_HAS_LIBARCHIVE
+	struct archive_entry *entry;
+	struct archive *a;
+#else
+	int fd;
+#endif
+
+	*buf = NULL;
+	*bufsz = 0;
+
+#ifdef RTE_HAS_LIBARCHIVE
+	a = archive_read_new();
+	if (a == NULL || archive_read_support_format_raw(a) != ARCHIVE_OK ||
+			archive_read_support_filter_xz(a) != ARCHIVE_OK ||
+			archive_read_open_filename(a, name, blocksize) != ARCHIVE_OK ||
+			archive_read_next_header(a, &entry) != ARCHIVE_OK)
+		goto out;
+#else
+	fd = open(name, O_RDONLY);
+	if (fd < 0)
+		goto out;
+#endif
+
+	do {
+		void *tmp;
+
+		tmp = realloc(*buf, *bufsz + blocksize);
+		if (tmp == NULL) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			break;
+		}
+		*buf = tmp;
+
+#ifdef RTE_HAS_LIBARCHIVE
+		err = archive_read_data(a, RTE_PTR_ADD(*buf, *bufsz), blocksize);
+#else
+		err = read(fd, RTE_PTR_ADD(*buf, *bufsz), blocksize);
+#endif
+		if (err < 0) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			break;
+		}
+		*bufsz += err;
+
+	} while (err != 0);
+
+	if (*buf != NULL)
+		ret = 0;
+out:
+#ifdef RTE_HAS_LIBARCHIVE
+	if (a != NULL)
+		archive_read_free(a);
+#else
+	if (fd >= 0)
+		close(fd);
+#endif
+	return ret;
+}
+
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	char path[PATH_MAX];
+	int ret;
+
+	ret = firmware_read(name, buf, bufsz);
+	if (ret < 0) {
+		snprintf(path, sizeof(path), "%s.xz", name);
+		path[PATH_MAX - 1] = '\0';
+#ifndef RTE_HAS_LIBARCHIVE
+		if (access(path, F_OK) == 0) {
+			RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",
+				path);
+		}
+#else
+		ret = firmware_read(path, buf, bufsz);
+#endif
+	}
+	return ret;
+}
diff --git a/lib/eal/unix/meson.build b/lib/eal/unix/meson.build
index dc711b4240..e3ecd3e956 100644
--- a/lib/eal/unix/meson.build
+++ b/lib/eal/unix/meson.build
@@ -5,5 +5,6 @@ sources += files(
         'eal_file.c',
         'eal_unix_memory.c',
         'eal_unix_timer.c',
+        'eal_firmware.c',
         'rte_thread.c',
 )
diff --git a/lib/eal/version.map b/lib/eal/version.map
index fe5c3dac98..020c058e5f 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -423,11 +423,13 @@ EXPERIMENTAL {
 	rte_version_release; # WINDOWS_NO_EXPORT
 	rte_version_suffix; # WINDOWS_NO_EXPORT
 	rte_version_year; # WINDOWS_NO_EXPORT
+
 };
 
 INTERNAL {
 	global:
 
+	rte_firmware_read;
 	rte_mem_lock;
 	rte_mem_map;
 	rte_mem_page_size;
diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
index 28c787c0b0..938ace92f5 100644
--- a/lib/eal/windows/eal.c
+++ b/lib/eal/windows/eal.c
@@ -21,6 +21,7 @@
 #include <eal_private.h>
 #include <rte_service_component.h>
 #include <rte_vfio.h>
+#include <rte_firmware.h>
 
 #include "eal_hugepages.h"
 #include "eal_trace.h"
@@ -463,3 +464,11 @@ rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 {
 	return -1;
 }
+
+int
+rte_firmware_read(__rte_unused const char *name,
+			__rte_unused void **buf,
+			__rte_unused size_t *bufsz)
+{
+	return -1;
+}
-- 
2.23.0

Re: [dpdk-dev] [PATCH 2/2] eal: handle compressed firmwares

[Jerin Jacob]

On Wed, Jun 2, 2021 at 3:29 PM David Marchand <david.marchand@redhat.com> wrote:
>
> Introduce an internal firmware loading helper to remove code duplication
> in our drivers and handle xz compressed firmwares by calling libarchive.
>
> This helper tries to look for .xz suffixes so that drivers are not aware
> the firmwares have been compressed.
>
> libarchive is set as an optional dependency: without libarchive, a
> runtime warning is emitted so that users know there is a compressed
> firmware.
>
> Windows implementation is left as an empty stub.
>
> Signed-off-by: David Marchand <david.marchand@redhat.com>

> + */
> +
> +#ifndef __RTE_FIRMWARE_H__
> +#define __RTE_FIRMWARE_H__
> +
> +#include <sys/types.h>
> +
> +#include <rte_compat.h>
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Load a firmware in a dynamically allocated buffer, dealing with compressed
> + * files if libarchive is available.
> + *
> + * @param name
> + *      Firmware filename to load.
> + * @param buf

Adding out to express the output useful. i.e
@param[out] buf

> + *      Buffer allocated by this function. If this function succeeds, the
> + *      caller is responsible for freeing the buffer.

I think, we can chnange to "freeing the buffer using free()" to avoid
confusion with rte_free()

> + * @param bufsz

@param[out] bufsz

> + *      Size of the data in the buffer.
> + *
> + * @return
> + *      0 if successful.
> + *      Negative otherwise, buf and bufsize contents are invalid.
> + */
> +__rte_internal
> +int
> +rte_firmware_read(const char *name, void **buf, size_t *bufsz);
> +
> +#endif
> diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
> new file mode 100644
> index 0000000000..ea66fecfe9
> --- /dev/null
> +++ b/lib/eal/unix/eal_firmware.c
> @@ -0,0 +1,106 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 Red Hat, Inc.
> + */
> +
> +#ifdef RTE_HAS_LIBARCHIVE
> +#include <archive.h>
> +#endif
> +#include <fcntl.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <unistd.h>
> +
> +#include <rte_common.h>
> +#include <rte_firmware.h>
> +#include <rte_log.h>
> +
> +static int
> +firmware_read(const char *name, void **buf, size_t *bufsz)
> +{
> +       const size_t blocksize = 4096;
> +       int ret = -1;
> +       int err;
> +#ifdef RTE_HAS_LIBARCHIVE


I think, better to have small inline functions for libarchive variant
vs normal file accessors
in the group for open, access, read etc to avoid the ifdef clutter and
manage with one ifdef.



> +       struct archive_entry *entry;
> +       struct archive *a;
> +#else
> +       int fd;
> +#endif
> +
> +       *buf = NULL;
> +       *bufsz = 0;
> +
> +#ifdef RTE_HAS_LIBARCHIVE

See above

> +       a = archive_read_new();
> +       if (a == NULL || archive_read_support_format_raw(a) != ARCHIVE_OK ||
> +                       archive_read_support_filter_xz(a) != ARCHIVE_OK ||
> +                       archive_read_open_filename(a, name, blocksize) != ARCHIVE_OK ||
> +                       archive_read_next_header(a, &entry) != ARCHIVE_OK)
> +               goto out;
> +#else
> +       fd = open(name, O_RDONLY);
> +       if (fd < 0)
> +               goto out;
> +#endif
> +
> +       do {
> +               void *tmp;
> +
> +               tmp = realloc(*buf, *bufsz + blocksize);
> +               if (tmp == NULL) {
> +                       free(*buf);
> +                       *buf = NULL;
> +                       *bufsz = 0;
> +                       break;
> +               }
> +               *buf = tmp;
> +
> +#ifdef RTE_HAS_LIBARCHIVE

See above

> +               err = archive_read_data(a, RTE_PTR_ADD(*buf, *bufsz), blocksize);
> +#else
> +               err = read(fd, RTE_PTR_ADD(*buf, *bufsz), blocksize);
> +#endif
> +               if (err < 0) {
> +                       free(*buf);
> +                       *buf = NULL;
> +                       *bufsz = 0;
> +                       break;
> +               }
> +               *bufsz += err;
> +
> +       } while (err != 0);
> +
> +       if (*buf != NULL)
> +               ret = 0;
> +out:
> +#ifdef RTE_HAS_LIBARCHIVE

See above

> +       if (a != NULL)
> +               archive_read_free(a);
> +#else
> +       if (fd >= 0)
> +               close(fd);
> +#endif
> +       return ret;
> +}
> +
> +int
> +rte_firmware_read(const char *name, void **buf, size_t *bufsz)
> +{
> +       char path[PATH_MAX];
> +       int ret;
> +
> +       ret = firmware_read(name, buf, bufsz);
> +       if (ret < 0) {
> +               snprintf(path, sizeof(path), "%s.xz", name);
> +               path[PATH_MAX - 1] = '\0';
> +#ifndef RTE_HAS_LIBARCHIVE

See above

> +               if (access(path, F_OK) == 0) {
> +                       RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",
> +                               path);
> +               }
> +#else
> +               ret = firmware_read(path, buf, bufsz);
> +#endif
>

Re: [dpdk-dev] [PATCH 2/2] eal: handle compressed firmwares

[David Marchand]

On Wed, Jun 2, 2021 at 1:13 PM Jerin Jacob <jerinjacobk@gmail.com> wrote:
> > +static int
> > +firmware_read(const char *name, void **buf, size_t *bufsz)
> > +{
> > +       const size_t blocksize = 4096;
> > +       int ret = -1;
> > +       int err;
> > +#ifdef RTE_HAS_LIBARCHIVE
>
>
> I think, better to have small inline functions for libarchive variant
> vs normal file accessors
> in the group for open, access, read etc to avoid the ifdef clutter and
> manage with one ifdef.

That may be a bit artificial, since there is no reuse of such helpers for now.
I'll have a try and see how it looks.


> > +int
> > +rte_firmware_read(const char *name, void **buf, size_t *bufsz)
> > +{
> > +       char path[PATH_MAX];
> > +       int ret;
> > +
> > +       ret = firmware_read(name, buf, bufsz);
> > +       if (ret < 0) {
> > +               snprintf(path, sizeof(path), "%s.xz", name);
> > +               path[PATH_MAX - 1] = '\0';
> > +#ifndef RTE_HAS_LIBARCHIVE
>
> See above

There is nothing to abstract here.

If you don't have libarchive, returning the .xz content to a driver is wrong.
I prefer to leave this block as is.



>
> > +               if (access(path, F_OK) == 0) {
> > +                       RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",
> > +                               path);
> > +               }
> > +#else
> > +               ret = firmware_read(path, buf, bufsz);
> > +#endif
> >
>

-- 
David Marchand

Re: [dpdk-dev] [EXT] [PATCH 2/2] eal: handle compressed firmwares

[Igor Russkikh]

> Introduce an internal firmware loading helper to remove code duplication
> in our drivers and handle xz compressed firmwares by calling libarchive.
> 
> This helper tries to look for .xz suffixes so that drivers are not aware
> the firmwares have been compressed.
> 
> libarchive is set as an optional dependency: without libarchive, a
> runtime warning is emitted so that users know there is a compressed
> firmware.
> 
> Windows implementation is left as an empty stub.
> 
> Signed-off-by: David Marchand <david.marchand@redhat.com>

for QEDE,

Reviewed-by: Igor Russkikh <irusskikh@marvell.com>

Devendra, please give it a try when possible.

Regards,
  Igor

Re: [dpdk-dev] [PATCH 2/2] eal: handle compressed firmwares

[Dmitry Kozlyuk]

2021-06-02 11:58 (UTC+0200), David Marchand:
> Introduce an internal firmware loading helper to remove code duplication
> in our drivers and handle xz compressed firmwares by calling libarchive.
> 
> This helper tries to look for .xz suffixes so that drivers are not aware
> the firmwares have been compressed.
> 
> libarchive is set as an optional dependency: without libarchive, a
> runtime warning is emitted so that users know there is a compressed
> firmware.
> 
> Windows implementation is left as an empty stub.

JFYI, it doesn't seem hard to implement as a follow-up if need be.
libarchive is available for Windows and even provides a .pc file.

[...]
> diff --git a/config/meson.build b/config/meson.build
> index 017bb2efbb..337daa2719 100644
> --- a/config/meson.build
> +++ b/config/meson.build
> @@ -166,6 +166,15 @@ if fdt_dep.found() and cc.has_header('fdt.h')
>      dpdk_extra_ldflags += '-lfdt'
>  endif
>  
> +has_libarchive = 0
> +archive_dep = cc.find_library('libarchive', required: false)
> +if archive_dep.found() and cc.has_header('archive.h')
> +    dpdk_conf.set10('RTE_HAS_LIBARCHIVE', true)
> +    has_libarchive = 1
> +    add_project_link_arguments('-larchive', language: 'c')
> +    dpdk_extra_ldflags += '-larchive'
> +endif
> +

Why not use pkg-config?
`has_libarchive` is unused.

>  libexecinfo = cc.find_library('libexecinfo', required: false)
>  if libexecinfo.found() and cc.has_header('execinfo.h')
>      add_project_link_arguments('-lexecinfo', language: 'c')
[...]
> diff --git a/lib/eal/version.map b/lib/eal/version.map
> index fe5c3dac98..020c058e5f 100644
> --- a/lib/eal/version.map
> +++ b/lib/eal/version.map
> @@ -423,11 +423,13 @@ EXPERIMENTAL {
>  	rte_version_release; # WINDOWS_NO_EXPORT
>  	rte_version_suffix; # WINDOWS_NO_EXPORT
>  	rte_version_year; # WINDOWS_NO_EXPORT
> +
>  };

Unnecessary empty line.

>  
>  INTERNAL {
>  	global:
>  
> +	rte_firmware_read;
>  	rte_mem_lock;
>  	rte_mem_map;
>  	rte_mem_page_size;

Re: [dpdk-dev] [PATCH 2/2] eal: handle compressed firmwares

[David Marchand]

On Wed, Jun 2, 2021 at 11:19 PM Dmitry Kozlyuk <dmitry.kozliuk@gmail.com> wrote:
>
> 2021-06-02 11:58 (UTC+0200), David Marchand:
> > Introduce an internal firmware loading helper to remove code duplication
> > in our drivers and handle xz compressed firmwares by calling libarchive.
> >
> > This helper tries to look for .xz suffixes so that drivers are not aware
> > the firmwares have been compressed.
> >
> > libarchive is set as an optional dependency: without libarchive, a
> > runtime warning is emitted so that users know there is a compressed
> > firmware.
> >
> > Windows implementation is left as an empty stub.
>
> JFYI, it doesn't seem hard to implement as a follow-up if need be.
> libarchive is available for Windows and even provides a .pc file.

I noticed.
But I am not sure the current API is abstracted enough since we pass a
firmware filename.

>
> [...]
> > diff --git a/config/meson.build b/config/meson.build
> > index 017bb2efbb..337daa2719 100644
> > --- a/config/meson.build
> > +++ b/config/meson.build
> > @@ -166,6 +166,15 @@ if fdt_dep.found() and cc.has_header('fdt.h')
> >      dpdk_extra_ldflags += '-lfdt'
> >  endif
> >
> > +has_libarchive = 0
> > +archive_dep = cc.find_library('libarchive', required: false)
> > +if archive_dep.found() and cc.has_header('archive.h')
> > +    dpdk_conf.set10('RTE_HAS_LIBARCHIVE', true)
> > +    has_libarchive = 1
> > +    add_project_link_arguments('-larchive', language: 'c')
> > +    dpdk_extra_ldflags += '-larchive'
> > +endif
> > +
>
> Why not use pkg-config?
> `has_libarchive` is unused.

Frankly, I just copied the logic already present in config/meson.build.
If there is better to do, please advise.


>
> >  libexecinfo = cc.find_library('libexecinfo', required: false)
> >  if libexecinfo.found() and cc.has_header('execinfo.h')
> >      add_project_link_arguments('-lexecinfo', language: 'c')
> [...]
> > diff --git a/lib/eal/version.map b/lib/eal/version.map
> > index fe5c3dac98..020c058e5f 100644
> > --- a/lib/eal/version.map
> > +++ b/lib/eal/version.map
> > @@ -423,11 +423,13 @@ EXPERIMENTAL {
> >       rte_version_release; # WINDOWS_NO_EXPORT
> >       rte_version_suffix; # WINDOWS_NO_EXPORT
> >       rte_version_year; # WINDOWS_NO_EXPORT
> > +
> >  };
>
> Unnecessary empty line.

Yep, already fixed in my local branch.
Thanks.


-- 
David Marchand

Re: [dpdk-dev] [PATCH 2/2] eal: handle compressed firmwares

[David Marchand]

On Thu, Jun 3, 2021 at 9:23 AM David Marchand <david.marchand@redhat.com> wrote:
> > > diff --git a/config/meson.build b/config/meson.build
> > > index 017bb2efbb..337daa2719 100644
> > > --- a/config/meson.build
> > > +++ b/config/meson.build
> > > @@ -166,6 +166,15 @@ if fdt_dep.found() and cc.has_header('fdt.h')
> > >      dpdk_extra_ldflags += '-lfdt'
> > >  endif
> > >
> > > +has_libarchive = 0
> > > +archive_dep = cc.find_library('libarchive', required: false)
> > > +if archive_dep.found() and cc.has_header('archive.h')
> > > +    dpdk_conf.set10('RTE_HAS_LIBARCHIVE', true)
> > > +    has_libarchive = 1
> > > +    add_project_link_arguments('-larchive', language: 'c')
> > > +    dpdk_extra_ldflags += '-larchive'
> > > +endif
> > > +
> >
> > Why not use pkg-config?
> > `has_libarchive` is unused.
>
> Frankly, I just copied the logic already present in config/meson.build.
> If there is better to do, please advise.

Ah ok, I think I understand.
Do you mean to align on libbsd?


-- 
David Marchand

Re: [dpdk-dev] [PATCH 2/2] eal: handle compressed firmwares

[Bruce Richardson]

On Thu, Jun 03, 2021 at 09:53:44AM +0200, David Marchand wrote:
> On Thu, Jun 3, 2021 at 9:23 AM David Marchand <david.marchand@redhat.com>
> wrote:
> > > > diff --git a/config/meson.build b/config/meson.build index
> > > > 017bb2efbb..337daa2719 100644 --- a/config/meson.build +++
> > > > b/config/meson.build @@ -166,6 +166,15 @@ if fdt_dep.found() and
> > > > cc.has_header('fdt.h') dpdk_extra_ldflags += '-lfdt' endif
> > > >
> > > > +has_libarchive = 0 +archive_dep = cc.find_library('libarchive',
> > > > required: false) +if archive_dep.found() and
> > > > cc.has_header('archive.h') +
> > > > dpdk_conf.set10('RTE_HAS_LIBARCHIVE', true) +    has_libarchive = 1
> > > > +    add_project_link_arguments('-larchive', language: 'c') +
> > > > dpdk_extra_ldflags += '-larchive' +endif +
> > >
> > > Why not use pkg-config?  `has_libarchive` is unused.
> >
> > Frankly, I just copied the logic already present in config/meson.build.
> > If there is better to do, please advise.
> 
> Ah ok, I think I understand.  Do you mean to align on libbsd?
> 
Yep. "dependency()" should be used for libs/packages that have a pkg-config
file, while "find_library()" is a fallback for literally just trying to
find a library that doesn't have a .pc file.


When specifying dependencies, it's best to explicitly state the lookup
method as pkg-config (only), because if not found via pkg-config, meson
will by default then attempt to use cmake (if present) to find the
requested package, and that can cause issues with cmake finding the
incorrect package version e.g. when doing 32-bit builds on 64-bit systems.
There is almost certainly a way to configure cmake not to do this, but by
limiting search to pkg-config it saves us having to document how to
configure proper library search paths for multiple tools. Setting
PKG_CONFIG_LIBDIR should be enough!

/Bruce

Re: [dpdk-dev] [EXT] [PATCH 0/2] Support compressed firmwares

[Igor Russkikh]

> Fedora 34 only provides compressed firmwares.
> 
> Introduce an internal driver helper to handle transparently compression.
> 
> I chose libarchive for decompressing as it seems widely available and
> DPDK had used it in the past.
> 
> Windows support only matters for net/ice and firmware loading was skipped
> in this driver before this series. Since I don't know if/how we want to
> load firmwares on Windows, I let an empty stub for this OS.
> 
> This series has been compile tested on Linux (I'll trust the CI for
> others OSes).
> I only tested basic init with a net/ice device (no DCF test).
> 
> So please drivers maintainers, check nothing is broken.

Hi David,

We (Marvell QED) already provide packed version of FW in linux-firmware:
https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/qed 

But thats a custom name, and zlib.

I'm just wondering if its a good solution to try transparently load .xz variant?
User may not expect that. M.b. through this api, give a driver an option to specify
archive format? Or even autodetect it from content?

Regards,
  Igor

Re: [dpdk-dev] [EXT] [PATCH 0/2] Support compressed firmwares

[David Marchand]

On Wed, Jun 2, 2021 at 12:38 PM Igor Russkikh <irusskikh@marvell.com> wrote:
> We (Marvell QED) already provide packed version of FW in linux-firmware:
> https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/qed
>
> But thats a custom name, and zlib.

Whatever binary blob is available in linux-firmware upstream repo, it
ends up compressed on Fedora 34 (and later).

# fc33
https://koji.fedoraproject.org/koji/fileinfo?rpmID=26115649&filename=/usr/lib/firmware/qed/qed_init_values-8.10.9.0.bin
https://koji.fedoraproject.org/koji/fileinfo?rpmID=26115649&filename=/usr/lib/firmware/qed/qed_init_values_zipped-8.10.10.0.bin
# fc34
https://koji.fedoraproject.org/koji/fileinfo?rpmID=26115322&filename=/usr/lib/firmware/qed/qed_init_values-8.10.9.0.bin.xz
https://koji.fedoraproject.org/koji/fileinfo?rpmID=26115322&filename=/usr/lib/firmware/qed/qed_init_values_zipped-8.10.10.0.bin.xz
# fc35
https://koji.fedoraproject.org/koji/fileinfo?rpmID=26115234&filename=/usr/lib/firmware/qed/qed_init_values-8.10.9.0.bin.xz
https://koji.fedoraproject.org/koji/fileinfo?rpmID=26115234&filename=/usr/lib/firmware/qed/qed_init_values_zipped-8.10.10.0.bin.xz

Did you try the qede pmd on fc34?


>
> I'm just wondering if its a good solution to try transparently load .xz variant?

The linux kernel (since v5.2, I think) uncompresses this transparently
without kernel drivers knowing.
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=82fd7a8142a10b8eb41313074b3859d82c0857dc


> User may not expect that. M.b. through this api, give a driver an option to specify
> archive format? Or even autodetect it from content?

User should (do ?) not care about firmwares, only drivers do.

If not doing transparently, each driver will have to implement (or ask
a common helper for) support of compressed blobs.
While this issue is common to all drivers.

As for autodetecting compression, libarchive can do this, I added only
xz support because this is the only supported compression in the Linux
kernel loader.


-- 
David Marchand

Re: [dpdk-dev] [EXT] [PATCH 0/2] Support compressed firmwares

[Igor Russkikh]

On 6/2/2021 1:05 PM, David Marchand wrote:
> On Wed, Jun 2, 2021 at 12:38 PM Igor Russkikh <irusskikh@marvell.com> wrote:
>> We (Marvell QED) already provide packed version of FW in linux-firmware:

>>
>> But thats a custom name, and zlib.
> 
> Whatever binary blob is available in linux-firmware upstream repo, it
> ends up compressed on Fedora 34 (and later).
> 
> Did you try the qede pmd on fc34?

Got it, thanks.
No, have not tried, but think qede pmd will fail now on fedora without your patch.

> The linux kernel (since v5.2, I think) uncompresses this transparently
> without kernel drivers knowing.

Was not aware of that, thanks.
In that perspective your patch fits good with the existing linux kernel behavior, agree.

Adding more people from my team.

Regards,
  Igor

[dpdk-dev] [PATCH v2 0/2] Support compressed firmwares

[David Marchand]

Fedora 34 only provides compressed firmwares.

Introduce an internal driver helper to handle transparently compression.

I chose libarchive for decompressing as it seems widely available and
DPDK had used it in the past.

Windows support only matters for net/ice and firmware loading was skipped
in this driver before this series. Since I don't know if/how we want to
load firmwares on Windows, I let an empty stub for this OS.

This series has been compile tested on Linux (I'll trust the CI for
others OSes).
I only tested basic init with a net/ice device (no DCF test).

So please drivers maintainers, check nothing is broken.


-- 
David Marchand

Changes since v1:
- address comments on patch2,


David Marchand (2):
  net/ice: factorize firmware loading
  eal: handle compressed firmwares

 .github/workflows/build.yml      |   1 +
 .travis.yml                      |   1 +
 config/meson.build               |   7 ++
 drivers/net/bnx2x/bnx2x.c        |  35 +++----
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++---------------
 drivers/net/ice/ice_ethdev.c     | 175 ++++++++++++-------------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 drivers/net/nfp/nfp_net.c        |  57 +++-------
 drivers/net/qede/qede_main.c     |  45 ++++----
 lib/eal/include/rte_firmware.h   |  32 ++++++
 lib/eal/unix/eal_firmware.c      | 149 ++++++++++++++++++++++++++
 lib/eal/unix/meson.build         |   1 +
 lib/eal/version.map              |   1 +
 lib/eal/windows/eal.c            |   9 ++
 15 files changed, 321 insertions(+), 298 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

-- 
2.23.0

[dpdk-dev] [PATCH v2 1/2] net/ice: factorize firmware loading

[David Marchand]

Both "normal" and "dcf" inits have their copy of some firmware loading
code.

The DSN query is moved in specific parts for the "normal" and "dcf" init.

A common helper ice_load_pkg is then introduced and takes an adapter
pointer as its main input.

This helper takes care of finding the right firmware file and loading
it.
The adapter active_pkg_type field is set by this helper.

The ice_access macro is removed from the osdep.h header: osdep.h should
only hosts wrappers for base driver code.

Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
 drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 4 files changed, 88 insertions(+), 179 deletions(-)

diff --git a/drivers/net/ice/base/ice_osdep.h b/drivers/net/ice/base/ice_osdep.h
index 878c5597d4..78093adb00 100644
--- a/drivers/net/ice/base/ice_osdep.h
+++ b/drivers/net/ice/base/ice_osdep.h
@@ -74,12 +74,6 @@ typedef uint64_t        s64;
 #define min(a, b) RTE_MIN(a, b)
 #define max(a, b) RTE_MAX(a, b)
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 #define FIELD_SIZEOF(t, f) RTE_SIZEOF_FIELD(t, f)
 #define ARRAY_SIZE(arr) RTE_DIM(arr)
 
diff --git a/drivers/net/ice/ice_dcf_parent.c b/drivers/net/ice/ice_dcf_parent.c
index 1d7aa8bc87..a30cfefaaf 100644
--- a/drivers/net/ice/ice_dcf_parent.c
+++ b/drivers/net/ice/ice_dcf_parent.c
@@ -298,13 +298,14 @@ static void ice_dcf_uninit_parent_hw(struct ice_hw *hw)
 }
 
 static int
-ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
+ice_dcf_load_pkg(struct ice_adapter *adapter)
 {
 	struct ice_dcf_adapter *dcf_adapter =
-			container_of(hw, struct ice_dcf_adapter, parent.hw);
+			container_of(&adapter->hw, struct ice_dcf_adapter, parent.hw);
 	struct virtchnl_pkg_info pkg_info;
 	struct dcf_virtchnl_cmd vc_cmd;
-	uint64_t dsn;
+	bool use_dsn;
+	uint64_t dsn = 0;
 
 	vc_cmd.v_op = VIRTCHNL_OP_DCF_GET_PKG_INFO;
 	vc_cmd.req_msglen = 0;
@@ -312,90 +313,11 @@ ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
 	vc_cmd.rsp_buflen = sizeof(pkg_info);
 	vc_cmd.rsp_msgbuf = (uint8_t *)&pkg_info;
 
-	if (ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd))
-		goto pkg_file_direct;
+	use_dsn = ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd) == 0;
+	if (use_dsn)
+		rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
 
-	rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_UPDATES "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_DEFAULT "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-pkg_file_direct:
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_UPDATES);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_DEFAULT);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	return -1;
-}
-
-static int
-ice_dcf_load_pkg(struct ice_hw *hw)
-{
-	char pkg_name[ICE_MAX_PKG_FILENAME_SIZE];
-	uint8_t *pkg_buf;
-	uint32_t buf_len;
-	struct stat st;
-	FILE *fp;
-	int err;
-
-	if (ice_dcf_request_pkg_name(hw, pkg_name)) {
-		PMD_INIT_LOG(ERR, "Failed to locate the package file");
-		return -ENOENT;
-	}
-
-	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_name);
-
-	err = stat(pkg_name, &st);
-	if (err) {
-		PMD_INIT_LOG(ERR, "Failed to get file status");
-		return err;
-	}
-
-	buf_len = st.st_size;
-	pkg_buf = rte_malloc(NULL, buf_len, 0);
-	if (!pkg_buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buffer of size %u for package",
-			     buf_len);
-		return -1;
-	}
-
-	fp = fopen(pkg_name, "rb");
-	if (!fp)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s", pkg_name);
-		err = -1;
-		goto ret;
-	}
-
-	err = fread(pkg_buf, buf_len, 1, fp);
-	fclose(fp);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data");
-		err = -1;
-		goto ret;
-	}
-
-	err = ice_copy_and_init_pkg(hw, pkg_buf, buf_len);
-	if (err)
-		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d", err);
-
-ret:
-	rte_free(pkg_buf);
-	return err;
+	return ice_load_pkg(adapter, use_dsn, dsn);
 }
 
 int
@@ -436,13 +358,12 @@ ice_dcf_init_parent_adapter(struct rte_eth_dev *eth_dev)
 		return err;
 	}
 
-	err = ice_dcf_load_pkg(parent_hw);
+	err = ice_dcf_load_pkg(parent_adapter);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to load package with error %d",
 			     err);
 		goto uninit_hw;
 	}
-	parent_adapter->active_pkg_type = ice_load_pkg_type(parent_hw);
 
 	parent_adapter->pf.main_vsi->idx = hw->num_vfs;
 	ice_dcf_update_pf_vsi_map(parent_hw,
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 5fd5f99b6f..05ccc41d95 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -1649,57 +1649,7 @@ ice_pf_setup(struct ice_pf *pf)
 	return 0;
 }
 
-/*
- * Extract device serial number from PCIe Configuration Space and
- * determine the pkg file path according to the DSN.
- */
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int
-ice_pkg_file_search_path(struct rte_pci_device *pci_dev, char *pkg_file)
-{
-	off_t pos;
-	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
-	uint32_t dsn_low, dsn_high;
-	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
-
-	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
-
-	if (pos) {
-		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		if (rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
-			 "ice-%08x%08x.pkg", dsn_high, dsn_low);
-	} else {
-		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
-		goto fail_dsn;
-	}
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-fail_dsn:
-	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
-		return 0;
-	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	return 0;
-}
-#endif
-
-enum ice_pkg_type
+static enum ice_pkg_type
 ice_load_pkg_type(struct ice_hw *hw)
 {
 	enum ice_pkg_type package_type;
@@ -1723,37 +1673,62 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int ice_load_pkg(struct rte_eth_dev *dev)
+#ifdef RTE_EXEC_ENV_WINDOWS
+#define ice_access _access
+#else
+#define ice_access access
+#endif
+
+int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
-	struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
+	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
 	int err;
-	uint8_t *buf;
+	uint8_t *buf = NULL;
 	int buf_len;
 	FILE *file;
 	struct stat fstat;
-	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
-	struct ice_adapter *ad =
-		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 
-	err = ice_pkg_file_search_path(pci_dev, pkg_file);
-	if (err) {
+	if (!use_dsn)
+		goto no_dsn;
+
+	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
+	snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
+		"ice-%016" PRIx64 ".pkg", dsn);
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+no_dsn:
+	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(pkg_file, 0))
+		goto load_fw;
+	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
+	if (ice_access(pkg_file, 0)) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
-		return err;
+		return -1;
 	}
 
+load_fw:
 	file = fopen(pkg_file, "rb");
 	if (!file)  {
 		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
 		return -1;
 	}
 
+	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
+
 	err = stat(pkg_file, &fstat);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		fclose(file);
-		return err;
+		goto out;
 	}
 
 	buf_len = fstat.st_size;
@@ -1762,44 +1737,33 @@ static int ice_load_pkg(struct rte_eth_dev *dev)
 	if (!buf) {
 		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
 				buf_len);
-		fclose(file);
-		return -1;
+		err = -1;
+		goto out;
 	}
 
 	err = fread(buf, buf_len, 1, file);
 	if (err != 1) {
 		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		fclose(file);
 		err = -1;
-		goto fail_exit;
+		goto out;
 	}
 
-	fclose(file);
-
 	err = ice_copy_and_init_pkg(hw, buf, buf_len);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
-		goto fail_exit;
+		goto out;
 	}
 
 	/* store the loaded pkg type info */
-	ad->active_pkg_type = ice_load_pkg_type(hw);
+	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
-	err = ice_init_hw_tbls(hw);
-	if (err) {
-		PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", err);
-		goto fail_init_tbls;
-	}
-
-	return 0;
-
-fail_init_tbls:
-	rte_free(hw->pkg_copy);
-fail_exit:
+out:
+	fclose(file);
 	rte_free(buf);
 	return err;
 }
-#endif
+
+#undef ice_access
 
 static void
 ice_base_queue_get(struct ice_pf *pf)
@@ -2030,6 +1994,12 @@ ice_dev_init(struct rte_eth_dev *dev)
 		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_vsi *vsi;
 	int ret;
+#ifndef RTE_EXEC_ENV_WINDOWS
+	off_t pos;
+	uint32_t dsn_low, dsn_high;
+	uint64_t dsn;
+	bool use_dsn;
+#endif
 
 	dev->dev_ops = &ice_eth_dev_ops;
 	dev->rx_queue_count = ice_rx_queue_count;
@@ -2081,7 +2051,30 @@ ice_dev_init(struct rte_eth_dev *dev)
 	}
 
 #ifndef RTE_EXEC_ENV_WINDOWS
-	ret = ice_load_pkg(dev);
+	use_dsn = false;
+	dsn = 0;
+	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
+	if (pos) {
+		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0 ||
+				rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
+			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
+		} else {
+			use_dsn = true;
+			dsn = (uint64_t)dsn_high << 32 | dsn_low;
+		}
+	} else {
+		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
+	}
+
+	ret = ice_load_pkg(pf->adapter, use_dsn, dsn);
+	if (ret == 0) {
+		ret = ice_init_hw_tbls(hw);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", ret);
+			rte_free(hw->pkg_copy);
+		}
+	}
+
 	if (ret) {
 		if (ad->devargs.safe_mode_support == 0) {
 			PMD_INIT_LOG(ERR, "Failed to load the DDP package,"
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index 2a8a8169d5..1997f9296f 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -531,7 +531,8 @@ struct ice_vsi_vlan_pvid_info {
 #define ICE_PF_TO_ETH_DEV(pf) \
 	(((struct ice_pf *)pf)->adapter->eth_dev)
 
-enum ice_pkg_type ice_load_pkg_type(struct ice_hw *hw);
+int
+ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn);
 struct ice_vsi *
 ice_setup_vsi(struct ice_pf *pf, enum ice_vsi_type type);
 int
-- 
2.23.0

Re: [dpdk-dev] [PATCH v2 1/2] net/ice: factorize firmware loading

[David Marchand]

On Thu, Jun 3, 2021 at 6:55 PM David Marchand <david.marchand@redhat.com> wrote:
>
> Both "normal" and "dcf" inits have their copy of some firmware loading
> code.
>
> The DSN query is moved in specific parts for the "normal" and "dcf" init.
>
> A common helper ice_load_pkg is then introduced and takes an adapter
> pointer as its main input.
>
> This helper takes care of finding the right firmware file and loading
> it.
> The adapter active_pkg_type field is set by this helper.
>
> The ice_access macro is removed from the osdep.h header: osdep.h should
> only hosts wrappers for base driver code.
>
> Signed-off-by: David Marchand <david.marchand@redhat.com>

Please, can you confirm nothing breaks for net/ice with this series?
Thanks.


-- 
David Marchand

[dpdk-dev] [PATCH v2 2/2] eal: handle compressed firmwares

[David Marchand]

Introduce an internal firmware loading helper to remove code duplication
in our drivers and handle xz compressed firmwares by calling libarchive.

This helper tries to look for .xz suffixes so that drivers are not aware
the firmwares have been compressed.

libarchive is set as an optional dependency: without libarchive, a
runtime warning is emitted so that users know there is a compressed
firmware.

Windows implementation is left as an empty stub.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
---
Changes since v1:
- used pkg-config for libarchive detection,
- updated doxygen annotations,
- added internal helpers in eal_firmware.c to enhance readability,
- dropped whitespace damage in version.map,

---
 .github/workflows/build.yml    |   1 +
 .travis.yml                    |   1 +
 config/meson.build             |   7 ++
 drivers/net/bnx2x/bnx2x.c      |  35 +++-----
 drivers/net/ice/ice_ethdev.c   |  60 +++----------
 drivers/net/nfp/nfp_net.c      |  57 +++----------
 drivers/net/qede/qede_main.c   |  45 ++++------
 lib/eal/include/rte_firmware.h |  32 +++++++
 lib/eal/unix/eal_firmware.c    | 149 +++++++++++++++++++++++++++++++++
 lib/eal/unix/meson.build       |   1 +
 lib/eal/version.map            |   1 +
 lib/eal/windows/eal.c          |   9 ++
 12 files changed, 256 insertions(+), 142 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c4d6dcdbf..7dac20ddeb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -93,6 +93,7 @@ jobs:
       run: sudo apt install -y ccache libnuma-dev python3-setuptools
         python3-wheel python3-pip python3-pyelftools ninja-build libbsd-dev
         libpcap-dev libibverbs-dev libcrypto++-dev libfdt-dev libjansson-dev
+        libarchive-dev
     - name: Install libabigail build dependencies if no cache is available
       if: env.ABI_CHECKS == 'true' && steps.libabigail-cache.outputs.cache-hit != 'true'
       run: sudo apt install -y autoconf automake libtool pkg-config libxml2-dev
diff --git a/.travis.yml b/.travis.yml
index 5b702cc9bb..23067d9e3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,7 @@ addons:
     packages: &required_packages
       - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
       - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
+      - [libarchive-dev]
 
 _aarch64_packages: &aarch64_packages
   - *required_packages
diff --git a/config/meson.build b/config/meson.build
index 017bb2efbb..c6985139b4 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -172,6 +172,13 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
     dpdk_extra_ldflags += '-lexecinfo'
 endif
 
+libarchive = dependency('libarchive', required: false, method: 'pkg-config')
+if libarchive.found()
+    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
+    add_project_link_arguments('-larchive', language: 'c')
+    dpdk_extra_ldflags += '-larchive'
+endif
+
 # check for libbsd
 libbsd = dependency('libbsd', required: false, method: 'pkg-config')
 if libbsd.found()
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
index 654878d9de..60292753e2 100644
--- a/drivers/net/bnx2x/bnx2x.c
+++ b/drivers/net/bnx2x/bnx2x.c
@@ -26,7 +26,9 @@
 #include <arpa/inet.h>
 #include <fcntl.h>
 #include <zlib.h>
+
 #include <rte_bitops.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
@@ -9655,44 +9657,33 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc)
 void bnx2x_load_firmware(struct bnx2x_softc *sc)
 {
 	const char *fwname;
-	int f;
-	struct stat st;
+	void *buf;
+	size_t bufsz;
 
 	fwname = sc->devinfo.device_id == CHIP_NUM_57711
 		? FW_NAME_57711 : FW_NAME_57810;
-	f = open(fwname, O_RDONLY);
-	if (f < 0) {
+	if (rte_firmware_read(fwname, &buf, &bufsz) != 0) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
 		return;
 	}
 
-	if (fstat(f, &st) < 0) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
-		close(f);
-		return;
-	}
-
-	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
+	sc->firmware = rte_zmalloc("bnx2x_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!sc->firmware) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
-		close(f);
-		return;
+		goto out;
 	}
 
-	if (read(f, sc->firmware, st.st_size) != st.st_size) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
-		close(f);
-		return;
-	}
-	close(f);
-
-	sc->fw_len = st.st_size;
+	sc->fw_len = bufsz;
 	if (sc->fw_len < FW_HEADER_LEN) {
 		PMD_DRV_LOG(NOTICE, sc,
 			    "Invalid fw size: %" PRIu64, sc->fw_len);
-		return;
+		goto out;
 	}
+
+	memcpy(sc->firmware, buf, sc->fw_len);
 	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
+out:
+	free(buf);
 }
 
 static void
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 05ccc41d95..6bb8c76ab3 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -10,6 +10,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include <rte_firmware.h>
 #include <rte_tailq.h>
 
 #include "base/ice_sched.h"
@@ -1673,22 +1674,14 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
 	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
 	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
+	void *buf;
+	size_t bufsz;
 	int err;
-	uint8_t *buf = NULL;
-	int buf_len;
-	FILE *file;
-	struct stat fstat;
 
 	if (!use_dsn)
 		goto no_dsn;
@@ -1698,57 +1691,31 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 		"ice-%016" PRIx64 ".pkg", dsn);
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 no_dsn:
 	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
+
 	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	if (ice_access(pkg_file, 0)) {
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
 		return -1;
 	}
 
 load_fw:
-	file = fopen(pkg_file, "rb");
-	if (!file)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
-		return -1;
-	}
-
 	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
 
-	err = stat(pkg_file, &fstat);
-	if (err) {
-		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		goto out;
-	}
-
-	buf_len = fstat.st_size;
-	buf = rte_malloc(NULL, buf_len, 0);
-
-	if (!buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
-				buf_len);
-		err = -1;
-		goto out;
-	}
-
-	err = fread(buf, buf_len, 1, file);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		err = -1;
-		goto out;
-	}
-
-	err = ice_copy_and_init_pkg(hw, buf, buf_len);
+	err = ice_copy_and_init_pkg(hw, buf, bufsz);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
 		goto out;
@@ -1758,13 +1725,10 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
 out:
-	fclose(file);
-	rte_free(buf);
+	free(buf);
 	return err;
 }
 
-#undef ice_access
-
 static void
 ice_base_queue_get(struct ice_pf *pf)
 {
diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
index 2ee88fbfc7..879da7ef59 100644
--- a/drivers/net/nfp/nfp_net.c
+++ b/drivers/net/nfp/nfp_net.c
@@ -29,6 +29,7 @@
 #include <rte_alarm.h>
 #include <rte_spinlock.h>
 #include <rte_service_component.h>
+#include <rte_firmware.h>
 
 #include "nfpcore/nfp_cpp.h"
 #include "nfpcore/nfp_nffw.h"
@@ -3366,12 +3367,10 @@ static int
 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 {
 	struct nfp_cpp *cpp = nsp->cpp;
-	int fw_f;
-	char *fw_buf;
+	void *fw_buf;
 	char fw_name[125];
 	char serial[40];
-	struct stat file_stat;
-	off_t fsize, bytes;
+	size_t fsize;
 
 	/* Looking for firmware file in order of priority */
 
@@ -3384,66 +3383,34 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 
 	snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 			serial);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Then try the PCI name */
 	snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 			dev->device.name);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Finally try the card type and media */
 	snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f < 0) {
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
 		PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 		return -ENOENT;
 	}
 
-read_fw:
-	if (fstat(fw_f, &file_stat) < 0) {
-		PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
-		close(fw_f);
-		return -ENOENT;
-	}
-
-	fsize = file_stat.st_size;
-	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
-			    fw_name, (uint64_t)fsize);
-
-	fw_buf = malloc((size_t)fsize);
-	if (!fw_buf) {
-		PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
-		close(fw_f);
-		return -ENOMEM;
-	}
-	memset(fw_buf, 0, fsize);
-
-	bytes = read(fw_f, fw_buf, fsize);
-	if (bytes != fsize) {
-		PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
-				   "Just %" PRIu64 " of %" PRIu64 " bytes read",
-				   (uint64_t)bytes, (uint64_t)fsize);
-		free(fw_buf);
-		close(fw_f);
-		return -EIO;
-	}
+load_fw:
+	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
+		fw_name, fsize);
 
 	PMD_DRV_LOG(INFO, "Uploading the firmware ...");
-	nfp_nsp_load_fw(nsp, fw_buf, bytes);
+	nfp_nsp_load_fw(nsp, fw_buf, fsize);
 	PMD_DRV_LOG(INFO, "Done");
 
 	free(fw_buf);
-	close(fw_f);
-
 	return 0;
 }
 
diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
index caa9d1d4f6..504e2c66a0 100644
--- a/drivers/net/qede/qede_main.c
+++ b/drivers/net/qede/qede_main.c
@@ -5,7 +5,9 @@
  */
 
 #include <limits.h>
+
 #include <rte_alarm.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #include "qede_ethdev.h"
@@ -127,51 +129,40 @@ static void qed_free_stream_mem(struct ecore_dev *edev)
 #ifdef CONFIG_ECORE_BINARY_FW
 static int qed_load_firmware_data(struct ecore_dev *edev)
 {
-	int fd;
-	struct stat st;
 	const char *fw = RTE_LIBRTE_QEDE_FW;
+	void *buf;
+	size_t bufsz;
+	int ret;
 
 	if (strcmp(fw, "") == 0)
 		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
 	else
 		strcpy(qede_fw_file, fw);
 
-	fd = open(qede_fw_file, O_RDONLY);
-	if (fd < 0) {
-		DP_ERR(edev, "Can't open firmware file\n");
-		return -ENOENT;
-	}
-
-	if (fstat(fd, &st) < 0) {
-		DP_ERR(edev, "Can't stat firmware file\n");
-		close(fd);
+	if (rte_firmware_read(qede_fw_file, &buf, &bufsz) < 0) {
+		DP_ERR(edev, "Can't read firmware data: %s\n", qede_fw_file);
 		return -1;
 	}
 
-	edev->firmware = rte_zmalloc("qede_fw", st.st_size,
-				    RTE_CACHE_LINE_SIZE);
+	edev->firmware = rte_zmalloc("qede_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!edev->firmware) {
 		DP_ERR(edev, "Can't allocate memory for firmware\n");
-		close(fd);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto out;
 	}
 
-	if (read(fd, edev->firmware, st.st_size) != st.st_size) {
-		DP_ERR(edev, "Can't read firmware data\n");
-		close(fd);
-		return -1;
-	}
-
-	edev->fw_len = st.st_size;
+	memcpy(edev->firmware, buf, bufsz);
+	edev->fw_len = bufsz;
 	if (edev->fw_len < 104) {
 		DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n",
 			  edev->fw_len);
-		close(fd);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
-
-	close(fd);
-	return 0;
+	ret = 0;
+out:
+	free(buf);
+	return ret;
 }
 #endif
 
diff --git a/lib/eal/include/rte_firmware.h b/lib/eal/include/rte_firmware.h
new file mode 100644
index 0000000000..3389e60ca0
--- /dev/null
+++ b/lib/eal/include/rte_firmware.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifndef __RTE_FIRMWARE_H__
+#define __RTE_FIRMWARE_H__
+
+#include <sys/types.h>
+
+#include <rte_compat.h>
+
+/**
+ * Load a firmware in a dynamically allocated buffer, dealing with compressed
+ * files if libarchive is available.
+ *
+ * @param[in] name
+ *      Firmware filename to load.
+ * @param[out] buf
+ *      Buffer allocated by this function. If this function succeeds, the
+ *      caller is responsible for calling free() on this buffer.
+ * @param[out] bufsz
+ *      Size of the data in the buffer.
+ *
+ * @return
+ *      0 if successful.
+ *      Negative otherwise, buf and bufsize contents are invalid.
+ */
+__rte_internal
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz);
+
+#endif
diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
new file mode 100644
index 0000000000..494cd5f058
--- /dev/null
+++ b/lib/eal/unix/eal_firmware.c
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifdef RTE_HAS_LIBARCHIVE
+#include <archive.h>
+#endif
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_firmware.h>
+#include <rte_log.h>
+
+#ifdef RTE_HAS_LIBARCHIVE
+
+struct firmware_read_ctx {
+	struct archive *a;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
+{
+	struct archive_entry *e;
+
+	ctx->a = archive_read_new();
+	if (ctx->a == NULL)
+		return -1;
+	if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK ||
+			archive_read_support_filter_xz(ctx->a) != ARCHIVE_OK ||
+			archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK ||
+			archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) {
+		archive_read_free(ctx->a);
+		ctx->a = NULL;
+		return -1;
+	}
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return archive_read_data(ctx->a, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	archive_read_free(ctx->a);
+	ctx->a = NULL;
+}
+
+#else /* !RTE_HAS_LIBARCHIVE */
+
+struct firmware_read_ctx {
+	int fd;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name,
+	__rte_unused size_t blocksize)
+{
+	ctx->fd = open(name, O_RDONLY);
+	if (ctx->fd < 0)
+		return -1;
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return read(ctx->fd, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	close(ctx->fd);
+	ctx->fd = -1;
+}
+
+#endif /* !RTE_HAS_LIBARCHIVE */
+
+static int
+firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	const size_t blocksize = 4096;
+	struct firmware_read_ctx ctx;
+	int ret = -1;
+	int err;
+
+	*buf = NULL;
+	*bufsz = 0;
+
+	if (firmware_open(&ctx, name, blocksize) < 0)
+		return -1;
+
+	do {
+		void *tmp;
+
+		tmp = realloc(*buf, *bufsz + blocksize);
+		if (tmp == NULL) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*buf = tmp;
+
+		err = firmware_read_block(&ctx, RTE_PTR_ADD(*buf, *bufsz), blocksize);
+		if (err < 0) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*bufsz += err;
+
+	} while (err != 0);
+
+	ret = 0;
+out:
+	firmware_close(&ctx);
+	return ret;
+}
+
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	char path[PATH_MAX];
+	int ret;
+
+	ret = firmware_read(name, buf, bufsz);
+	if (ret < 0) {
+		snprintf(path, sizeof(path), "%s.xz", name);
+		path[PATH_MAX - 1] = '\0';
+#ifndef RTE_HAS_LIBARCHIVE
+		if (access(path, F_OK) == 0) {
+			RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",
+				path);
+		}
+#else
+		ret = firmware_read(path, buf, bufsz);
+#endif
+	}
+	return ret;
+}
diff --git a/lib/eal/unix/meson.build b/lib/eal/unix/meson.build
index dc711b4240..e3ecd3e956 100644
--- a/lib/eal/unix/meson.build
+++ b/lib/eal/unix/meson.build
@@ -5,5 +5,6 @@ sources += files(
         'eal_file.c',
         'eal_unix_memory.c',
         'eal_unix_timer.c',
+        'eal_firmware.c',
         'rte_thread.c',
 )
diff --git a/lib/eal/version.map b/lib/eal/version.map
index fe5c3dac98..2df65c6903 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -428,6 +428,7 @@ EXPERIMENTAL {
 INTERNAL {
 	global:
 
+	rte_firmware_read;
 	rte_mem_lock;
 	rte_mem_map;
 	rte_mem_page_size;
diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
index 28c787c0b0..938ace92f5 100644
--- a/lib/eal/windows/eal.c
+++ b/lib/eal/windows/eal.c
@@ -21,6 +21,7 @@
 #include <eal_private.h>
 #include <rte_service_component.h>
 #include <rte_vfio.h>
+#include <rte_firmware.h>
 
 #include "eal_hugepages.h"
 #include "eal_trace.h"
@@ -463,3 +464,11 @@ rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 {
 	return -1;
 }
+
+int
+rte_firmware_read(__rte_unused const char *name,
+			__rte_unused void **buf,
+			__rte_unused size_t *bufsz)
+{
+	return -1;
+}
-- 
2.23.0

Re: [dpdk-dev] [PATCH v2 2/2] eal: handle compressed firmwares

[Dmitry Kozlyuk]

2021-06-03 18:55 (UTC+0200), David Marchand:
[...]
> diff --git a/config/meson.build b/config/meson.build
> index 017bb2efbb..c6985139b4 100644
> --- a/config/meson.build
> +++ b/config/meson.build
> @@ -172,6 +172,13 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
>      dpdk_extra_ldflags += '-lexecinfo'
>  endif
>  
> +libarchive = dependency('libarchive', required: false, method: 'pkg-config')
> +if libarchive.found()
> +    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
> +    add_project_link_arguments('-larchive', language: 'c')
> +    dpdk_extra_ldflags += '-larchive'
> +endif
> +

Suggestion:

diff --git a/config/meson.build b/config/meson.build
index c6985139b4..c3668798c1 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -175,7 +175,6 @@ endif
 libarchive = dependency('libarchive', required: false, method: 'pkg-config')
 if libarchive.found()
     dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
-    add_project_link_arguments('-larchive', language: 'c')
     dpdk_extra_ldflags += '-larchive'
 endif
 
diff --git a/lib/eal/meson.build b/lib/eal/meson.build
index 1722924f67..5a018d97d6 100644
--- a/lib/eal/meson.build
+++ b/lib/eal/meson.build
@@ -16,6 +16,7 @@ subdir(exec_env)
 subdir(arch_subdir)
 
 deps += ['kvargs']
+ext_deps += libarchive
 if not is_windows
     deps += ['telemetry']
 endif

Re: [dpdk-dev] [PATCH v2 2/2] eal: handle compressed firmwares

[David Marchand]

On Fri, Jun 4, 2021 at 12:29 AM Dmitry Kozlyuk <dmitry.kozliuk@gmail.com> wrote:
>
> 2021-06-03 18:55 (UTC+0200), David Marchand:
> [...]
> > diff --git a/config/meson.build b/config/meson.build
> > index 017bb2efbb..c6985139b4 100644
> > --- a/config/meson.build
> > +++ b/config/meson.build
> > @@ -172,6 +172,13 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
> >      dpdk_extra_ldflags += '-lexecinfo'
> >  endif
> >
> > +libarchive = dependency('libarchive', required: false, method: 'pkg-config')
> > +if libarchive.found()
> > +    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
> > +    add_project_link_arguments('-larchive', language: 'c')
> > +    dpdk_extra_ldflags += '-larchive'
> > +endif
> > +
>
> Suggestion:
>
> diff --git a/config/meson.build b/config/meson.build
> index c6985139b4..c3668798c1 100644
> --- a/config/meson.build
> +++ b/config/meson.build
> @@ -175,7 +175,6 @@ endif
>  libarchive = dependency('libarchive', required: false, method: 'pkg-config')
>  if libarchive.found()
>      dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
> -    add_project_link_arguments('-larchive', language: 'c')
>      dpdk_extra_ldflags += '-larchive'
>  endif
>
> diff --git a/lib/eal/meson.build b/lib/eal/meson.build
> index 1722924f67..5a018d97d6 100644
> --- a/lib/eal/meson.build
> +++ b/lib/eal/meson.build
> @@ -16,6 +16,7 @@ subdir(exec_env)
>  subdir(arch_subdir)
>
>  deps += ['kvargs']
> +ext_deps += libarchive
>  if not is_windows
>      deps += ['telemetry']
>  endif
>

I had tried something close when preparing v2 (only keeping
RTE_HAS_LIBARCHIVE in config/meson.build and putting extra_ldflags and
ext_deps in lib/eal/unix/meson.build) but both my try and your
suggestion break static compilation for the helloworld example.


$ ./devtools/test-meson-builds.sh -vv
...
## Building helloworld
gmake: Entering directory
'/home/dmarchan/builds/build-x86-generic/install/usr/local/share/dpdk/examples/helloworld'
rm -f build/helloworld build/helloworld-static build/helloworld-shared
test -d build && rmdir -p build || true
cc  -I/home/dmarchan/intel-ipsec-mb/install/include -O3
-I/home/dmarchan/builds/build-x86-generic/install/usr/local/include
-include rte_config.h -march=nehalem -I/usr/usr/include
-I/opt/isa-l/include  -DALLOW_EXPERIMENTAL_API main.c -o
build/helloworld-shared  -L/home/dmarchan/intel-ipsec-mb/install/lib
-Wl,--as-needed
-L/home/dmarchan/builds/build-x86-generic/install/usr/local/lib
-lrte_node -lrte_graph -lrte_bpf -lrte_flow_classify -lrte_pipeline
-lrte_table -lrte_port -lrte_fib -lrte_ipsec -lrte_vhost -lrte_stack
-lrte_security -lrte_sched -lrte_reorder -lrte_rib -lrte_regexdev
-lrte_rawdev -lrte_pdump -lrte_power -lrte_member -lrte_lpm
-lrte_latencystats -lrte_kni -lrte_jobstats -lrte_ip_frag -lrte_gso
-lrte_gro -lrte_eventdev -lrte_efd -lrte_distributor -lrte_cryptodev
-lrte_compressdev -lrte_cfgfile -lrte_bitratestats -lrte_bbdev
-lrte_acl -lrte_timer -lrte_hash -lrte_metrics -lrte_cmdline -lrte_pci
-lrte_ethdev -lrte_meter -lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu
-lrte_ring -lrte_eal -lrte_telemetry -lrte_kvargs -lbsd
ln -sf helloworld-shared build/helloworld
cc  -I/home/dmarchan/intel-ipsec-mb/install/include -O3
-I/home/dmarchan/builds/build-x86-generic/install/usr/local/include
-include rte_config.h -march=nehalem -I/usr/usr/include
-I/opt/isa-l/include  -DALLOW_EXPERIMENTAL_API main.c -o
build/helloworld-static  -L/home/dmarchan/intel-ipsec-mb/install/lib
-Wl,--whole-archive
-L/home/dmarchan/builds/build-x86-generic/install/usr/local/lib
-l:librte_common_cpt.a -l:librte_common_dpaax.a
-l:librte_common_iavf.a -l:librte_common_octeontx.a
-l:librte_common_octeontx2.a -l:librte_bus_dpaa.a
-l:librte_bus_fslmc.a -l:librte_bus_ifpga.a -l:librte_bus_pci.a
-l:librte_bus_vdev.a -l:librte_bus_vmbus.a -l:librte_common_cnxk.a
-l:librte_common_mlx5.a -l:librte_common_qat.a
-l:librte_common_sfc_efx.a -l:librte_mempool_bucket.a
-l:librte_mempool_cnxk.a -l:librte_mempool_dpaa.a
-l:librte_mempool_dpaa2.a -l:librte_mempool_octeontx.a
-l:librte_mempool_octeontx2.a -l:librte_mempool_ring.a
-l:librte_mempool_stack.a -l:librte_net_af_packet.a
-l:librte_net_af_xdp.a -l:librte_net_ark.a -l:librte_net_atlantic.a
-l:librte_net_avp.a -l:librte_net_axgbe.a -l:librte_net_bnx2x.a
-l:librte_net_bnxt.a -l:librte_net_bond.a -l:librte_net_cxgbe.a
-l:librte_net_dpaa.a -l:librte_net_dpaa2.a -l:librte_net_e1000.a
-l:librte_net_ena.a -l:librte_net_enetc.a -l:librte_net_enic.a
-l:librte_net_failsafe.a -l:librte_net_fm10k.a -l:librte_net_hinic.a
-l:librte_net_hns3.a -l:librte_net_i40e.a -l:librte_net_iavf.a
-l:librte_net_ice.a -l:librte_net_igc.a -l:librte_net_ionic.a
-l:librte_net_ipn3ke.a -l:librte_net_ixgbe.a -l:librte_net_kni.a
-l:librte_net_liquidio.a -l:librte_net_memif.a -l:librte_net_mlx4.a
-l:librte_net_mlx5.a -l:librte_net_netvsc.a -l:librte_net_nfp.a
-l:librte_net_null.a -l:librte_net_octeontx.a
-l:librte_net_octeontx2.a -l:librte_net_octeontx_ep.a
-l:librte_net_pcap.a -l:librte_net_pfe.a -l:librte_net_qede.a
-l:librte_net_ring.a -l:librte_net_sfc.a -l:librte_net_softnic.a
-l:librte_net_szedata2.a -l:librte_net_tap.a -l:librte_net_thunderx.a
-l:librte_net_txgbe.a -l:librte_net_vdev_netvsc.a
-l:librte_net_vhost.a -l:librte_net_virtio.a -l:librte_net_vmxnet3.a
-l:librte_raw_dpaa2_cmdif.a -l:librte_raw_dpaa2_qdma.a
-l:librte_raw_ifpga.a -l:librte_raw_ioat.a -l:librte_raw_ntb.a
-l:librte_raw_octeontx2_dma.a -l:librte_raw_octeontx2_ep.a
-l:librte_raw_skeleton.a -l:librte_crypto_aesni_gcm.a
-l:librte_crypto_aesni_mb.a -l:librte_crypto_bcmfs.a
-l:librte_crypto_caam_jr.a -l:librte_crypto_ccp.a
-l:librte_crypto_dpaa_sec.a -l:librte_crypto_dpaa2_sec.a
-l:librte_crypto_kasumi.a -l:librte_crypto_nitrox.a
-l:librte_crypto_null.a -l:librte_crypto_octeontx.a
-l:librte_crypto_octeontx2.a -l:librte_crypto_openssl.a
-l:librte_crypto_scheduler.a -l:librte_crypto_snow3g.a
-l:librte_crypto_virtio.a -l:librte_crypto_zuc.a
-l:librte_compress_isal.a -l:librte_compress_mlx5.a
-l:librte_compress_octeontx.a -l:librte_compress_zlib.a
-l:librte_regex_mlx5.a -l:librte_regex_octeontx2.a
-l:librte_vdpa_ifc.a -l:librte_vdpa_mlx5.a -l:librte_event_cnxk.a
-l:librte_event_dlb2.a -l:librte_event_dpaa.a -l:librte_event_dpaa2.a
-l:librte_event_dsw.a -l:librte_event_octeontx2.a
-l:librte_event_opdl.a -l:librte_event_skeleton.a -l:librte_event_sw.a
-l:librte_event_octeontx.a -l:librte_baseband_acc100.a
-l:librte_baseband_fpga_5gnr_fec.a -l:librte_baseband_fpga_lte_fec.a
-l:librte_baseband_null.a -l:librte_baseband_turbo_sw.a
-l:librte_node.a -l:librte_graph.a -l:librte_bpf.a
-l:librte_flow_classify.a -l:librte_pipeline.a -l:librte_table.a
-l:librte_port.a -l:librte_fib.a -l:librte_ipsec.a -l:librte_vhost.a
-l:librte_stack.a -l:librte_security.a -l:librte_sched.a
-l:librte_reorder.a -l:librte_rib.a -l:librte_regexdev.a
-l:librte_rawdev.a -l:librte_pdump.a -l:librte_power.a
-l:librte_member.a -l:librte_lpm.a -l:librte_latencystats.a
-l:librte_kni.a -l:librte_jobstats.a -l:librte_ip_frag.a
-l:librte_gso.a -l:librte_gro.a -l:librte_eventdev.a -l:librte_efd.a
-l:librte_distributor.a -l:librte_cryptodev.a -l:librte_compressdev.a
-l:librte_cfgfile.a -l:librte_bitratestats.a -l:librte_bbdev.a
-l:librte_acl.a -l:librte_timer.a -l:librte_hash.a -l:librte_metrics.a
-l:librte_cmdline.a -l:librte_pci.a -l:librte_ethdev.a
-l:librte_meter.a -l:librte_net.a -l:librte_mbuf.a -l:librte_mempool.a
-l:librte_rcu.a -l:librte_ring.a -l:librte_eal.a -l:librte_telemetry.a
-l:librte_kvargs.a -Wl,--no-whole-archive -Wl,--export-dynamic
-lIPSec_MB /usr/lib/gcc/x86_64-redhat-linux/10/../../../../lib64/librt.so
-Wl,--as-needed -lrte_node -lrte_graph -lrte_bpf -lrte_flow_classify
-lrte_pipeline -lrte_table -lrte_port -lrte_fib -lrte_ipsec
-lrte_vhost -lrte_stack -lrte_security -lrte_sched -lrte_reorder
-lrte_rib -lrte_regexdev -lrte_rawdev -lrte_pdump -lrte_power
-lrte_member -lrte_lpm -lrte_latencystats -lrte_kni -lrte_jobstats
-lrte_ip_frag -lrte_gso -lrte_gro -lrte_eventdev -lrte_efd
-lrte_distributor -lrte_cryptodev -lrte_compressdev -lrte_cfgfile
-lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer -lrte_hash
-lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev -lrte_meter
-lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring -lrte_eal
-lrte_telemetry -lrte_kvargs -pthread -lm -ldl -lnuma -lfdt -larchive
-lbsd -larchive -lacl -llzma -lzstd -llz4 -lbz2 -lz -lxml2
-L/usr/usr/lib64 -lmlx5 -lpthread -L/usr/usr/lib64 -lpthread -libverbs
-lpthread -lcrypto -lz -ldl -pthread -lbpf -lz -lmlx4 -lpthread
-L/usr/usr/lib64 -libverbs -lpthread -lpcap -Wl,-R/usr/lib64 -lsze2
-L/opt/isa-l/lib -lisal -lelf -lz -ljansson
/usr/bin/ld: cannot find -lacl
/usr/bin/ld: cannot find -llz4
/usr/bin/ld: cannot find -lbz2
collect2: error: ld returned 1 exit status
gmake: *** [Makefile:43: build/helloworld-static] Error 1
gmake: Leaving directory
'/home/dmarchan/builds/build-x86-generic/install/usr/local/share/dpdk/examples/helloworld'



If it helps, here is what pkg-config reports:

$ PKG_CONFIG_PATH=/opt/isa-l/lib/pkgconfig:$HOME/builds/build-x86-generic/install/usr/local/lib/pkgconfig
pkg-config --static --libs libarchive
-larchive -lcrypto -lacl -llzma -lzstd -llz4 -lbz2 -lz -lxml2

$ ls /lib*/lib{acl,lz4,bz2}*
/lib64/libacl.so.1  /lib64/libacl.so.1.1.2253  /lib64/libbz2.so.1
/lib64/libbz2.so.1.0.8  /lib64/liblz4.so.1  /lib64/liblz4.so.1.9.1


$ PKG_CONFIG_PATH=/opt/isa-l/lib/pkgconfig:$HOME/builds/build-x86-generic/install/usr/local/lib/pkgconfig
pkg-config --libs libdpdk
-Wl,--as-needed -L/usr/local/lib -lrte_node -lrte_graph -lrte_bpf
-lrte_flow_classify -lrte_pipeline -lrte_table -lrte_port -lrte_fib
-lrte_ipsec -lrte_vhost -lrte_stack -lrte_security -lrte_sched
-lrte_reorder -lrte_rib -lrte_regexdev -lrte_rawdev -lrte_pdump
-lrte_power -lrte_member -lrte_lpm -lrte_latencystats -lrte_kni
-lrte_jobstats -lrte_ip_frag -lrte_gso -lrte_gro -lrte_eventdev
-lrte_efd -lrte_distributor -lrte_cryptodev -lrte_compressdev
-lrte_cfgfile -lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer
-lrte_hash -lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev
-lrte_meter -lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring
-lrte_eal -lrte_telemetry -lrte_kvargs -lbsd
[dmarchan@wsfd-netdev66 dpdk]$
PKG_CONFIG_PATH=/opt/isa-l/lib/pkgconfig:$HOME/builds/build-x86-generic/install/usr/local/lib/pkgconfig
pkg-config --static --libs libdpdk
-Wl,--whole-archive -L/usr/local/lib -l:librte_common_cpt.a
-l:librte_common_dpaax.a -l:librte_common_iavf.a
-l:librte_common_octeontx.a -l:librte_common_octeontx2.a
-l:librte_bus_dpaa.a -l:librte_bus_fslmc.a -l:librte_bus_ifpga.a
-l:librte_bus_pci.a -l:librte_bus_vdev.a -l:librte_bus_vmbus.a
-l:librte_common_cnxk.a -l:librte_common_mlx5.a -l:librte_common_qat.a
-l:librte_common_sfc_efx.a -l:librte_mempool_bucket.a
-l:librte_mempool_cnxk.a -l:librte_mempool_dpaa.a
-l:librte_mempool_dpaa2.a -l:librte_mempool_octeontx.a
-l:librte_mempool_octeontx2.a -l:librte_mempool_ring.a
-l:librte_mempool_stack.a -l:librte_net_af_packet.a
-l:librte_net_af_xdp.a -l:librte_net_ark.a -l:librte_net_atlantic.a
-l:librte_net_avp.a -l:librte_net_axgbe.a -l:librte_net_bnx2x.a
-l:librte_net_bnxt.a -l:librte_net_bond.a -l:librte_net_cxgbe.a
-l:librte_net_dpaa.a -l:librte_net_dpaa2.a -l:librte_net_e1000.a
-l:librte_net_ena.a -l:librte_net_enetc.a -l:librte_net_enic.a
-l:librte_net_failsafe.a -l:librte_net_fm10k.a -l:librte_net_hinic.a
-l:librte_net_hns3.a -l:librte_net_i40e.a -l:librte_net_iavf.a
-l:librte_net_ice.a -l:librte_net_igc.a -l:librte_net_ionic.a
-l:librte_net_ipn3ke.a -l:librte_net_ixgbe.a -l:librte_net_kni.a
-l:librte_net_liquidio.a -l:librte_net_memif.a -l:librte_net_mlx4.a
-l:librte_net_mlx5.a -l:librte_net_netvsc.a -l:librte_net_nfp.a
-l:librte_net_null.a -l:librte_net_octeontx.a
-l:librte_net_octeontx2.a -l:librte_net_octeontx_ep.a
-l:librte_net_pcap.a -l:librte_net_pfe.a -l:librte_net_qede.a
-l:librte_net_ring.a -l:librte_net_sfc.a -l:librte_net_softnic.a
-l:librte_net_szedata2.a -l:librte_net_tap.a -l:librte_net_thunderx.a
-l:librte_net_txgbe.a -l:librte_net_vdev_netvsc.a
-l:librte_net_vhost.a -l:librte_net_virtio.a -l:librte_net_vmxnet3.a
-l:librte_raw_dpaa2_cmdif.a -l:librte_raw_dpaa2_qdma.a
-l:librte_raw_ifpga.a -l:librte_raw_ioat.a -l:librte_raw_ntb.a
-l:librte_raw_octeontx2_dma.a -l:librte_raw_octeontx2_ep.a
-l:librte_raw_skeleton.a -l:librte_crypto_aesni_gcm.a
-l:librte_crypto_aesni_mb.a -l:librte_crypto_bcmfs.a
-l:librte_crypto_caam_jr.a -l:librte_crypto_ccp.a
-l:librte_crypto_dpaa_sec.a -l:librte_crypto_dpaa2_sec.a
-l:librte_crypto_kasumi.a -l:librte_crypto_nitrox.a
-l:librte_crypto_null.a -l:librte_crypto_octeontx.a
-l:librte_crypto_octeontx2.a -l:librte_crypto_openssl.a
-l:librte_crypto_scheduler.a -l:librte_crypto_snow3g.a
-l:librte_crypto_virtio.a -l:librte_crypto_zuc.a
-l:librte_compress_isal.a -l:librte_compress_mlx5.a
-l:librte_compress_octeontx.a -l:librte_compress_zlib.a
-l:librte_regex_mlx5.a -l:librte_regex_octeontx2.a
-l:librte_vdpa_ifc.a -l:librte_vdpa_mlx5.a -l:librte_event_cnxk.a
-l:librte_event_dlb2.a -l:librte_event_dpaa.a -l:librte_event_dpaa2.a
-l:librte_event_dsw.a -l:librte_event_octeontx2.a
-l:librte_event_opdl.a -l:librte_event_skeleton.a -l:librte_event_sw.a
-l:librte_event_octeontx.a -l:librte_baseband_acc100.a
-l:librte_baseband_fpga_5gnr_fec.a -l:librte_baseband_fpga_lte_fec.a
-l:librte_baseband_null.a -l:librte_baseband_turbo_sw.a
-l:librte_node.a -l:librte_graph.a -l:librte_bpf.a
-l:librte_flow_classify.a -l:librte_pipeline.a -l:librte_table.a
-l:librte_port.a -l:librte_fib.a -l:librte_ipsec.a -l:librte_vhost.a
-l:librte_stack.a -l:librte_security.a -l:librte_sched.a
-l:librte_reorder.a -l:librte_rib.a -l:librte_regexdev.a
-l:librte_rawdev.a -l:librte_pdump.a -l:librte_power.a
-l:librte_member.a -l:librte_lpm.a -l:librte_latencystats.a
-l:librte_kni.a -l:librte_jobstats.a -l:librte_ip_frag.a
-l:librte_gso.a -l:librte_gro.a -l:librte_eventdev.a -l:librte_efd.a
-l:librte_distributor.a -l:librte_cryptodev.a -l:librte_compressdev.a
-l:librte_cfgfile.a -l:librte_bitratestats.a -l:librte_bbdev.a
-l:librte_acl.a -l:librte_timer.a -l:librte_hash.a -l:librte_metrics.a
-l:librte_cmdline.a -l:librte_pci.a -l:librte_ethdev.a
-l:librte_meter.a -l:librte_net.a -l:librte_mbuf.a -l:librte_mempool.a
-l:librte_rcu.a -l:librte_ring.a -l:librte_eal.a -l:librte_telemetry.a
-l:librte_kvargs.a -Wl,--no-whole-archive -Wl,--export-dynamic
-lIPSec_MB /usr/lib/gcc/x86_64-redhat-linux/10/../../../../lib64/librt.so
-Wl,--as-needed -lrte_node -lrte_graph -lrte_bpf -lrte_flow_classify
-lrte_pipeline -lrte_table -lrte_port -lrte_fib -lrte_ipsec
-lrte_vhost -lrte_stack -lrte_security -lrte_sched -lrte_reorder
-lrte_rib -lrte_regexdev -lrte_rawdev -lrte_pdump -lrte_power
-lrte_member -lrte_lpm -lrte_latencystats -lrte_kni -lrte_jobstats
-lrte_ip_frag -lrte_gso -lrte_gro -lrte_eventdev -lrte_efd
-lrte_distributor -lrte_cryptodev -lrte_compressdev -lrte_cfgfile
-lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer -lrte_hash
-lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev -lrte_meter
-lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring -lrte_eal
-lrte_telemetry -lrte_kvargs -pthread -lm -ldl -lnuma -lfdt -larchive
-lbsd -larchive -lacl -llzma -lzstd -llz4 -lbz2 -lz -lxml2
-L/usr/usr/lib64 -lmlx5 -lpthread -L/usr/usr/lib64 -lpthread -libverbs
-lpthread -lcrypto -lz -ldl -pthread -lbpf -lz -lmlx4 -lpthread
-L/usr/usr/lib64 -libverbs -lpthread -lpcap -Wl,-R/usr/lib64 -lsze2
-L/opt/isa-l/lib -lisal -lelf -lz -ljansson


-- 
David Marchand

Re: [dpdk-dev] [PATCH v2 2/2] eal: handle compressed firmwares

[Dmitry Kozlyuk]

2021-06-04 09:27 (UTC+0200), David Marchand:
> On Fri, Jun 4, 2021 at 12:29 AM Dmitry Kozlyuk <dmitry.kozliuk@gmail.com> wrote:
> >
> > 2021-06-03 18:55 (UTC+0200), David Marchand:
> > [...]  
> > > diff --git a/config/meson.build b/config/meson.build
> > > index 017bb2efbb..c6985139b4 100644
> > > --- a/config/meson.build
> > > +++ b/config/meson.build
> > > @@ -172,6 +172,13 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
> > >      dpdk_extra_ldflags += '-lexecinfo'
> > >  endif
> > >
> > > +libarchive = dependency('libarchive', required: false, method: 'pkg-config')
> > > +if libarchive.found()
> > > +    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
> > > +    add_project_link_arguments('-larchive', language: 'c')
> > > +    dpdk_extra_ldflags += '-larchive'
> > > +endif
> > > +  
> >
> > Suggestion:
> >
> > diff --git a/config/meson.build b/config/meson.build
> > index c6985139b4..c3668798c1 100644
> > --- a/config/meson.build
> > +++ b/config/meson.build
> > @@ -175,7 +175,6 @@ endif
> >  libarchive = dependency('libarchive', required: false, method: 'pkg-config')
> >  if libarchive.found()
> >      dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
> > -    add_project_link_arguments('-larchive', language: 'c')
> >      dpdk_extra_ldflags += '-larchive'
> >  endif
> >
> > diff --git a/lib/eal/meson.build b/lib/eal/meson.build
> > index 1722924f67..5a018d97d6 100644
> > --- a/lib/eal/meson.build
> > +++ b/lib/eal/meson.build
> > @@ -16,6 +16,7 @@ subdir(exec_env)
> >  subdir(arch_subdir)
> >
> >  deps += ['kvargs']
> > +ext_deps += libarchive
> >  if not is_windows
> >      deps += ['telemetry']
> >  endif
> >  
> 
> I had tried something close when preparing v2 (only keeping
> RTE_HAS_LIBARCHIVE in config/meson.build and putting extra_ldflags and
> ext_deps in lib/eal/unix/meson.build) but both my try and your
> suggestion break static compilation for the helloworld example.
> 
> 
> $ ./devtools/test-meson-builds.sh -vv
> ...
> ## Building helloworld
[snip]

Thanks for details.
Indeed, libarchive.pc lists all libraries present at libarchive build time
in Libs.private, despite that libarchive static linkage doesn't require them.
We'll have to go your way, sorry for misdirection.
Maybe it's worth a comment.

From libarchive README:

	I've attempted to minimize static link pollution. If you don't
	explicitly invoke a particular feature (such as support for a
	particular compression or format), it won't get pulled in to
	statically-linked programs. In particular, if you don't explicitly
	enable a particular compression or decompression support, you won't
	need to link against the corresponding compression or decompression
	libraries. This also reduces the size of statically-linked binaries
	in environments where that matters.

Re: [dpdk-dev] [PATCH v2 2/2] eal: handle compressed firmwares

[David Marchand]

On Fri, Jun 4, 2021 at 11:40 PM Dmitry Kozlyuk <dmitry.kozliuk@gmail.com> wrote:
> > I had tried something close when preparing v2 (only keeping
> > RTE_HAS_LIBARCHIVE in config/meson.build and putting extra_ldflags and
> > ext_deps in lib/eal/unix/meson.build) but both my try and your
> > suggestion break static compilation for the helloworld example.
> >
> >
> > $ ./devtools/test-meson-builds.sh -vv
> > ...
> > ## Building helloworld
> [snip]
>
> Thanks for details.
> Indeed, libarchive.pc lists all libraries present at libarchive build time
> in Libs.private, despite that libarchive static linkage doesn't require them.
> We'll have to go your way, sorry for misdirection.
> Maybe it's worth a comment.
>
> From libarchive README:
>
>         I've attempted to minimize static link pollution. If you don't
>         explicitly invoke a particular feature (such as support for a
>         particular compression or format), it won't get pulled in to
>         statically-linked programs. In particular, if you don't explicitly
>         enable a particular compression or decompression support, you won't
>         need to link against the corresponding compression or decompression
>         libraries. This also reduces the size of statically-linked binaries
>         in environments where that matters.

Yes, I can explain this in v3, but I'll wait a bit to see if issues
with this series are discovered.
Thanks Dmitry.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v2 0/2] Support compressed firmwares

[David Marchand]

On Thu, Jun 3, 2021 at 6:55 PM David Marchand <david.marchand@redhat.com> wrote:
>
> Fedora 34 only provides compressed firmwares.
>
> Introduce an internal driver helper to handle transparently compression.
>
> I chose libarchive for decompressing as it seems widely available and
> DPDK had used it in the past.
>
> Windows support only matters for net/ice and firmware loading was skipped
> in this driver before this series. Since I don't know if/how we want to
> load firmwares on Windows, I let an empty stub for this OS.
>
> This series has been compile tested on Linux (I'll trust the CI for
> others OSes).
> I only tested basic init with a net/ice device (no DCF test).
>
> So please drivers maintainers, check nothing is broken.

- Trying with direct mails to maintainers of drivers impacted by this.


- I also want opinions on backporting this change to stable releases.

F34 just got released and will live until 2022-05-17.
F34 got released with 20.11 dpdk.
Afaiu, F35 and maybe F36 will also use dpdk 20.11 (since the dpdk
package is aligned on the version used by the ovs package on Fedora).
Fedora will need this in downstream 20.11 for net/bnx2x, net/ice,
net/nfp and net/qede drivers to work with firmwares in /lib/firmware.


Not sure about Debian/Ubuntu.

Christian, Luca, Timothy?


Thanks!

-- 
David Marchand

[dpdk-dev] [PATCH v3 0/2] Support compressed firmwares

[David Marchand]

Fedora 34 only provides compressed firmwares.

Introduce an internal driver helper to handle transparently compression.

I chose libarchive for decompressing as it seems widely available and
DPDK had used it in the past.

Windows support only matters for net/ice and firmware loading was skipped
in this driver before this series. Since I don't know if/how we want to
load firmwares on Windows, I let an empty stub for this OS.

This series has been compile tested on Linux (I'll trust the CI for
others OSes).
I only tested basic init with a net/ice device (no DCF test).

So please drivers maintainers, check nothing is broken.


-- 
David Marchand

Changes since v2:
- update comment on libarchive link dependency,

Changes since v1:
- address comments on patch2,


David Marchand (2):
  net/ice: factorize firmware loading
  eal: handle compressed firmwares

 .github/workflows/build.yml      |   1 +
 .travis.yml                      |   1 +
 config/meson.build               |  10 ++
 drivers/net/bnx2x/bnx2x.c        |  35 +++----
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++---------------
 drivers/net/ice/ice_ethdev.c     | 175 ++++++++++++-------------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 drivers/net/nfp/nfp_net.c        |  57 +++-------
 drivers/net/qede/qede_main.c     |  45 ++++----
 lib/eal/include/rte_firmware.h   |  32 ++++++
 lib/eal/unix/eal_firmware.c      | 149 ++++++++++++++++++++++++++
 lib/eal/unix/meson.build         |   1 +
 lib/eal/version.map              |   1 +
 lib/eal/windows/eal.c            |   9 ++
 15 files changed, 324 insertions(+), 298 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

-- 
2.23.0

[dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[David Marchand]

Both "normal" and "dcf" inits have their copy of some firmware loading
code.

The DSN query is moved in specific parts for the "normal" and "dcf" init.

A common helper ice_load_pkg is then introduced and takes an adapter
pointer as its main input.

This helper takes care of finding the right firmware file and loading
it.
The adapter active_pkg_type field is set by this helper.

The ice_access macro is removed from the osdep.h header: osdep.h should
only hosts wrappers for base driver code.

Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
 drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 4 files changed, 88 insertions(+), 179 deletions(-)

diff --git a/drivers/net/ice/base/ice_osdep.h b/drivers/net/ice/base/ice_osdep.h
index 878c5597d4..78093adb00 100644
--- a/drivers/net/ice/base/ice_osdep.h
+++ b/drivers/net/ice/base/ice_osdep.h
@@ -74,12 +74,6 @@ typedef uint64_t        s64;
 #define min(a, b) RTE_MIN(a, b)
 #define max(a, b) RTE_MAX(a, b)
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 #define FIELD_SIZEOF(t, f) RTE_SIZEOF_FIELD(t, f)
 #define ARRAY_SIZE(arr) RTE_DIM(arr)
 
diff --git a/drivers/net/ice/ice_dcf_parent.c b/drivers/net/ice/ice_dcf_parent.c
index 19420a0f58..318239abdc 100644
--- a/drivers/net/ice/ice_dcf_parent.c
+++ b/drivers/net/ice/ice_dcf_parent.c
@@ -298,13 +298,14 @@ static void ice_dcf_uninit_parent_hw(struct ice_hw *hw)
 }
 
 static int
-ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
+ice_dcf_load_pkg(struct ice_adapter *adapter)
 {
 	struct ice_dcf_adapter *dcf_adapter =
-			container_of(hw, struct ice_dcf_adapter, parent.hw);
+			container_of(&adapter->hw, struct ice_dcf_adapter, parent.hw);
 	struct virtchnl_pkg_info pkg_info;
 	struct dcf_virtchnl_cmd vc_cmd;
-	uint64_t dsn;
+	bool use_dsn;
+	uint64_t dsn = 0;
 
 	vc_cmd.v_op = VIRTCHNL_OP_DCF_GET_PKG_INFO;
 	vc_cmd.req_msglen = 0;
@@ -312,90 +313,11 @@ ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
 	vc_cmd.rsp_buflen = sizeof(pkg_info);
 	vc_cmd.rsp_msgbuf = (uint8_t *)&pkg_info;
 
-	if (ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd))
-		goto pkg_file_direct;
+	use_dsn = ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd) == 0;
+	if (use_dsn)
+		rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
 
-	rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_UPDATES "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_DEFAULT "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-pkg_file_direct:
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_UPDATES);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_DEFAULT);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	return -1;
-}
-
-static int
-ice_dcf_load_pkg(struct ice_hw *hw)
-{
-	char pkg_name[ICE_MAX_PKG_FILENAME_SIZE];
-	uint8_t *pkg_buf;
-	uint32_t buf_len;
-	struct stat st;
-	FILE *fp;
-	int err;
-
-	if (ice_dcf_request_pkg_name(hw, pkg_name)) {
-		PMD_INIT_LOG(ERR, "Failed to locate the package file");
-		return -ENOENT;
-	}
-
-	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_name);
-
-	err = stat(pkg_name, &st);
-	if (err) {
-		PMD_INIT_LOG(ERR, "Failed to get file status");
-		return err;
-	}
-
-	buf_len = st.st_size;
-	pkg_buf = rte_malloc(NULL, buf_len, 0);
-	if (!pkg_buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buffer of size %u for package",
-			     buf_len);
-		return -1;
-	}
-
-	fp = fopen(pkg_name, "rb");
-	if (!fp)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s", pkg_name);
-		err = -1;
-		goto ret;
-	}
-
-	err = fread(pkg_buf, buf_len, 1, fp);
-	fclose(fp);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data");
-		err = -1;
-		goto ret;
-	}
-
-	err = ice_copy_and_init_pkg(hw, pkg_buf, buf_len);
-	if (err)
-		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d", err);
-
-ret:
-	rte_free(pkg_buf);
-	return err;
+	return ice_load_pkg(adapter, use_dsn, dsn);
 }
 
 int
@@ -435,13 +357,12 @@ ice_dcf_init_parent_adapter(struct rte_eth_dev *eth_dev)
 		return err;
 	}
 
-	err = ice_dcf_load_pkg(parent_hw);
+	err = ice_dcf_load_pkg(parent_adapter);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to load package with error %d",
 			     err);
 		goto uninit_hw;
 	}
-	parent_adapter->active_pkg_type = ice_load_pkg_type(parent_hw);
 
 	parent_adapter->pf.main_vsi->idx = hw->num_vfs;
 	ice_dcf_update_pf_vsi_map(parent_hw,
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 09e38590e5..d180b73c32 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -1649,57 +1649,7 @@ ice_pf_setup(struct ice_pf *pf)
 	return 0;
 }
 
-/*
- * Extract device serial number from PCIe Configuration Space and
- * determine the pkg file path according to the DSN.
- */
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int
-ice_pkg_file_search_path(struct rte_pci_device *pci_dev, char *pkg_file)
-{
-	off_t pos;
-	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
-	uint32_t dsn_low, dsn_high;
-	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
-
-	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
-
-	if (pos) {
-		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		if (rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
-			 "ice-%08x%08x.pkg", dsn_high, dsn_low);
-	} else {
-		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
-		goto fail_dsn;
-	}
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-fail_dsn:
-	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
-		return 0;
-	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	return 0;
-}
-#endif
-
-enum ice_pkg_type
+static enum ice_pkg_type
 ice_load_pkg_type(struct ice_hw *hw)
 {
 	enum ice_pkg_type package_type;
@@ -1723,37 +1673,62 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int ice_load_pkg(struct rte_eth_dev *dev)
+#ifdef RTE_EXEC_ENV_WINDOWS
+#define ice_access _access
+#else
+#define ice_access access
+#endif
+
+int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
-	struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
+	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
 	int err;
-	uint8_t *buf;
+	uint8_t *buf = NULL;
 	int buf_len;
 	FILE *file;
 	struct stat fstat;
-	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
-	struct ice_adapter *ad =
-		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 
-	err = ice_pkg_file_search_path(pci_dev, pkg_file);
-	if (err) {
+	if (!use_dsn)
+		goto no_dsn;
+
+	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
+	snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
+		"ice-%016" PRIx64 ".pkg", dsn);
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+no_dsn:
+	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(pkg_file, 0))
+		goto load_fw;
+	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
+	if (ice_access(pkg_file, 0)) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
-		return err;
+		return -1;
 	}
 
+load_fw:
 	file = fopen(pkg_file, "rb");
 	if (!file)  {
 		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
 		return -1;
 	}
 
+	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
+
 	err = stat(pkg_file, &fstat);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		fclose(file);
-		return err;
+		goto out;
 	}
 
 	buf_len = fstat.st_size;
@@ -1762,44 +1737,33 @@ static int ice_load_pkg(struct rte_eth_dev *dev)
 	if (!buf) {
 		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
 				buf_len);
-		fclose(file);
-		return -1;
+		err = -1;
+		goto out;
 	}
 
 	err = fread(buf, buf_len, 1, file);
 	if (err != 1) {
 		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		fclose(file);
 		err = -1;
-		goto fail_exit;
+		goto out;
 	}
 
-	fclose(file);
-
 	err = ice_copy_and_init_pkg(hw, buf, buf_len);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
-		goto fail_exit;
+		goto out;
 	}
 
 	/* store the loaded pkg type info */
-	ad->active_pkg_type = ice_load_pkg_type(hw);
+	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
-	err = ice_init_hw_tbls(hw);
-	if (err) {
-		PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", err);
-		goto fail_init_tbls;
-	}
-
-	return 0;
-
-fail_init_tbls:
-	rte_free(hw->pkg_copy);
-fail_exit:
+out:
+	fclose(file);
 	rte_free(buf);
 	return err;
 }
-#endif
+
+#undef ice_access
 
 static void
 ice_base_queue_get(struct ice_pf *pf)
@@ -2030,6 +1994,12 @@ ice_dev_init(struct rte_eth_dev *dev)
 		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_vsi *vsi;
 	int ret;
+#ifndef RTE_EXEC_ENV_WINDOWS
+	off_t pos;
+	uint32_t dsn_low, dsn_high;
+	uint64_t dsn;
+	bool use_dsn;
+#endif
 
 	dev->dev_ops = &ice_eth_dev_ops;
 	dev->rx_queue_count = ice_rx_queue_count;
@@ -2080,7 +2050,30 @@ ice_dev_init(struct rte_eth_dev *dev)
 	}
 
 #ifndef RTE_EXEC_ENV_WINDOWS
-	ret = ice_load_pkg(dev);
+	use_dsn = false;
+	dsn = 0;
+	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
+	if (pos) {
+		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0 ||
+				rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
+			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
+		} else {
+			use_dsn = true;
+			dsn = (uint64_t)dsn_high << 32 | dsn_low;
+		}
+	} else {
+		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
+	}
+
+	ret = ice_load_pkg(pf->adapter, use_dsn, dsn);
+	if (ret == 0) {
+		ret = ice_init_hw_tbls(hw);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", ret);
+			rte_free(hw->pkg_copy);
+		}
+	}
+
 	if (ret) {
 		if (ad->devargs.safe_mode_support == 0) {
 			PMD_INIT_LOG(ERR, "Failed to load the DDP package,"
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index 7f3c26fb6f..edafdf168b 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -534,7 +534,8 @@ struct ice_vsi_vlan_pvid_info {
 #define ICE_PF_TO_ETH_DEV(pf) \
 	(((struct ice_pf *)pf)->adapter->eth_dev)
 
-enum ice_pkg_type ice_load_pkg_type(struct ice_hw *hw);
+int
+ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn);
 struct ice_vsi *
 ice_setup_vsi(struct ice_pf *pf, enum ice_vsi_type type);
 int
-- 
2.23.0

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[Wang, Haiyue]

Hi David,

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> Sent: Tuesday, June 29, 2021 16:07
> To: dev@dpdk.org
> Cc: Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> Subject: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> 
> Both "normal" and "dcf" inits have their copy of some firmware loading
> code.
> 
> The DSN query is moved in specific parts for the "normal" and "dcf" init.
> 
> A common helper ice_load_pkg is then introduced and takes an adapter
> pointer as its main input.
> 
> This helper takes care of finding the right firmware file and loading
> it.
> The adapter active_pkg_type field is set by this helper.
> 
> The ice_access macro is removed from the osdep.h header: osdep.h should
> only hosts wrappers for base driver code.
> 
> Signed-off-by: David Marchand <david.marchand@redhat.com>
> ---
>  drivers/net/ice/base/ice_osdep.h |   6 --
>  drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
>  drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
>  drivers/net/ice/ice_ethdev.h     |   3 +-
>  4 files changed, 88 insertions(+), 179 deletions(-)
> 


> +	if (!use_dsn)
> +		goto no_dsn;
> +
> +	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
> +	snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
> +		"ice-%016" PRIx64 ".pkg", dsn);
> +	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
> +		ICE_MAX_PKG_FILENAME_SIZE);
> +	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> +		goto load_fw;
> +
> +	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
> +		ICE_MAX_PKG_FILENAME_SIZE);
> +	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> +		goto load_fw;
> +
> +no_dsn:
> +	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
> +	if (!ice_access(pkg_file, 0))
> +		goto load_fw;
> +	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
> +	if (ice_access(pkg_file, 0)) {
>  		PMD_INIT_LOG(ERR, "failed to search file path\n");
> -		return err;
> +		return -1;
>  	}
> 
> +load_fw:
>  	file = fopen(pkg_file, "rb");
>  	if (!file)  {
>  		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
>  		return -1;
>  	}
> 

I'm wondering what's full name for ice firmware in F34, has any *.xz
postfix ? If so, the search method will also needs to be updated, since
we will check each file can be accessed: 

#define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
#define ICE_PKG_FILE_UPDATES "/lib/firmware/updates/intel/ice/ddp/ice.pkg"


> 2.23.0

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[Wang, Haiyue]

Hi David & Qi,

> -----Original Message-----
> From: Wang, Haiyue
> Sent: Monday, July 5, 2021 09:43
> To: David Marchand <david.marchand@redhat.com>; dev@dpdk.org
> Cc: Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> 
> Hi David,
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> > Sent: Tuesday, June 29, 2021 16:07
> > To: dev@dpdk.org
> > Cc: Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> > Subject: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> >
> > Both "normal" and "dcf" inits have their copy of some firmware loading
> > code.
> >
> > The DSN query is moved in specific parts for the "normal" and "dcf" init.
> >
> > A common helper ice_load_pkg is then introduced and takes an adapter
> > pointer as its main input.
> >
> > This helper takes care of finding the right firmware file and loading
> > it.
> > The adapter active_pkg_type field is set by this helper.
> >
> > The ice_access macro is removed from the osdep.h header: osdep.h should
> > only hosts wrappers for base driver code.
> >
> > Signed-off-by: David Marchand <david.marchand@redhat.com>
> > ---
> >  drivers/net/ice/base/ice_osdep.h |   6 --
> >  drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
> >  drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
> >  drivers/net/ice/ice_ethdev.h     |   3 +-
> >  4 files changed, 88 insertions(+), 179 deletions(-)
> >
> 
> 
> > +	if (!use_dsn)
> > +		goto no_dsn;
> > +
> > +	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
> > +	snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
> > +		"ice-%016" PRIx64 ".pkg", dsn);
> > +	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
> > +		ICE_MAX_PKG_FILENAME_SIZE);
> > +	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> > +		goto load_fw;
> > +
> > +	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
> > +		ICE_MAX_PKG_FILENAME_SIZE);
> > +	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> > +		goto load_fw;
> > +
> > +no_dsn:
> > +	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
> > +	if (!ice_access(pkg_file, 0))
> > +		goto load_fw;
> > +	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
> > +	if (ice_access(pkg_file, 0)) {
> >  		PMD_INIT_LOG(ERR, "failed to search file path\n");
> > -		return err;
> > +		return -1;
> >  	}
> >
> > +load_fw:
> >  	file = fopen(pkg_file, "rb");
> >  	if (!file)  {
> >  		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
> >  		return -1;
> >  	}
> >
> 
> I'm wondering what's full name for ice firmware in F34, has any *.xz
> postfix ? If so, the search method will also needs to be updated, since
> we will check each file can be accessed:
> 
> #define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
> #define ICE_PKG_FILE_UPDATES "/lib/firmware/updates/intel/ice/ddp/ice.pkg"
> 

We need to update the default/comms search method:

I try the F34:

tree /lib/firmware/intel/ice/
/lib/firmware/intel/ice/
├── ddp
│   ├── ice-1.3.16.0.pkg.xz
│   └── ice.pkg.xz -> ice-1.3.16.0.pkg.xz
└── ddp-comms
    └── ice_comms-1.3.20.0.pkg.xz

It matches the upstream version:

https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/intel/ice

d---------	ddp-comms	50	logstatsplain
d---------	ddp	44	logstatsplain


> 
> > 2.23.0

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[David Marchand]

On Mon, Jul 5, 2021 at 3:43 AM Wang, Haiyue <haiyue.wang@intel.com> wrote:
>
> Hi David,
>
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> > Sent: Tuesday, June 29, 2021 16:07
> > To: dev@dpdk.org
> > Cc: Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> > Subject: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> >
> > Both "normal" and "dcf" inits have their copy of some firmware loading
> > code.
> >
> > The DSN query is moved in specific parts for the "normal" and "dcf" init.
> >
> > A common helper ice_load_pkg is then introduced and takes an adapter
> > pointer as its main input.
> >
> > This helper takes care of finding the right firmware file and loading
> > it.
> > The adapter active_pkg_type field is set by this helper.
> >
> > The ice_access macro is removed from the osdep.h header: osdep.h should
> > only hosts wrappers for base driver code.
> >
> > Signed-off-by: David Marchand <david.marchand@redhat.com>
> > ---
> >  drivers/net/ice/base/ice_osdep.h |   6 --
> >  drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
> >  drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
> >  drivers/net/ice/ice_ethdev.h     |   3 +-
> >  4 files changed, 88 insertions(+), 179 deletions(-)
> >
>
>
> > +     if (!use_dsn)
> > +             goto no_dsn;
> > +
> > +     memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
> > +     snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
> > +             "ice-%016" PRIx64 ".pkg", dsn);
> > +     strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
> > +             ICE_MAX_PKG_FILENAME_SIZE);
> > +     if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> > +             goto load_fw;
> > +
> > +     strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
> > +             ICE_MAX_PKG_FILENAME_SIZE);
> > +     if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> > +             goto load_fw;
> > +
> > +no_dsn:
> > +     strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
> > +     if (!ice_access(pkg_file, 0))
> > +             goto load_fw;
> > +     strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
> > +     if (ice_access(pkg_file, 0)) {
> >               PMD_INIT_LOG(ERR, "failed to search file path\n");
> > -             return err;
> > +             return -1;
> >       }
> >
> > +load_fw:
> >       file = fopen(pkg_file, "rb");
> >       if (!file)  {
> >               PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
> >               return -1;
> >       }
> >
>
> I'm wondering what's full name for ice firmware in F34, has any *.xz
> postfix ? If so, the search method will also needs to be updated, since
> we will check each file can be accessed:
>
> #define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
> #define ICE_PKG_FILE_UPDATES "/lib/firmware/updates/intel/ice/ddp/ice.pkg"

This first patch is a preparation to have a single helper to
select/open the firmware.
I don't get what you mean.

Is there a change in behavior with this patch?


-- 
David Marchand

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[Wang, Haiyue]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Monday, July 5, 2021 15:08
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> 
> On Mon, Jul 5, 2021 at 3:43 AM Wang, Haiyue <haiyue.wang@intel.com> wrote:
> >
> > Hi David,
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> > > Sent: Tuesday, June 29, 2021 16:07
> > > To: dev@dpdk.org
> > > Cc: Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> > > Subject: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> > >
> > > Both "normal" and "dcf" inits have their copy of some firmware loading
> > > code.
> > >
> > > The DSN query is moved in specific parts for the "normal" and "dcf" init.
> > >
> > > A common helper ice_load_pkg is then introduced and takes an adapter
> > > pointer as its main input.
> > >
> > > This helper takes care of finding the right firmware file and loading
> > > it.
> > > The adapter active_pkg_type field is set by this helper.
> > >
> > > The ice_access macro is removed from the osdep.h header: osdep.h should
> > > only hosts wrappers for base driver code.
> > >
> > > Signed-off-by: David Marchand <david.marchand@redhat.com>
> > > ---
> > >  drivers/net/ice/base/ice_osdep.h |   6 --
> > >  drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
> > >  drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
> > >  drivers/net/ice/ice_ethdev.h     |   3 +-
> > >  4 files changed, 88 insertions(+), 179 deletions(-)
> > >
> >
> >
> > > +     if (!use_dsn)
> > > +             goto no_dsn;
> > > +
> > > +     memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
> > > +     snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
> > > +             "ice-%016" PRIx64 ".pkg", dsn);
> > > +     strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
> > > +             ICE_MAX_PKG_FILENAME_SIZE);
> > > +     if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> > > +             goto load_fw;
> > > +
> > > +     strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
> > > +             ICE_MAX_PKG_FILENAME_SIZE);
> > > +     if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> > > +             goto load_fw;
> > > +
> > > +no_dsn:
> > > +     strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
> > > +     if (!ice_access(pkg_file, 0))
> > > +             goto load_fw;
> > > +     strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
> > > +     if (ice_access(pkg_file, 0)) {
> > >               PMD_INIT_LOG(ERR, "failed to search file path\n");
> > > -             return err;
> > > +             return -1;
> > >       }
> > >
> > > +load_fw:
> > >       file = fopen(pkg_file, "rb");
> > >       if (!file)  {
> > >               PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
> > >               return -1;
> > >       }
> > >
> >
> > I'm wondering what's full name for ice firmware in F34, has any *.xz
> > postfix ? If so, the search method will also needs to be updated, since
> > we will check each file can be accessed:
> >
> > #define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
> > #define ICE_PKG_FILE_UPDATES "/lib/firmware/updates/intel/ice/ddp/ice.pkg"
> 
> This first patch is a preparation to have a single helper to
> select/open the firmware.
> I don't get what you mean.

Since the pkg file has the *.xz, now the search method doesn't work. You fix
the read only. ;-)

> 
> Is there a change in behavior with this patch?
> 
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[David Marchand]

On Mon, Jul 5, 2021 at 10:02 AM Wang, Haiyue <haiyue.wang@intel.com> wrote:
> > > I'm wondering what's full name for ice firmware in F34, has any *.xz
> > > postfix ? If so, the search method will also needs to be updated, since
> > > we will check each file can be accessed:
> > >
> > > #define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
> > > #define ICE_PKG_FILE_UPDATES "/lib/firmware/updates/intel/ice/ddp/ice.pkg"
> >
> > This first patch is a preparation to have a single helper to
> > select/open the firmware.
> > I don't get what you mean.
>
> Since the pkg file has the *.xz, now the search method doesn't work. You fix
> the read only. ;-)

This patch fixes nothing wrt to F34.
It simply prepares for the next patch, because I did not want to fix
in two places with the same change.
I can squash everything in a single patch if you prefer.


In the end, I end up with the same question, but for the whole series:
> > Is there a change in behavior with this patch?


-- 
David Marchand

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[Zhang, Qi Z]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Monday, July 5, 2021 4:34 PM
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z
> <qi.z.zhang@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> 
> On Mon, Jul 5, 2021 at 10:02 AM Wang, Haiyue <haiyue.wang@intel.com>
> wrote:
> > > > I'm wondering what's full name for ice firmware in F34, has any
> > > > *.xz postfix ? If so, the search method will also needs to be
> > > > updated, since we will check each file can be accessed:
> > > >
> > > > #define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
> > > > #define ICE_PKG_FILE_UPDATES
> "/lib/firmware/updates/intel/ice/ddp/ice.pkg"
> > >
> > > This first patch is a preparation to have a single helper to
> > > select/open the firmware.
> > > I don't get what you mean.
> >
> > Since the pkg file has the *.xz, now the search method doesn't work.
> > You fix the read only. ;-)
> 
> This patch fixes nothing wrt to F34.
> It simply prepares for the next patch, because I did not want to fix in two places
> with the same change.

I agree with this , the patch just do code refactory, it looks good to have , no matter if we need this for F34 or not.

Haiyue, did you see any problem on F34 with both patches be applied or just this one be applied?



> I can squash everything in a single patch if you prefer.


> 
> 
> In the end, I end up with the same question, but for the whole series:
> > > Is there a change in behavior with this patch?
> 
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[Wang, Haiyue]

> -----Original Message-----
> From: Zhang, Qi Z <qi.z.zhang@intel.com>
> Sent: Monday, July 5, 2021 18:00
> To: David Marchand <david.marchand@redhat.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Yang, Qiming <qiming.yang@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> 
> 
> 
> > -----Original Message-----
> > From: David Marchand <david.marchand@redhat.com>
> > Sent: Monday, July 5, 2021 4:34 PM
> > To: Wang, Haiyue <haiyue.wang@intel.com>
> > Cc: dev@dpdk.org; Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z
> > <qi.z.zhang@intel.com>
> > Subject: Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> >
> > On Mon, Jul 5, 2021 at 10:02 AM Wang, Haiyue <haiyue.wang@intel.com>
> > wrote:
> > > > > I'm wondering what's full name for ice firmware in F34, has any
> > > > > *.xz postfix ? If so, the search method will also needs to be
> > > > > updated, since we will check each file can be accessed:
> > > > >
> > > > > #define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
> > > > > #define ICE_PKG_FILE_UPDATES
> > "/lib/firmware/updates/intel/ice/ddp/ice.pkg"
> > > >
> > > > This first patch is a preparation to have a single helper to
> > > > select/open the firmware.
> > > > I don't get what you mean.
> > >
> > > Since the pkg file has the *.xz, now the search method doesn't work.
> > > You fix the read only. ;-)
> >
> > This patch fixes nothing wrt to F34.
> > It simply prepares for the next patch, because I did not want to fix in two places
> > with the same change.
> 
> I agree with this , the patch just do code refactory, it looks good to have , no matter if we need
> this for F34 or not.
> 
> Haiyue, did you see any problem on F34 with both patches be applied or just this one be applied?

Just install a VM to see the firmware name, not test yet. ;-)

> 
> 
> 
> > I can squash everything in a single patch if you prefer.
> 
> 
> >
> >
> > In the end, I end up with the same question, but for the whole series:
> > > > Is there a change in behavior with this patch?
> >
> >
> > --
> > David Marchand
> 

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[Wang, Haiyue]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Monday, July 5, 2021 16:34
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> 
> On Mon, Jul 5, 2021 at 10:02 AM Wang, Haiyue <haiyue.wang@intel.com> wrote:
> > > > I'm wondering what's full name for ice firmware in F34, has any *.xz
> > > > postfix ? If so, the search method will also needs to be updated, since
> > > > we will check each file can be accessed:
> > > >
> > > > #define ICE_PKG_FILE_DEFAULT "/lib/firmware/intel/ice/ddp/ice.pkg"
> > > > #define ICE_PKG_FILE_UPDATES "/lib/firmware/updates/intel/ice/ddp/ice.pkg"
> > >
> > > This first patch is a preparation to have a single helper to
> > > select/open the firmware.
> > > I don't get what you mean.
> >
> > Since the pkg file has the *.xz, now the search method doesn't work. You fix
> > the read only. ;-)
> 
> This patch fixes nothing wrt to F34.
> It simply prepares for the next patch, because I did not want to fix
> in two places with the same change.
> I can squash everything in a single patch if you prefer.
> 
> 
> In the end, I end up with the same question, but for the whole series:
> > > Is there a change in behavior with this patch?

Sorry, after applied the whole patch to read the change, I get the real
design.


Looks like one patch is clear, since I'm lost in Patch#1, I thought it used
'*.pkg' to get the pkg name.... after applied, I read the source code directly,
yes, it’s clear in ice_load_pkg: the rte_firmware_read has handled "*.xz" right.

static int
ice_dcf_load_pkg(struct ice_adapter *adapter)
{
	struct ice_dcf_adapter *dcf_adapter =
			container_of(&adapter->hw, struct ice_dcf_adapter, parent.hw);
	struct virtchnl_pkg_info pkg_info;
	struct dcf_virtchnl_cmd vc_cmd;
	bool use_dsn;
	uint64_t dsn = 0;

	vc_cmd.v_op = VIRTCHNL_OP_DCF_GET_PKG_INFO;
	vc_cmd.req_msglen = 0;
	vc_cmd.req_msg = NULL;
	vc_cmd.rsp_buflen = sizeof(pkg_info);
	vc_cmd.rsp_msgbuf = (uint8_t *)&pkg_info;

	use_dsn = ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd) == 0;
	if (use_dsn)
		rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));

	return ice_load_pkg(adapter, use_dsn, dsn);
}

> 
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[Wang, Haiyue]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> Sent: Tuesday, June 29, 2021 16:07
> To: dev@dpdk.org
> Cc: Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> Subject: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> 
> Both "normal" and "dcf" inits have their copy of some firmware loading
> code.
> 
> The DSN query is moved in specific parts for the "normal" and "dcf" init.
> 
> A common helper ice_load_pkg is then introduced and takes an adapter
> pointer as its main input.
> 
> This helper takes care of finding the right firmware file and loading
> it.
> The adapter active_pkg_type field is set by this helper.
> 
> The ice_access macro is removed from the osdep.h header: osdep.h should
> only hosts wrappers for base driver code.
> 
> Signed-off-by: David Marchand <david.marchand@redhat.com>
> ---
>  drivers/net/ice/base/ice_osdep.h |   6 --
>  drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
>  drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
>  drivers/net/ice/ice_ethdev.h     |   3 +-
>  4 files changed, 88 insertions(+), 179 deletions(-)
> 

Squash into a single patch seems great help to understand the compress firmware
context. ;-)

But it's OK as two patches.

Acked-by: Haiyue Wang <haiyue.wang@intel.com>

> --
> 2.23.0

Re: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading

[David Marchand]

On Mon, Jul 5, 2021 at 3:18 PM Wang, Haiyue <haiyue.wang@intel.com> wrote:
>
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> > Sent: Tuesday, June 29, 2021 16:07
> > To: dev@dpdk.org
> > Cc: Yang, Qiming <qiming.yang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>
> > Subject: [dpdk-dev] [PATCH v3 1/2] net/ice: factorize firmware loading
> >
> > Both "normal" and "dcf" inits have their copy of some firmware loading
> > code.
> >
> > The DSN query is moved in specific parts for the "normal" and "dcf" init.
> >
> > A common helper ice_load_pkg is then introduced and takes an adapter
> > pointer as its main input.
> >
> > This helper takes care of finding the right firmware file and loading
> > it.
> > The adapter active_pkg_type field is set by this helper.
> >
> > The ice_access macro is removed from the osdep.h header: osdep.h should
> > only hosts wrappers for base driver code.
> >
> > Signed-off-by: David Marchand <david.marchand@redhat.com>
> > ---
> >  drivers/net/ice/base/ice_osdep.h |   6 --
> >  drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
> >  drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
> >  drivers/net/ice/ice_ethdev.h     |   3 +-
> >  4 files changed, 88 insertions(+), 179 deletions(-)
> >
>
> Squash into a single patch seems great help to understand the compress firmware
> context. ;-)
>
> But it's OK as two patches.
>
> Acked-by: Haiyue Wang <haiyue.wang@intel.com>

Cool, thanks for the review and test!


-- 
David Marchand

[dpdk-dev] [PATCH v3 2/2] eal: handle compressed firmwares

[David Marchand]

Introduce an internal firmware loading helper to remove code duplication
in our drivers and handle xz compressed firmwares by calling libarchive.

This helper tries to look for .xz suffixes so that drivers are not aware
the firmwares have been compressed.

libarchive is set as an optional dependency: without libarchive, a
runtime warning is emitted so that users know there is a compressed
firmware.

Windows implementation is left as an empty stub.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
---
Changes since v2:
- added a comment on libarchive link dependency,

Changes since v1:
- used pkg-config for libarchive detection,
- updated doxygen annotations,
- added internal helpers in eal_firmware.c to enhance readability,
- dropped whitespace damage in version.map,

---
 .github/workflows/build.yml    |   1 +
 .travis.yml                    |   1 +
 config/meson.build             |  10 +++
 drivers/net/bnx2x/bnx2x.c      |  35 +++-----
 drivers/net/ice/ice_ethdev.c   |  60 +++----------
 drivers/net/nfp/nfp_net.c      |  57 +++----------
 drivers/net/qede/qede_main.c   |  45 ++++------
 lib/eal/include/rte_firmware.h |  32 +++++++
 lib/eal/unix/eal_firmware.c    | 149 +++++++++++++++++++++++++++++++++
 lib/eal/unix/meson.build       |   1 +
 lib/eal/version.map            |   1 +
 lib/eal/windows/eal.c          |   9 ++
 12 files changed, 259 insertions(+), 142 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c4d6dcdbf..7dac20ddeb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -93,6 +93,7 @@ jobs:
       run: sudo apt install -y ccache libnuma-dev python3-setuptools
         python3-wheel python3-pip python3-pyelftools ninja-build libbsd-dev
         libpcap-dev libibverbs-dev libcrypto++-dev libfdt-dev libjansson-dev
+        libarchive-dev
     - name: Install libabigail build dependencies if no cache is available
       if: env.ABI_CHECKS == 'true' && steps.libabigail-cache.outputs.cache-hit != 'true'
       run: sudo apt install -y autoconf automake libtool pkg-config libxml2-dev
diff --git a/.travis.yml b/.travis.yml
index 5b702cc9bb..23067d9e3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,7 @@ addons:
     packages: &required_packages
       - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
       - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
+      - [libarchive-dev]
 
 _aarch64_packages: &aarch64_packages
   - *required_packages
diff --git a/config/meson.build b/config/meson.build
index 017bb2efbb..639aa74e12 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -172,6 +172,16 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
     dpdk_extra_ldflags += '-lexecinfo'
 endif
 
+libarchive = dependency('libarchive', required: false, method: 'pkg-config')
+if libarchive.found()
+    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
+    # Push libarchive link dependency at the project level to support
+    # statically linking dpdk apps. Details at:
+    # https://inbox.dpdk.org/dev/20210605004024.660267a1@sovereign/
+    add_project_link_arguments('-larchive', language: 'c')
+    dpdk_extra_ldflags += '-larchive'
+endif
+
 # check for libbsd
 libbsd = dependency('libbsd', required: false, method: 'pkg-config')
 if libbsd.found()
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
index 654878d9de..60292753e2 100644
--- a/drivers/net/bnx2x/bnx2x.c
+++ b/drivers/net/bnx2x/bnx2x.c
@@ -26,7 +26,9 @@
 #include <arpa/inet.h>
 #include <fcntl.h>
 #include <zlib.h>
+
 #include <rte_bitops.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
@@ -9655,44 +9657,33 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc)
 void bnx2x_load_firmware(struct bnx2x_softc *sc)
 {
 	const char *fwname;
-	int f;
-	struct stat st;
+	void *buf;
+	size_t bufsz;
 
 	fwname = sc->devinfo.device_id == CHIP_NUM_57711
 		? FW_NAME_57711 : FW_NAME_57810;
-	f = open(fwname, O_RDONLY);
-	if (f < 0) {
+	if (rte_firmware_read(fwname, &buf, &bufsz) != 0) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
 		return;
 	}
 
-	if (fstat(f, &st) < 0) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
-		close(f);
-		return;
-	}
-
-	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
+	sc->firmware = rte_zmalloc("bnx2x_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!sc->firmware) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
-		close(f);
-		return;
+		goto out;
 	}
 
-	if (read(f, sc->firmware, st.st_size) != st.st_size) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
-		close(f);
-		return;
-	}
-	close(f);
-
-	sc->fw_len = st.st_size;
+	sc->fw_len = bufsz;
 	if (sc->fw_len < FW_HEADER_LEN) {
 		PMD_DRV_LOG(NOTICE, sc,
 			    "Invalid fw size: %" PRIu64, sc->fw_len);
-		return;
+		goto out;
 	}
+
+	memcpy(sc->firmware, buf, sc->fw_len);
 	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
+out:
+	free(buf);
 }
 
 static void
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d180b73c32..06da1bbd94 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -10,6 +10,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include <rte_firmware.h>
 #include <rte_tailq.h>
 
 #include "base/ice_sched.h"
@@ -1673,22 +1674,14 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
 	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
 	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
+	void *buf;
+	size_t bufsz;
 	int err;
-	uint8_t *buf = NULL;
-	int buf_len;
-	FILE *file;
-	struct stat fstat;
 
 	if (!use_dsn)
 		goto no_dsn;
@@ -1698,57 +1691,31 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 		"ice-%016" PRIx64 ".pkg", dsn);
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 no_dsn:
 	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
+
 	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	if (ice_access(pkg_file, 0)) {
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
 		return -1;
 	}
 
 load_fw:
-	file = fopen(pkg_file, "rb");
-	if (!file)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
-		return -1;
-	}
-
 	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
 
-	err = stat(pkg_file, &fstat);
-	if (err) {
-		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		goto out;
-	}
-
-	buf_len = fstat.st_size;
-	buf = rte_malloc(NULL, buf_len, 0);
-
-	if (!buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
-				buf_len);
-		err = -1;
-		goto out;
-	}
-
-	err = fread(buf, buf_len, 1, file);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		err = -1;
-		goto out;
-	}
-
-	err = ice_copy_and_init_pkg(hw, buf, buf_len);
+	err = ice_copy_and_init_pkg(hw, buf, bufsz);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
 		goto out;
@@ -1758,13 +1725,10 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
 out:
-	fclose(file);
-	rte_free(buf);
+	free(buf);
 	return err;
 }
 
-#undef ice_access
-
 static void
 ice_base_queue_get(struct ice_pf *pf)
 {
diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
index 2ee88fbfc7..879da7ef59 100644
--- a/drivers/net/nfp/nfp_net.c
+++ b/drivers/net/nfp/nfp_net.c
@@ -29,6 +29,7 @@
 #include <rte_alarm.h>
 #include <rte_spinlock.h>
 #include <rte_service_component.h>
+#include <rte_firmware.h>
 
 #include "nfpcore/nfp_cpp.h"
 #include "nfpcore/nfp_nffw.h"
@@ -3366,12 +3367,10 @@ static int
 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 {
 	struct nfp_cpp *cpp = nsp->cpp;
-	int fw_f;
-	char *fw_buf;
+	void *fw_buf;
 	char fw_name[125];
 	char serial[40];
-	struct stat file_stat;
-	off_t fsize, bytes;
+	size_t fsize;
 
 	/* Looking for firmware file in order of priority */
 
@@ -3384,66 +3383,34 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 
 	snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 			serial);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Then try the PCI name */
 	snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 			dev->device.name);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Finally try the card type and media */
 	snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f < 0) {
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
 		PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 		return -ENOENT;
 	}
 
-read_fw:
-	if (fstat(fw_f, &file_stat) < 0) {
-		PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
-		close(fw_f);
-		return -ENOENT;
-	}
-
-	fsize = file_stat.st_size;
-	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
-			    fw_name, (uint64_t)fsize);
-
-	fw_buf = malloc((size_t)fsize);
-	if (!fw_buf) {
-		PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
-		close(fw_f);
-		return -ENOMEM;
-	}
-	memset(fw_buf, 0, fsize);
-
-	bytes = read(fw_f, fw_buf, fsize);
-	if (bytes != fsize) {
-		PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
-				   "Just %" PRIu64 " of %" PRIu64 " bytes read",
-				   (uint64_t)bytes, (uint64_t)fsize);
-		free(fw_buf);
-		close(fw_f);
-		return -EIO;
-	}
+load_fw:
+	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
+		fw_name, fsize);
 
 	PMD_DRV_LOG(INFO, "Uploading the firmware ...");
-	nfp_nsp_load_fw(nsp, fw_buf, bytes);
+	nfp_nsp_load_fw(nsp, fw_buf, fsize);
 	PMD_DRV_LOG(INFO, "Done");
 
 	free(fw_buf);
-	close(fw_f);
-
 	return 0;
 }
 
diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
index caa9d1d4f6..504e2c66a0 100644
--- a/drivers/net/qede/qede_main.c
+++ b/drivers/net/qede/qede_main.c
@@ -5,7 +5,9 @@
  */
 
 #include <limits.h>
+
 #include <rte_alarm.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #include "qede_ethdev.h"
@@ -127,51 +129,40 @@ static void qed_free_stream_mem(struct ecore_dev *edev)
 #ifdef CONFIG_ECORE_BINARY_FW
 static int qed_load_firmware_data(struct ecore_dev *edev)
 {
-	int fd;
-	struct stat st;
 	const char *fw = RTE_LIBRTE_QEDE_FW;
+	void *buf;
+	size_t bufsz;
+	int ret;
 
 	if (strcmp(fw, "") == 0)
 		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
 	else
 		strcpy(qede_fw_file, fw);
 
-	fd = open(qede_fw_file, O_RDONLY);
-	if (fd < 0) {
-		DP_ERR(edev, "Can't open firmware file\n");
-		return -ENOENT;
-	}
-
-	if (fstat(fd, &st) < 0) {
-		DP_ERR(edev, "Can't stat firmware file\n");
-		close(fd);
+	if (rte_firmware_read(qede_fw_file, &buf, &bufsz) < 0) {
+		DP_ERR(edev, "Can't read firmware data: %s\n", qede_fw_file);
 		return -1;
 	}
 
-	edev->firmware = rte_zmalloc("qede_fw", st.st_size,
-				    RTE_CACHE_LINE_SIZE);
+	edev->firmware = rte_zmalloc("qede_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!edev->firmware) {
 		DP_ERR(edev, "Can't allocate memory for firmware\n");
-		close(fd);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto out;
 	}
 
-	if (read(fd, edev->firmware, st.st_size) != st.st_size) {
-		DP_ERR(edev, "Can't read firmware data\n");
-		close(fd);
-		return -1;
-	}
-
-	edev->fw_len = st.st_size;
+	memcpy(edev->firmware, buf, bufsz);
+	edev->fw_len = bufsz;
 	if (edev->fw_len < 104) {
 		DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n",
 			  edev->fw_len);
-		close(fd);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
-
-	close(fd);
-	return 0;
+	ret = 0;
+out:
+	free(buf);
+	return ret;
 }
 #endif
 
diff --git a/lib/eal/include/rte_firmware.h b/lib/eal/include/rte_firmware.h
new file mode 100644
index 0000000000..3389e60ca0
--- /dev/null
+++ b/lib/eal/include/rte_firmware.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifndef __RTE_FIRMWARE_H__
+#define __RTE_FIRMWARE_H__
+
+#include <sys/types.h>
+
+#include <rte_compat.h>
+
+/**
+ * Load a firmware in a dynamically allocated buffer, dealing with compressed
+ * files if libarchive is available.
+ *
+ * @param[in] name
+ *      Firmware filename to load.
+ * @param[out] buf
+ *      Buffer allocated by this function. If this function succeeds, the
+ *      caller is responsible for calling free() on this buffer.
+ * @param[out] bufsz
+ *      Size of the data in the buffer.
+ *
+ * @return
+ *      0 if successful.
+ *      Negative otherwise, buf and bufsize contents are invalid.
+ */
+__rte_internal
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz);
+
+#endif
diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
new file mode 100644
index 0000000000..494cd5f058
--- /dev/null
+++ b/lib/eal/unix/eal_firmware.c
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifdef RTE_HAS_LIBARCHIVE
+#include <archive.h>
+#endif
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_firmware.h>
+#include <rte_log.h>
+
+#ifdef RTE_HAS_LIBARCHIVE
+
+struct firmware_read_ctx {
+	struct archive *a;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
+{
+	struct archive_entry *e;
+
+	ctx->a = archive_read_new();
+	if (ctx->a == NULL)
+		return -1;
+	if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK ||
+			archive_read_support_filter_xz(ctx->a) != ARCHIVE_OK ||
+			archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK ||
+			archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) {
+		archive_read_free(ctx->a);
+		ctx->a = NULL;
+		return -1;
+	}
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return archive_read_data(ctx->a, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	archive_read_free(ctx->a);
+	ctx->a = NULL;
+}
+
+#else /* !RTE_HAS_LIBARCHIVE */
+
+struct firmware_read_ctx {
+	int fd;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name,
+	__rte_unused size_t blocksize)
+{
+	ctx->fd = open(name, O_RDONLY);
+	if (ctx->fd < 0)
+		return -1;
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return read(ctx->fd, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	close(ctx->fd);
+	ctx->fd = -1;
+}
+
+#endif /* !RTE_HAS_LIBARCHIVE */
+
+static int
+firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	const size_t blocksize = 4096;
+	struct firmware_read_ctx ctx;
+	int ret = -1;
+	int err;
+
+	*buf = NULL;
+	*bufsz = 0;
+
+	if (firmware_open(&ctx, name, blocksize) < 0)
+		return -1;
+
+	do {
+		void *tmp;
+
+		tmp = realloc(*buf, *bufsz + blocksize);
+		if (tmp == NULL) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*buf = tmp;
+
+		err = firmware_read_block(&ctx, RTE_PTR_ADD(*buf, *bufsz), blocksize);
+		if (err < 0) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*bufsz += err;
+
+	} while (err != 0);
+
+	ret = 0;
+out:
+	firmware_close(&ctx);
+	return ret;
+}
+
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	char path[PATH_MAX];
+	int ret;
+
+	ret = firmware_read(name, buf, bufsz);
+	if (ret < 0) {
+		snprintf(path, sizeof(path), "%s.xz", name);
+		path[PATH_MAX - 1] = '\0';
+#ifndef RTE_HAS_LIBARCHIVE
+		if (access(path, F_OK) == 0) {
+			RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",
+				path);
+		}
+#else
+		ret = firmware_read(path, buf, bufsz);
+#endif
+	}
+	return ret;
+}
diff --git a/lib/eal/unix/meson.build b/lib/eal/unix/meson.build
index dc711b4240..e3ecd3e956 100644
--- a/lib/eal/unix/meson.build
+++ b/lib/eal/unix/meson.build
@@ -5,5 +5,6 @@ sources += files(
         'eal_file.c',
         'eal_unix_memory.c',
         'eal_unix_timer.c',
+        'eal_firmware.c',
         'rte_thread.c',
 )
diff --git a/lib/eal/version.map b/lib/eal/version.map
index fe5c3dac98..2df65c6903 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -428,6 +428,7 @@ EXPERIMENTAL {
 INTERNAL {
 	global:
 
+	rte_firmware_read;
 	rte_mem_lock;
 	rte_mem_map;
 	rte_mem_page_size;
diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
index 8483f6b02c..6fe2bdd282 100644
--- a/lib/eal/windows/eal.c
+++ b/lib/eal/windows/eal.c
@@ -21,6 +21,7 @@
 #include <eal_private.h>
 #include <rte_service_component.h>
 #include <rte_vfio.h>
+#include <rte_firmware.h>
 
 #include "eal_hugepages.h"
 #include "eal_trace.h"
@@ -465,3 +466,11 @@ rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 {
 	return -1;
 }
+
+int
+rte_firmware_read(__rte_unused const char *name,
+			__rte_unused void **buf,
+			__rte_unused size_t *bufsz)
+{
+	return -1;
+}
-- 
2.23.0

Re: [dpdk-dev] [PATCH v3 2/2] eal: handle compressed firmwares

[Aaron Conole]

David Marchand <david.marchand@redhat.com> writes:

> Introduce an internal firmware loading helper to remove code duplication
> in our drivers and handle xz compressed firmwares by calling libarchive.
>
> This helper tries to look for .xz suffixes so that drivers are not aware
> the firmwares have been compressed.
>
> libarchive is set as an optional dependency: without libarchive, a
> runtime warning is emitted so that users know there is a compressed
> firmware.
>
> Windows implementation is left as an empty stub.
>
> Signed-off-by: David Marchand <david.marchand@redhat.com>
> Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
> ---
> Changes since v2:
> - added a comment on libarchive link dependency,
>
> Changes since v1:
> - used pkg-config for libarchive detection,
> - updated doxygen annotations,
> - added internal helpers in eal_firmware.c to enhance readability,
> - dropped whitespace damage in version.map,
>
> ---
>  .github/workflows/build.yml    |   1 +
>  .travis.yml                    |   1 +
>  config/meson.build             |  10 +++
>  drivers/net/bnx2x/bnx2x.c      |  35 +++-----
>  drivers/net/ice/ice_ethdev.c   |  60 +++----------
>  drivers/net/nfp/nfp_net.c      |  57 +++----------
>  drivers/net/qede/qede_main.c   |  45 ++++------
>  lib/eal/include/rte_firmware.h |  32 +++++++
>  lib/eal/unix/eal_firmware.c    | 149 +++++++++++++++++++++++++++++++++
>  lib/eal/unix/meson.build       |   1 +
>  lib/eal/version.map            |   1 +
>  lib/eal/windows/eal.c          |   9 ++
>  12 files changed, 259 insertions(+), 142 deletions(-)
>  create mode 100644 lib/eal/include/rte_firmware.h
>  create mode 100644 lib/eal/unix/eal_firmware.c
>
> diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
> index 7c4d6dcdbf..7dac20ddeb 100644
> --- a/.github/workflows/build.yml
> +++ b/.github/workflows/build.yml
> @@ -93,6 +93,7 @@ jobs:
>        run: sudo apt install -y ccache libnuma-dev python3-setuptools
>          python3-wheel python3-pip python3-pyelftools ninja-build libbsd-dev
>          libpcap-dev libibverbs-dev libcrypto++-dev libfdt-dev libjansson-dev
> +        libarchive-dev
>      - name: Install libabigail build dependencies if no cache is available
>        if: env.ABI_CHECKS == 'true' && steps.libabigail-cache.outputs.cache-hit != 'true'
>        run: sudo apt install -y autoconf automake libtool pkg-config libxml2-dev
> diff --git a/.travis.yml b/.travis.yml
> index 5b702cc9bb..23067d9e3c 100644
> --- a/.travis.yml
> +++ b/.travis.yml
> @@ -16,6 +16,7 @@ addons:
>      packages: &required_packages
>        - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
>        - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
> +      - [libarchive-dev]
>  
>  _aarch64_packages: &aarch64_packages
>    - *required_packages
> diff --git a/config/meson.build b/config/meson.build
> index 017bb2efbb..639aa74e12 100644
> --- a/config/meson.build
> +++ b/config/meson.build
> @@ -172,6 +172,16 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
>      dpdk_extra_ldflags += '-lexecinfo'
>  endif
>  
> +libarchive = dependency('libarchive', required: false, method: 'pkg-config')
> +if libarchive.found()
> +    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
> +    # Push libarchive link dependency at the project level to support
> +    # statically linking dpdk apps. Details at:
> +    # https://inbox.dpdk.org/dev/20210605004024.660267a1@sovereign/
> +    add_project_link_arguments('-larchive', language: 'c')
> +    dpdk_extra_ldflags += '-larchive'
> +endif
> +
>  # check for libbsd
>  libbsd = dependency('libbsd', required: false, method: 'pkg-config')
>  if libbsd.found()
> diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
> index 654878d9de..60292753e2 100644
> --- a/drivers/net/bnx2x/bnx2x.c
> +++ b/drivers/net/bnx2x/bnx2x.c
> @@ -26,7 +26,9 @@
>  #include <arpa/inet.h>
>  #include <fcntl.h>
>  #include <zlib.h>
> +
>  #include <rte_bitops.h>
> +#include <rte_firmware.h>
>  #include <rte_string_fns.h>
>  
>  #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
> @@ -9655,44 +9657,33 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc)
>  void bnx2x_load_firmware(struct bnx2x_softc *sc)
>  {
>  	const char *fwname;
> -	int f;
> -	struct stat st;
> +	void *buf;
> +	size_t bufsz;
>  
>  	fwname = sc->devinfo.device_id == CHIP_NUM_57711
>  		? FW_NAME_57711 : FW_NAME_57810;
> -	f = open(fwname, O_RDONLY);
> -	if (f < 0) {
> +	if (rte_firmware_read(fwname, &buf, &bufsz) != 0) {
>  		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
>  		return;
>  	}
>  
> -	if (fstat(f, &st) < 0) {
> -		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
> -		close(f);
> -		return;
> -	}
> -
> -	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
> +	sc->firmware = rte_zmalloc("bnx2x_fw", bufsz, RTE_CACHE_LINE_SIZE);
>  	if (!sc->firmware) {
>  		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
> -		close(f);
> -		return;
> +		goto out;
>  	}
>  
> -	if (read(f, sc->firmware, st.st_size) != st.st_size) {
> -		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
> -		close(f);
> -		return;
> -	}
> -	close(f);
> -
> -	sc->fw_len = st.st_size;
> +	sc->fw_len = bufsz;
>  	if (sc->fw_len < FW_HEADER_LEN) {
>  		PMD_DRV_LOG(NOTICE, sc,
>  			    "Invalid fw size: %" PRIu64, sc->fw_len);
> -		return;
> +		goto out;
>  	}
> +
> +	memcpy(sc->firmware, buf, sc->fw_len);
>  	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
> +out:
> +	free(buf);
>  }
>  
>  static void
> diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
> index d180b73c32..06da1bbd94 100644
> --- a/drivers/net/ice/ice_ethdev.c
> +++ b/drivers/net/ice/ice_ethdev.c
> @@ -10,6 +10,7 @@
>  #include <sys/stat.h>
>  #include <unistd.h>
>  
> +#include <rte_firmware.h>
>  #include <rte_tailq.h>
>  
>  #include "base/ice_sched.h"
> @@ -1673,22 +1674,14 @@ ice_load_pkg_type(struct ice_hw *hw)
>  	return package_type;
>  }
>  
> -#ifdef RTE_EXEC_ENV_WINDOWS
> -#define ice_access _access
> -#else
> -#define ice_access access
> -#endif
> -
>  int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
>  {
>  	struct ice_hw *hw = &adapter->hw;
>  	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
>  	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
> +	void *buf;
> +	size_t bufsz;
>  	int err;
> -	uint8_t *buf = NULL;
> -	int buf_len;
> -	FILE *file;
> -	struct stat fstat;
>  
>  	if (!use_dsn)
>  		goto no_dsn;
> @@ -1698,57 +1691,31 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
>  		"ice-%016" PRIx64 ".pkg", dsn);
>  	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
>  		ICE_MAX_PKG_FILENAME_SIZE);
> -	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> +	strcat(pkg_file, opt_ddp_filename);
> +	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
>  		goto load_fw;
>  
>  	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
>  		ICE_MAX_PKG_FILENAME_SIZE);
> -	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
> +	strcat(pkg_file, opt_ddp_filename);
> +	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
>  		goto load_fw;
>  
>  no_dsn:
>  	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
> -	if (!ice_access(pkg_file, 0))
> +	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
>  		goto load_fw;
> +
>  	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
> -	if (ice_access(pkg_file, 0)) {
> +	if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
>  		PMD_INIT_LOG(ERR, "failed to search file path\n");
>  		return -1;
>  	}
>  
>  load_fw:
> -	file = fopen(pkg_file, "rb");
> -	if (!file)  {
> -		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
> -		return -1;
> -	}
> -
>  	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
>  
> -	err = stat(pkg_file, &fstat);
> -	if (err) {
> -		PMD_INIT_LOG(ERR, "failed to get file stats\n");
> -		goto out;
> -	}
> -
> -	buf_len = fstat.st_size;
> -	buf = rte_malloc(NULL, buf_len, 0);
> -
> -	if (!buf) {
> -		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
> -				buf_len);
> -		err = -1;
> -		goto out;
> -	}
> -
> -	err = fread(buf, buf_len, 1, file);
> -	if (err != 1) {
> -		PMD_INIT_LOG(ERR, "failed to read package data\n");
> -		err = -1;
> -		goto out;
> -	}
> -
> -	err = ice_copy_and_init_pkg(hw, buf, buf_len);
> +	err = ice_copy_and_init_pkg(hw, buf, bufsz);
>  	if (err) {
>  		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
>  		goto out;
> @@ -1758,13 +1725,10 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
>  	adapter->active_pkg_type = ice_load_pkg_type(hw);
>  
>  out:
> -	fclose(file);
> -	rte_free(buf);
> +	free(buf);
>  	return err;
>  }
>  
> -#undef ice_access
> -
>  static void
>  ice_base_queue_get(struct ice_pf *pf)
>  {
> diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
> index 2ee88fbfc7..879da7ef59 100644
> --- a/drivers/net/nfp/nfp_net.c
> +++ b/drivers/net/nfp/nfp_net.c
> @@ -29,6 +29,7 @@
>  #include <rte_alarm.h>
>  #include <rte_spinlock.h>
>  #include <rte_service_component.h>
> +#include <rte_firmware.h>
>  
>  #include "nfpcore/nfp_cpp.h"
>  #include "nfpcore/nfp_nffw.h"
> @@ -3366,12 +3367,10 @@ static int
>  nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
>  {
>  	struct nfp_cpp *cpp = nsp->cpp;
> -	int fw_f;
> -	char *fw_buf;
> +	void *fw_buf;
>  	char fw_name[125];
>  	char serial[40];
> -	struct stat file_stat;
> -	off_t fsize, bytes;
> +	size_t fsize;
>  
>  	/* Looking for firmware file in order of priority */
>  
> @@ -3384,66 +3383,34 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
>  
>  	snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
>  			serial);
> -
>  	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
> -	fw_f = open(fw_name, O_RDONLY);
> -	if (fw_f >= 0)
> -		goto read_fw;
> +	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
> +		goto load_fw;
>  
>  	/* Then try the PCI name */
>  	snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
>  			dev->device.name);
> -
>  	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
> -	fw_f = open(fw_name, O_RDONLY);
> -	if (fw_f >= 0)
> -		goto read_fw;
> +	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
> +		goto load_fw;
>  
>  	/* Finally try the card type and media */
>  	snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
>  	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
> -	fw_f = open(fw_name, O_RDONLY);
> -	if (fw_f < 0) {
> +	if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
>  		PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
>  		return -ENOENT;
>  	}
>  
> -read_fw:
> -	if (fstat(fw_f, &file_stat) < 0) {
> -		PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
> -		close(fw_f);
> -		return -ENOENT;
> -	}
> -
> -	fsize = file_stat.st_size;
> -	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
> -			    fw_name, (uint64_t)fsize);
> -
> -	fw_buf = malloc((size_t)fsize);
> -	if (!fw_buf) {
> -		PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
> -		close(fw_f);
> -		return -ENOMEM;
> -	}
> -	memset(fw_buf, 0, fsize);
> -
> -	bytes = read(fw_f, fw_buf, fsize);
> -	if (bytes != fsize) {
> -		PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
> -				   "Just %" PRIu64 " of %" PRIu64 " bytes read",
> -				   (uint64_t)bytes, (uint64_t)fsize);
> -		free(fw_buf);
> -		close(fw_f);
> -		return -EIO;
> -	}
> +load_fw:
> +	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
> +		fw_name, fsize);
>  
>  	PMD_DRV_LOG(INFO, "Uploading the firmware ...");
> -	nfp_nsp_load_fw(nsp, fw_buf, bytes);
> +	nfp_nsp_load_fw(nsp, fw_buf, fsize);
>  	PMD_DRV_LOG(INFO, "Done");
>  
>  	free(fw_buf);
> -	close(fw_f);
> -
>  	return 0;
>  }
>  
> diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
> index caa9d1d4f6..504e2c66a0 100644
> --- a/drivers/net/qede/qede_main.c
> +++ b/drivers/net/qede/qede_main.c
> @@ -5,7 +5,9 @@
>   */
>  
>  #include <limits.h>
> +
>  #include <rte_alarm.h>
> +#include <rte_firmware.h>
>  #include <rte_string_fns.h>
>  
>  #include "qede_ethdev.h"
> @@ -127,51 +129,40 @@ static void qed_free_stream_mem(struct ecore_dev *edev)
>  #ifdef CONFIG_ECORE_BINARY_FW
>  static int qed_load_firmware_data(struct ecore_dev *edev)
>  {
> -	int fd;
> -	struct stat st;
>  	const char *fw = RTE_LIBRTE_QEDE_FW;
> +	void *buf;
> +	size_t bufsz;
> +	int ret;
>  
>  	if (strcmp(fw, "") == 0)
>  		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
>  	else
>  		strcpy(qede_fw_file, fw);
>  
> -	fd = open(qede_fw_file, O_RDONLY);
> -	if (fd < 0) {
> -		DP_ERR(edev, "Can't open firmware file\n");
> -		return -ENOENT;
> -	}
> -
> -	if (fstat(fd, &st) < 0) {
> -		DP_ERR(edev, "Can't stat firmware file\n");
> -		close(fd);
> +	if (rte_firmware_read(qede_fw_file, &buf, &bufsz) < 0) {
> +		DP_ERR(edev, "Can't read firmware data: %s\n", qede_fw_file);
>  		return -1;
>  	}
>  
> -	edev->firmware = rte_zmalloc("qede_fw", st.st_size,
> -				    RTE_CACHE_LINE_SIZE);
> +	edev->firmware = rte_zmalloc("qede_fw", bufsz, RTE_CACHE_LINE_SIZE);
>  	if (!edev->firmware) {
>  		DP_ERR(edev, "Can't allocate memory for firmware\n");
> -		close(fd);
> -		return -ENOMEM;
> +		ret = -ENOMEM;
> +		goto out;
>  	}
>  
> -	if (read(fd, edev->firmware, st.st_size) != st.st_size) {
> -		DP_ERR(edev, "Can't read firmware data\n");
> -		close(fd);
> -		return -1;
> -	}
> -
> -	edev->fw_len = st.st_size;
> +	memcpy(edev->firmware, buf, bufsz);
> +	edev->fw_len = bufsz;
>  	if (edev->fw_len < 104) {
>  		DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n",
>  			  edev->fw_len);
> -		close(fd);
> -		return -EINVAL;
> +		ret = -EINVAL;
> +		goto out;
>  	}
> -
> -	close(fd);
> -	return 0;
> +	ret = 0;
> +out:
> +	free(buf);
> +	return ret;
>  }
>  #endif
>  
> diff --git a/lib/eal/include/rte_firmware.h b/lib/eal/include/rte_firmware.h
> new file mode 100644
> index 0000000000..3389e60ca0
> --- /dev/null
> +++ b/lib/eal/include/rte_firmware.h
> @@ -0,0 +1,32 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 Red Hat, Inc.
> + */
> +
> +#ifndef __RTE_FIRMWARE_H__
> +#define __RTE_FIRMWARE_H__
> +
> +#include <sys/types.h>
> +
> +#include <rte_compat.h>
> +
> +/**
> + * Load a firmware in a dynamically allocated buffer, dealing with compressed
> + * files if libarchive is available.
> + *
> + * @param[in] name
> + *      Firmware filename to load.
> + * @param[out] buf
> + *      Buffer allocated by this function. If this function succeeds, the
> + *      caller is responsible for calling free() on this buffer.
> + * @param[out] bufsz
> + *      Size of the data in the buffer.
> + *
> + * @return
> + *      0 if successful.
> + *      Negative otherwise, buf and bufsize contents are invalid.
> + */
> +__rte_internal
> +int
> +rte_firmware_read(const char *name, void **buf, size_t *bufsz);
> +
> +#endif
> diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
> new file mode 100644
> index 0000000000..494cd5f058
> --- /dev/null
> +++ b/lib/eal/unix/eal_firmware.c
> @@ -0,0 +1,149 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 Red Hat, Inc.
> + */
> +
> +#ifdef RTE_HAS_LIBARCHIVE
> +#include <archive.h>
> +#endif
> +#include <fcntl.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <unistd.h>
> +
> +#include <rte_common.h>
> +#include <rte_firmware.h>
> +#include <rte_log.h>
> +
> +#ifdef RTE_HAS_LIBARCHIVE
> +
> +struct firmware_read_ctx {
> +	struct archive *a;
> +};
> +
> +static int
> +firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
> +{
> +	struct archive_entry *e;
> +
> +	ctx->a = archive_read_new();
> +	if (ctx->a == NULL)
> +		return -1;
> +	if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK ||
> +			archive_read_support_filter_xz(ctx->a) != ARCHIVE_OK ||
> +			archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK ||
> +			archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) {
> +		archive_read_free(ctx->a);
> +		ctx->a = NULL;
> +		return -1;
> +	}
> +	return 0;
> +}
> +
> +static ssize_t
> +firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
> +{
> +	return archive_read_data(ctx->a, buf, count);
> +}
> +
> +static void
> +firmware_close(struct firmware_read_ctx *ctx)
> +{
> +	archive_read_free(ctx->a);
> +	ctx->a = NULL;
> +}
> +
> +#else /* !RTE_HAS_LIBARCHIVE */
> +
> +struct firmware_read_ctx {
> +	int fd;
> +};
> +
> +static int
> +firmware_open(struct firmware_read_ctx *ctx, const char *name,
> +	__rte_unused size_t blocksize)
> +{
> +	ctx->fd = open(name, O_RDONLY);
> +	if (ctx->fd < 0)
> +		return -1;
> +	return 0;
> +}
> +
> +static ssize_t
> +firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
> +{
> +	return read(ctx->fd, buf, count);
> +}
> +
> +static void
> +firmware_close(struct firmware_read_ctx *ctx)
> +{
> +	close(ctx->fd);
> +	ctx->fd = -1;
> +}
> +
> +#endif /* !RTE_HAS_LIBARCHIVE */
> +
> +static int
> +firmware_read(const char *name, void **buf, size_t *bufsz)
> +{
> +	const size_t blocksize = 4096;
> +	struct firmware_read_ctx ctx;
> +	int ret = -1;
> +	int err;
> +
> +	*buf = NULL;
> +	*bufsz = 0;
> +
> +	if (firmware_open(&ctx, name, blocksize) < 0)
> +		return -1;
> +
> +	do {
> +		void *tmp;
> +
> +		tmp = realloc(*buf, *bufsz + blocksize);
> +		if (tmp == NULL) {
> +			free(*buf);
> +			*buf = NULL;
> +			*bufsz = 0;
> +			goto out;
> +		}
> +		*buf = tmp;
> +
> +		err = firmware_read_block(&ctx, RTE_PTR_ADD(*buf, *bufsz), blocksize);
> +		if (err < 0) {
> +			free(*buf);
> +			*buf = NULL;
> +			*bufsz = 0;
> +			goto out;
> +		}
> +		*bufsz += err;
> +
> +	} while (err != 0);
> +
> +	ret = 0;
> +out:
> +	firmware_close(&ctx);
> +	return ret;
> +}
> +
> +int
> +rte_firmware_read(const char *name, void **buf, size_t *bufsz)
> +{
> +	char path[PATH_MAX];
> +	int ret;
> +
> +	ret = firmware_read(name, buf, bufsz);
> +	if (ret < 0) {
> +		snprintf(path, sizeof(path), "%s.xz", name);
> +		path[PATH_MAX - 1] = '\0';
> +#ifndef RTE_HAS_LIBARCHIVE
> +		if (access(path, F_OK) == 0) {
> +			RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",

Maybe 'not linked' instead of 'not available'.  A user might decide to
install libarchive and then be confused that the error message still
pops up (without realizing they need to re-compile).

Otherwise,
Acked-by: Aaron Conole <aconole@redhat.com>

> +				path);
> +		}
> +#else
> +		ret = firmware_read(path, buf, bufsz);
> +#endif
> +	}
> +	return ret;
> +}
> diff --git a/lib/eal/unix/meson.build b/lib/eal/unix/meson.build
> index dc711b4240..e3ecd3e956 100644
> --- a/lib/eal/unix/meson.build
> +++ b/lib/eal/unix/meson.build
> @@ -5,5 +5,6 @@ sources += files(
>          'eal_file.c',
>          'eal_unix_memory.c',
>          'eal_unix_timer.c',
> +        'eal_firmware.c',
>          'rte_thread.c',
>  )
> diff --git a/lib/eal/version.map b/lib/eal/version.map
> index fe5c3dac98..2df65c6903 100644
> --- a/lib/eal/version.map
> +++ b/lib/eal/version.map
> @@ -428,6 +428,7 @@ EXPERIMENTAL {
>  INTERNAL {
>  	global:
>  
> +	rte_firmware_read;
>  	rte_mem_lock;
>  	rte_mem_map;
>  	rte_mem_page_size;
> diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
> index 8483f6b02c..6fe2bdd282 100644
> --- a/lib/eal/windows/eal.c
> +++ b/lib/eal/windows/eal.c
> @@ -21,6 +21,7 @@
>  #include <eal_private.h>
>  #include <rte_service_component.h>
>  #include <rte_vfio.h>
> +#include <rte_firmware.h>
>  
>  #include "eal_hugepages.h"
>  #include "eal_trace.h"
> @@ -465,3 +466,11 @@ rte_vfio_container_dma_unmap(__rte_unused int container_fd,
>  {
>  	return -1;
>  }
> +
> +int
> +rte_firmware_read(__rte_unused const char *name,
> +			__rte_unused void **buf,
> +			__rte_unused size_t *bufsz)
> +{
> +	return -1;
> +}

Re: [dpdk-dev] [PATCH v3 2/2] eal: handle compressed firmwares

[Wang, Haiyue]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> Sent: Tuesday, June 29, 2021 16:07
> To: dev@dpdk.org
> Cc: Igor Russkikh <irusskikh@marvell.com>; Aaron Conole <aconole@redhat.com>; Michael Santana
> <maicolgabriel@hotmail.com>; Richardson, Bruce <bruce.richardson@intel.com>; Rasesh Mody
> <rmody@marvell.com>; Shahed Shaikh <shshaikh@marvell.com>; Yang, Qiming <qiming.yang@intel.com>; Zhang,
> Qi Z <qi.z.zhang@intel.com>; Heinrich Kuhn <heinrich.kuhn@netronome.com>; Devendra Singh Rawat
> <dsinghrawat@marvell.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>; Dmitry
> Kozlyuk <dmitry.kozliuk@gmail.com>; Narcisa Ana Maria Vasile <navasile@linux.microsoft.com>; Dmitry
> Malloy <dmitrym@microsoft.com>; Kadam, Pallavi <pallavi.kadam@intel.com>
> Subject: [dpdk-dev] [PATCH v3 2/2] eal: handle compressed firmwares
> 
> Introduce an internal firmware loading helper to remove code duplication
> in our drivers and handle xz compressed firmwares by calling libarchive.
> 
> This helper tries to look for .xz suffixes so that drivers are not aware
> the firmwares have been compressed.
> 
> libarchive is set as an optional dependency: without libarchive, a
> runtime warning is emitted so that users know there is a compressed
> firmware.
> 
> Windows implementation is left as an empty stub.
> 
> Signed-off-by: David Marchand <david.marchand@redhat.com>
> Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
> ---
> Changes since v2:
> - added a comment on libarchive link dependency,
> 
> Changes since v1:
> - used pkg-config for libarchive detection,
> - updated doxygen annotations,
> - added internal helpers in eal_firmware.c to enhance readability,
> - dropped whitespace damage in version.map,
> 
> ---
>  .github/workflows/build.yml    |   1 +
>  .travis.yml                    |   1 +
>  config/meson.build             |  10 +++
>  drivers/net/bnx2x/bnx2x.c      |  35 +++-----
>  drivers/net/ice/ice_ethdev.c   |  60 +++----------
>  drivers/net/nfp/nfp_net.c      |  57 +++----------
>  drivers/net/qede/qede_main.c   |  45 ++++------
>  lib/eal/include/rte_firmware.h |  32 +++++++
>  lib/eal/unix/eal_firmware.c    | 149 +++++++++++++++++++++++++++++++++
>  lib/eal/unix/meson.build       |   1 +
>  lib/eal/version.map            |   1 +
>  lib/eal/windows/eal.c          |   9 ++
>  12 files changed, 259 insertions(+), 142 deletions(-)
>  create mode 100644 lib/eal/include/rte_firmware.h
>  create mode 100644 lib/eal/unix/eal_firmware.c
> 


> +int
> +rte_firmware_read(const char *name, void **buf, size_t *bufsz)
> +{
> +	char path[PATH_MAX];
> +	int ret;
> +
> +	ret = firmware_read(name, buf, bufsz);
> +	if (ret < 0) {
> +		snprintf(path, sizeof(path), "%s.xz", name);
> +		path[PATH_MAX - 1] = '\0';
> +#ifndef RTE_HAS_LIBARCHIVE
> +		if (access(path, F_OK) == 0) {
> +			RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",
> +				path);
> +		}
> +#else
> +		ret = firmware_read(path, buf, bufsz);
> +#endif
> +	}
> +	return ret;
> +}


Since ice PMD needs to check if the firmware file with different name can be accessed
by some kind of order, before doing the final firmware selection. Should we also add
the firmware access API for handling this ?

bool
rte_firmware_access(const char *name)
{
	char path[PATH_MAX];

	if (access(name, F_OK) == 0)
		return true;

	snprintf(path, sizeof(path), "%s.xz", name);
	path[PATH_MAX - 1] = '\0';
	if (access(path, F_OK) == 0)
		return true;

	return false;
}


> --
> 2.23.0

Re: [dpdk-dev] [PATCH v3 2/2] eal: handle compressed firmwares

[David Marchand]

On Mon, Jul 5, 2021 at 8:35 AM Wang, Haiyue <haiyue.wang@intel.com> wrote:
> > +int
> > +rte_firmware_read(const char *name, void **buf, size_t *bufsz)
> > +{
> > +     char path[PATH_MAX];
> > +     int ret;
> > +
> > +     ret = firmware_read(name, buf, bufsz);
> > +     if (ret < 0) {
> > +             snprintf(path, sizeof(path), "%s.xz", name);
> > +             path[PATH_MAX - 1] = '\0';
> > +#ifndef RTE_HAS_LIBARCHIVE
> > +             if (access(path, F_OK) == 0) {
> > +                     RTE_LOG(WARNING, EAL, "libarchive not available, %s cannot be decompressed\n",
> > +                             path);
> > +             }
> > +#else
> > +             ret = firmware_read(path, buf, bufsz);
> > +#endif
> > +     }
> > +     return ret;
> > +}
>
>
> Since ice PMD needs to check if the firmware file with different name can be accessed
> by some kind of order, before doing the final firmware selection. Should we also add
> the firmware access API for handling this ?

I don't see the need.
Is the behavior changed for net/ice with this patch?


-- 
David Marchand

Re: [dpdk-dev] [PATCH v3 2/2] eal: handle compressed firmwares

[Wang, Haiyue]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of David Marchand
> Sent: Tuesday, June 29, 2021 16:07
> To: dev@dpdk.org
> Cc: Igor Russkikh <irusskikh@marvell.com>; Aaron Conole <aconole@redhat.com>; Michael Santana
> <maicolgabriel@hotmail.com>; Richardson, Bruce <bruce.richardson@intel.com>; Rasesh Mody
> <rmody@marvell.com>; Shahed Shaikh <shshaikh@marvell.com>; Yang, Qiming <qiming.yang@intel.com>; Zhang,
> Qi Z <qi.z.zhang@intel.com>; Heinrich Kuhn <heinrich.kuhn@netronome.com>; Devendra Singh Rawat
> <dsinghrawat@marvell.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>; Dmitry
> Kozlyuk <dmitry.kozliuk@gmail.com>; Narcisa Ana Maria Vasile <navasile@linux.microsoft.com>; Dmitry
> Malloy <dmitrym@microsoft.com>; Kadam, Pallavi <pallavi.kadam@intel.com>
> Subject: [dpdk-dev] [PATCH v3 2/2] eal: handle compressed firmwares
> 
> Introduce an internal firmware loading helper to remove code duplication
> in our drivers and handle xz compressed firmwares by calling libarchive.
> 
> This helper tries to look for .xz suffixes so that drivers are not aware
> the firmwares have been compressed.
> 
> libarchive is set as an optional dependency: without libarchive, a
> runtime warning is emitted so that users know there is a compressed
> firmware.
> 
> Windows implementation is left as an empty stub.
> 
> Signed-off-by: David Marchand <david.marchand@redhat.com>
> Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
> ---
> Changes since v2:
> - added a comment on libarchive link dependency,
> 
> Changes since v1:
> - used pkg-config for libarchive detection,
> - updated doxygen annotations,
> - added internal helpers in eal_firmware.c to enhance readability,
> - dropped whitespace damage in version.map,
> 
> ---
>  .github/workflows/build.yml    |   1 +
>  .travis.yml                    |   1 +
>  config/meson.build             |  10 +++
>  drivers/net/bnx2x/bnx2x.c      |  35 +++-----
>  drivers/net/ice/ice_ethdev.c   |  60 +++----------

For ice PMD:

Tested-by: Haiyue Wang <haiyue.wang@intel.com>

>  drivers/net/nfp/nfp_net.c      |  57 +++----------
>  drivers/net/qede/qede_main.c   |  45 ++++------
>  lib/eal/include/rte_firmware.h |  32 +++++++
>  lib/eal/unix/eal_firmware.c    | 149 +++++++++++++++++++++++++++++++++
>  lib/eal/unix/meson.build       |   1 +
>  lib/eal/version.map            |   1 +
>  lib/eal/windows/eal.c          |   9 ++
>  12 files changed, 259 insertions(+), 142 deletions(-)
>  create mode 100644 lib/eal/include/rte_firmware.h
>  create mode 100644 lib/eal/unix/eal_firmware.c
> 


> --
> 2.23.0

[dpdk-dev] [PATCH v4 0/2] Support compressed firmwares

[David Marchand]

Fedora 34 only provides compressed firmwares.

Introduce an internal driver helper to handle transparently compression.

I chose libarchive for decompressing as it seems widely available and
DPDK had used it in the past.

Windows support only matters for net/ice and firmware loading was skipped
in this driver before this series. Since I don't know if/how we want to
load firmwares on Windows, I let an empty stub for this OS.

This series has been compile tested on Linux (I'll trust the CI for
others OSes).
I only tested basic init with a net/ice device (no DCF test).

So please drivers maintainers, check nothing is broken.


-- 
David Marchand

Changes since v3:
- add release note update,

Changes since v2:
- update comment on libarchive link dependency,

Changes since v1:
- address comments on patch2,


David Marchand (2):
  net/ice: factorize firmware loading
  eal: handle compressed firmwares

 .github/workflows/build.yml            |   1 +
 .travis.yml                            |   1 +
 config/meson.build                     |  10 ++
 doc/guides/rel_notes/release_21_08.rst |   6 +
 drivers/net/bnx2x/bnx2x.c              |  35 ++---
 drivers/net/ice/base/ice_osdep.h       |   6 -
 drivers/net/ice/ice_dcf_parent.c       |  97 ++------------
 drivers/net/ice/ice_ethdev.c           | 175 ++++++++++---------------
 drivers/net/ice/ice_ethdev.h           |   3 +-
 drivers/net/nfp/nfp_net.c              |  57 ++------
 drivers/net/qede/qede_main.c           |  45 +++----
 lib/eal/include/rte_firmware.h         |  32 +++++
 lib/eal/unix/eal_firmware.c            | 149 +++++++++++++++++++++
 lib/eal/unix/meson.build               |   1 +
 lib/eal/version.map                    |   1 +
 lib/eal/windows/eal.c                  |   9 ++
 16 files changed, 330 insertions(+), 298 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

-- 
2.23.0

[dpdk-dev] [PATCH v4 1/2] net/ice: factorize firmware loading

[David Marchand]

Both "normal" and "dcf" inits have their copy of some firmware loading
code.

The DSN query is moved in specific parts for the "normal" and "dcf" init.

A common helper ice_load_pkg is then introduced and takes an adapter
pointer as its main input.

This helper takes care of finding the right firmware file and loading
it.
The adapter active_pkg_type field is set by this helper.

The ice_access macro is removed from the osdep.h header: osdep.h should
only hosts wrappers for base driver code.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Haiyue Wang <haiyue.wang@intel.com>
---
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
 drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 4 files changed, 88 insertions(+), 179 deletions(-)

diff --git a/drivers/net/ice/base/ice_osdep.h b/drivers/net/ice/base/ice_osdep.h
index 878c5597d4..78093adb00 100644
--- a/drivers/net/ice/base/ice_osdep.h
+++ b/drivers/net/ice/base/ice_osdep.h
@@ -74,12 +74,6 @@ typedef uint64_t        s64;
 #define min(a, b) RTE_MIN(a, b)
 #define max(a, b) RTE_MAX(a, b)
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 #define FIELD_SIZEOF(t, f) RTE_SIZEOF_FIELD(t, f)
 #define ARRAY_SIZE(arr) RTE_DIM(arr)
 
diff --git a/drivers/net/ice/ice_dcf_parent.c b/drivers/net/ice/ice_dcf_parent.c
index 19420a0f58..318239abdc 100644
--- a/drivers/net/ice/ice_dcf_parent.c
+++ b/drivers/net/ice/ice_dcf_parent.c
@@ -298,13 +298,14 @@ static void ice_dcf_uninit_parent_hw(struct ice_hw *hw)
 }
 
 static int
-ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
+ice_dcf_load_pkg(struct ice_adapter *adapter)
 {
 	struct ice_dcf_adapter *dcf_adapter =
-			container_of(hw, struct ice_dcf_adapter, parent.hw);
+			container_of(&adapter->hw, struct ice_dcf_adapter, parent.hw);
 	struct virtchnl_pkg_info pkg_info;
 	struct dcf_virtchnl_cmd vc_cmd;
-	uint64_t dsn;
+	bool use_dsn;
+	uint64_t dsn = 0;
 
 	vc_cmd.v_op = VIRTCHNL_OP_DCF_GET_PKG_INFO;
 	vc_cmd.req_msglen = 0;
@@ -312,90 +313,11 @@ ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
 	vc_cmd.rsp_buflen = sizeof(pkg_info);
 	vc_cmd.rsp_msgbuf = (uint8_t *)&pkg_info;
 
-	if (ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd))
-		goto pkg_file_direct;
+	use_dsn = ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd) == 0;
+	if (use_dsn)
+		rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
 
-	rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_UPDATES "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_DEFAULT "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-pkg_file_direct:
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_UPDATES);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_DEFAULT);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	return -1;
-}
-
-static int
-ice_dcf_load_pkg(struct ice_hw *hw)
-{
-	char pkg_name[ICE_MAX_PKG_FILENAME_SIZE];
-	uint8_t *pkg_buf;
-	uint32_t buf_len;
-	struct stat st;
-	FILE *fp;
-	int err;
-
-	if (ice_dcf_request_pkg_name(hw, pkg_name)) {
-		PMD_INIT_LOG(ERR, "Failed to locate the package file");
-		return -ENOENT;
-	}
-
-	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_name);
-
-	err = stat(pkg_name, &st);
-	if (err) {
-		PMD_INIT_LOG(ERR, "Failed to get file status");
-		return err;
-	}
-
-	buf_len = st.st_size;
-	pkg_buf = rte_malloc(NULL, buf_len, 0);
-	if (!pkg_buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buffer of size %u for package",
-			     buf_len);
-		return -1;
-	}
-
-	fp = fopen(pkg_name, "rb");
-	if (!fp)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s", pkg_name);
-		err = -1;
-		goto ret;
-	}
-
-	err = fread(pkg_buf, buf_len, 1, fp);
-	fclose(fp);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data");
-		err = -1;
-		goto ret;
-	}
-
-	err = ice_copy_and_init_pkg(hw, pkg_buf, buf_len);
-	if (err)
-		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d", err);
-
-ret:
-	rte_free(pkg_buf);
-	return err;
+	return ice_load_pkg(adapter, use_dsn, dsn);
 }
 
 int
@@ -435,13 +357,12 @@ ice_dcf_init_parent_adapter(struct rte_eth_dev *eth_dev)
 		return err;
 	}
 
-	err = ice_dcf_load_pkg(parent_hw);
+	err = ice_dcf_load_pkg(parent_adapter);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to load package with error %d",
 			     err);
 		goto uninit_hw;
 	}
-	parent_adapter->active_pkg_type = ice_load_pkg_type(parent_hw);
 
 	parent_adapter->pf.main_vsi->idx = hw->num_vfs;
 	ice_dcf_update_pf_vsi_map(parent_hw,
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 09e38590e5..d180b73c32 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -1649,57 +1649,7 @@ ice_pf_setup(struct ice_pf *pf)
 	return 0;
 }
 
-/*
- * Extract device serial number from PCIe Configuration Space and
- * determine the pkg file path according to the DSN.
- */
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int
-ice_pkg_file_search_path(struct rte_pci_device *pci_dev, char *pkg_file)
-{
-	off_t pos;
-	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
-	uint32_t dsn_low, dsn_high;
-	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
-
-	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
-
-	if (pos) {
-		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		if (rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
-			 "ice-%08x%08x.pkg", dsn_high, dsn_low);
-	} else {
-		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
-		goto fail_dsn;
-	}
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-fail_dsn:
-	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
-		return 0;
-	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	return 0;
-}
-#endif
-
-enum ice_pkg_type
+static enum ice_pkg_type
 ice_load_pkg_type(struct ice_hw *hw)
 {
 	enum ice_pkg_type package_type;
@@ -1723,37 +1673,62 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int ice_load_pkg(struct rte_eth_dev *dev)
+#ifdef RTE_EXEC_ENV_WINDOWS
+#define ice_access _access
+#else
+#define ice_access access
+#endif
+
+int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
-	struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
+	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
 	int err;
-	uint8_t *buf;
+	uint8_t *buf = NULL;
 	int buf_len;
 	FILE *file;
 	struct stat fstat;
-	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
-	struct ice_adapter *ad =
-		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 
-	err = ice_pkg_file_search_path(pci_dev, pkg_file);
-	if (err) {
+	if (!use_dsn)
+		goto no_dsn;
+
+	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
+	snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
+		"ice-%016" PRIx64 ".pkg", dsn);
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+no_dsn:
+	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(pkg_file, 0))
+		goto load_fw;
+	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
+	if (ice_access(pkg_file, 0)) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
-		return err;
+		return -1;
 	}
 
+load_fw:
 	file = fopen(pkg_file, "rb");
 	if (!file)  {
 		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
 		return -1;
 	}
 
+	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
+
 	err = stat(pkg_file, &fstat);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		fclose(file);
-		return err;
+		goto out;
 	}
 
 	buf_len = fstat.st_size;
@@ -1762,44 +1737,33 @@ static int ice_load_pkg(struct rte_eth_dev *dev)
 	if (!buf) {
 		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
 				buf_len);
-		fclose(file);
-		return -1;
+		err = -1;
+		goto out;
 	}
 
 	err = fread(buf, buf_len, 1, file);
 	if (err != 1) {
 		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		fclose(file);
 		err = -1;
-		goto fail_exit;
+		goto out;
 	}
 
-	fclose(file);
-
 	err = ice_copy_and_init_pkg(hw, buf, buf_len);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
-		goto fail_exit;
+		goto out;
 	}
 
 	/* store the loaded pkg type info */
-	ad->active_pkg_type = ice_load_pkg_type(hw);
+	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
-	err = ice_init_hw_tbls(hw);
-	if (err) {
-		PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", err);
-		goto fail_init_tbls;
-	}
-
-	return 0;
-
-fail_init_tbls:
-	rte_free(hw->pkg_copy);
-fail_exit:
+out:
+	fclose(file);
 	rte_free(buf);
 	return err;
 }
-#endif
+
+#undef ice_access
 
 static void
 ice_base_queue_get(struct ice_pf *pf)
@@ -2030,6 +1994,12 @@ ice_dev_init(struct rte_eth_dev *dev)
 		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_vsi *vsi;
 	int ret;
+#ifndef RTE_EXEC_ENV_WINDOWS
+	off_t pos;
+	uint32_t dsn_low, dsn_high;
+	uint64_t dsn;
+	bool use_dsn;
+#endif
 
 	dev->dev_ops = &ice_eth_dev_ops;
 	dev->rx_queue_count = ice_rx_queue_count;
@@ -2080,7 +2050,30 @@ ice_dev_init(struct rte_eth_dev *dev)
 	}
 
 #ifndef RTE_EXEC_ENV_WINDOWS
-	ret = ice_load_pkg(dev);
+	use_dsn = false;
+	dsn = 0;
+	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
+	if (pos) {
+		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0 ||
+				rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
+			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
+		} else {
+			use_dsn = true;
+			dsn = (uint64_t)dsn_high << 32 | dsn_low;
+		}
+	} else {
+		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
+	}
+
+	ret = ice_load_pkg(pf->adapter, use_dsn, dsn);
+	if (ret == 0) {
+		ret = ice_init_hw_tbls(hw);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", ret);
+			rte_free(hw->pkg_copy);
+		}
+	}
+
 	if (ret) {
 		if (ad->devargs.safe_mode_support == 0) {
 			PMD_INIT_LOG(ERR, "Failed to load the DDP package,"
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index 7f3c26fb6f..edafdf168b 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -534,7 +534,8 @@ struct ice_vsi_vlan_pvid_info {
 #define ICE_PF_TO_ETH_DEV(pf) \
 	(((struct ice_pf *)pf)->adapter->eth_dev)
 
-enum ice_pkg_type ice_load_pkg_type(struct ice_hw *hw);
+int
+ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn);
 struct ice_vsi *
 ice_setup_vsi(struct ice_pf *pf, enum ice_vsi_type type);
 int
-- 
2.23.0

[dpdk-dev] [PATCH v4 2/2] eal: handle compressed firmwares

[David Marchand]

Introduce an internal firmware loading helper to remove code duplication
in our drivers and handle xz compressed firmwares by calling libarchive.

This helper tries to look for .xz suffixes so that drivers are not aware
the firmwares have been compressed.

libarchive is set as an optional dependency: without libarchive, a
runtime warning is emitted so that users know there is a compressed
firmware.

Windows implementation is left as an empty stub.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
Acked-by: Aaron Conole <aconole@redhat.com>
Tested-by: Haiyue Wang <haiyue.wang@intel.com>
---
Changes since v3:
- added release note update,
- updated warning message when uncompressing is unavailable,

Changes since v2:
- added a comment on libarchive link dependency,

Changes since v1:
- used pkg-config for libarchive detection,
- updated doxygen annotations,
- added internal helpers in eal_firmware.c to enhance readability,
- dropped whitespace damage in version.map,

---
 .github/workflows/build.yml            |   1 +
 .travis.yml                            |   1 +
 config/meson.build                     |  10 ++
 doc/guides/rel_notes/release_21_08.rst |   6 +
 drivers/net/bnx2x/bnx2x.c              |  35 +++---
 drivers/net/ice/ice_ethdev.c           |  60 ++--------
 drivers/net/nfp/nfp_net.c              |  57 ++--------
 drivers/net/qede/qede_main.c           |  45 +++-----
 lib/eal/include/rte_firmware.h         |  32 ++++++
 lib/eal/unix/eal_firmware.c            | 149 +++++++++++++++++++++++++
 lib/eal/unix/meson.build               |   1 +
 lib/eal/version.map                    |   1 +
 lib/eal/windows/eal.c                  |   9 ++
 13 files changed, 265 insertions(+), 142 deletions(-)
 create mode 100644 lib/eal/include/rte_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c4d6dcdbf..7dac20ddeb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -93,6 +93,7 @@ jobs:
       run: sudo apt install -y ccache libnuma-dev python3-setuptools
         python3-wheel python3-pip python3-pyelftools ninja-build libbsd-dev
         libpcap-dev libibverbs-dev libcrypto++-dev libfdt-dev libjansson-dev
+        libarchive-dev
     - name: Install libabigail build dependencies if no cache is available
       if: env.ABI_CHECKS == 'true' && steps.libabigail-cache.outputs.cache-hit != 'true'
       run: sudo apt install -y autoconf automake libtool pkg-config libxml2-dev
diff --git a/.travis.yml b/.travis.yml
index 5b702cc9bb..23067d9e3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,7 @@ addons:
     packages: &required_packages
       - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
       - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
+      - [libarchive-dev]
 
 _aarch64_packages: &aarch64_packages
   - *required_packages
diff --git a/config/meson.build b/config/meson.build
index 017bb2efbb..639aa74e12 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -172,6 +172,16 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
     dpdk_extra_ldflags += '-lexecinfo'
 endif
 
+libarchive = dependency('libarchive', required: false, method: 'pkg-config')
+if libarchive.found()
+    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
+    # Push libarchive link dependency at the project level to support
+    # statically linking dpdk apps. Details at:
+    # https://inbox.dpdk.org/dev/20210605004024.660267a1@sovereign/
+    add_project_link_arguments('-larchive', language: 'c')
+    dpdk_extra_ldflags += '-larchive'
+endif
+
 # check for libbsd
 libbsd = dependency('libbsd', required: false, method: 'pkg-config')
 if libbsd.found()
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 0a05cb02fa..df6f9ebafc 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -61,6 +61,12 @@ New Features
   representing sub-domains of functionality. Each auxiliary device
   represents a part of its parent functionality.
 
+* **Added XZ compressed firmware support.**
+
+  Using ``rte_firmware_read``, a driver can now handle XZ compressed firmwares
+  in a transparent way, with EAL uncompressing using libarchive if this library
+  is available when building DPDK.
+
 * **Added Baseband PHY CNXK PMD.**
 
   Added Baseband PHY PMD which allows to configure BPHY hardware block
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
index 654878d9de..60292753e2 100644
--- a/drivers/net/bnx2x/bnx2x.c
+++ b/drivers/net/bnx2x/bnx2x.c
@@ -26,7 +26,9 @@
 #include <arpa/inet.h>
 #include <fcntl.h>
 #include <zlib.h>
+
 #include <rte_bitops.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
@@ -9655,44 +9657,33 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc)
 void bnx2x_load_firmware(struct bnx2x_softc *sc)
 {
 	const char *fwname;
-	int f;
-	struct stat st;
+	void *buf;
+	size_t bufsz;
 
 	fwname = sc->devinfo.device_id == CHIP_NUM_57711
 		? FW_NAME_57711 : FW_NAME_57810;
-	f = open(fwname, O_RDONLY);
-	if (f < 0) {
+	if (rte_firmware_read(fwname, &buf, &bufsz) != 0) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
 		return;
 	}
 
-	if (fstat(f, &st) < 0) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
-		close(f);
-		return;
-	}
-
-	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
+	sc->firmware = rte_zmalloc("bnx2x_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!sc->firmware) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
-		close(f);
-		return;
+		goto out;
 	}
 
-	if (read(f, sc->firmware, st.st_size) != st.st_size) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
-		close(f);
-		return;
-	}
-	close(f);
-
-	sc->fw_len = st.st_size;
+	sc->fw_len = bufsz;
 	if (sc->fw_len < FW_HEADER_LEN) {
 		PMD_DRV_LOG(NOTICE, sc,
 			    "Invalid fw size: %" PRIu64, sc->fw_len);
-		return;
+		goto out;
 	}
+
+	memcpy(sc->firmware, buf, sc->fw_len);
 	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
+out:
+	free(buf);
 }
 
 static void
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index d180b73c32..06da1bbd94 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -10,6 +10,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include <rte_firmware.h>
 #include <rte_tailq.h>
 
 #include "base/ice_sched.h"
@@ -1673,22 +1674,14 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
 	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
 	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
+	void *buf;
+	size_t bufsz;
 	int err;
-	uint8_t *buf = NULL;
-	int buf_len;
-	FILE *file;
-	struct stat fstat;
 
 	if (!use_dsn)
 		goto no_dsn;
@@ -1698,57 +1691,31 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 		"ice-%016" PRIx64 ".pkg", dsn);
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 no_dsn:
 	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
+
 	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	if (ice_access(pkg_file, 0)) {
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
 		return -1;
 	}
 
 load_fw:
-	file = fopen(pkg_file, "rb");
-	if (!file)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
-		return -1;
-	}
-
 	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
 
-	err = stat(pkg_file, &fstat);
-	if (err) {
-		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		goto out;
-	}
-
-	buf_len = fstat.st_size;
-	buf = rte_malloc(NULL, buf_len, 0);
-
-	if (!buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
-				buf_len);
-		err = -1;
-		goto out;
-	}
-
-	err = fread(buf, buf_len, 1, file);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		err = -1;
-		goto out;
-	}
-
-	err = ice_copy_and_init_pkg(hw, buf, buf_len);
+	err = ice_copy_and_init_pkg(hw, buf, bufsz);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
 		goto out;
@@ -1758,13 +1725,10 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
 out:
-	fclose(file);
-	rte_free(buf);
+	free(buf);
 	return err;
 }
 
-#undef ice_access
-
 static void
 ice_base_queue_get(struct ice_pf *pf)
 {
diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
index 2ee88fbfc7..879da7ef59 100644
--- a/drivers/net/nfp/nfp_net.c
+++ b/drivers/net/nfp/nfp_net.c
@@ -29,6 +29,7 @@
 #include <rte_alarm.h>
 #include <rte_spinlock.h>
 #include <rte_service_component.h>
+#include <rte_firmware.h>
 
 #include "nfpcore/nfp_cpp.h"
 #include "nfpcore/nfp_nffw.h"
@@ -3366,12 +3367,10 @@ static int
 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 {
 	struct nfp_cpp *cpp = nsp->cpp;
-	int fw_f;
-	char *fw_buf;
+	void *fw_buf;
 	char fw_name[125];
 	char serial[40];
-	struct stat file_stat;
-	off_t fsize, bytes;
+	size_t fsize;
 
 	/* Looking for firmware file in order of priority */
 
@@ -3384,66 +3383,34 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 
 	snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 			serial);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Then try the PCI name */
 	snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 			dev->device.name);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Finally try the card type and media */
 	snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f < 0) {
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
 		PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 		return -ENOENT;
 	}
 
-read_fw:
-	if (fstat(fw_f, &file_stat) < 0) {
-		PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
-		close(fw_f);
-		return -ENOENT;
-	}
-
-	fsize = file_stat.st_size;
-	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
-			    fw_name, (uint64_t)fsize);
-
-	fw_buf = malloc((size_t)fsize);
-	if (!fw_buf) {
-		PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
-		close(fw_f);
-		return -ENOMEM;
-	}
-	memset(fw_buf, 0, fsize);
-
-	bytes = read(fw_f, fw_buf, fsize);
-	if (bytes != fsize) {
-		PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
-				   "Just %" PRIu64 " of %" PRIu64 " bytes read",
-				   (uint64_t)bytes, (uint64_t)fsize);
-		free(fw_buf);
-		close(fw_f);
-		return -EIO;
-	}
+load_fw:
+	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
+		fw_name, fsize);
 
 	PMD_DRV_LOG(INFO, "Uploading the firmware ...");
-	nfp_nsp_load_fw(nsp, fw_buf, bytes);
+	nfp_nsp_load_fw(nsp, fw_buf, fsize);
 	PMD_DRV_LOG(INFO, "Done");
 
 	free(fw_buf);
-	close(fw_f);
-
 	return 0;
 }
 
diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
index caa9d1d4f6..504e2c66a0 100644
--- a/drivers/net/qede/qede_main.c
+++ b/drivers/net/qede/qede_main.c
@@ -5,7 +5,9 @@
  */
 
 #include <limits.h>
+
 #include <rte_alarm.h>
+#include <rte_firmware.h>
 #include <rte_string_fns.h>
 
 #include "qede_ethdev.h"
@@ -127,51 +129,40 @@ static void qed_free_stream_mem(struct ecore_dev *edev)
 #ifdef CONFIG_ECORE_BINARY_FW
 static int qed_load_firmware_data(struct ecore_dev *edev)
 {
-	int fd;
-	struct stat st;
 	const char *fw = RTE_LIBRTE_QEDE_FW;
+	void *buf;
+	size_t bufsz;
+	int ret;
 
 	if (strcmp(fw, "") == 0)
 		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
 	else
 		strcpy(qede_fw_file, fw);
 
-	fd = open(qede_fw_file, O_RDONLY);
-	if (fd < 0) {
-		DP_ERR(edev, "Can't open firmware file\n");
-		return -ENOENT;
-	}
-
-	if (fstat(fd, &st) < 0) {
-		DP_ERR(edev, "Can't stat firmware file\n");
-		close(fd);
+	if (rte_firmware_read(qede_fw_file, &buf, &bufsz) < 0) {
+		DP_ERR(edev, "Can't read firmware data: %s\n", qede_fw_file);
 		return -1;
 	}
 
-	edev->firmware = rte_zmalloc("qede_fw", st.st_size,
-				    RTE_CACHE_LINE_SIZE);
+	edev->firmware = rte_zmalloc("qede_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!edev->firmware) {
 		DP_ERR(edev, "Can't allocate memory for firmware\n");
-		close(fd);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto out;
 	}
 
-	if (read(fd, edev->firmware, st.st_size) != st.st_size) {
-		DP_ERR(edev, "Can't read firmware data\n");
-		close(fd);
-		return -1;
-	}
-
-	edev->fw_len = st.st_size;
+	memcpy(edev->firmware, buf, bufsz);
+	edev->fw_len = bufsz;
 	if (edev->fw_len < 104) {
 		DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n",
 			  edev->fw_len);
-		close(fd);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
-
-	close(fd);
-	return 0;
+	ret = 0;
+out:
+	free(buf);
+	return ret;
 }
 #endif
 
diff --git a/lib/eal/include/rte_firmware.h b/lib/eal/include/rte_firmware.h
new file mode 100644
index 0000000000..3389e60ca0
--- /dev/null
+++ b/lib/eal/include/rte_firmware.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifndef __RTE_FIRMWARE_H__
+#define __RTE_FIRMWARE_H__
+
+#include <sys/types.h>
+
+#include <rte_compat.h>
+
+/**
+ * Load a firmware in a dynamically allocated buffer, dealing with compressed
+ * files if libarchive is available.
+ *
+ * @param[in] name
+ *      Firmware filename to load.
+ * @param[out] buf
+ *      Buffer allocated by this function. If this function succeeds, the
+ *      caller is responsible for calling free() on this buffer.
+ * @param[out] bufsz
+ *      Size of the data in the buffer.
+ *
+ * @return
+ *      0 if successful.
+ *      Negative otherwise, buf and bufsize contents are invalid.
+ */
+__rte_internal
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz);
+
+#endif
diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
new file mode 100644
index 0000000000..2463d3b52c
--- /dev/null
+++ b/lib/eal/unix/eal_firmware.c
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifdef RTE_HAS_LIBARCHIVE
+#include <archive.h>
+#endif
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_firmware.h>
+#include <rte_log.h>
+
+#ifdef RTE_HAS_LIBARCHIVE
+
+struct firmware_read_ctx {
+	struct archive *a;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
+{
+	struct archive_entry *e;
+
+	ctx->a = archive_read_new();
+	if (ctx->a == NULL)
+		return -1;
+	if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK ||
+			archive_read_support_filter_xz(ctx->a) != ARCHIVE_OK ||
+			archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK ||
+			archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) {
+		archive_read_free(ctx->a);
+		ctx->a = NULL;
+		return -1;
+	}
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return archive_read_data(ctx->a, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	archive_read_free(ctx->a);
+	ctx->a = NULL;
+}
+
+#else /* !RTE_HAS_LIBARCHIVE */
+
+struct firmware_read_ctx {
+	int fd;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name,
+	__rte_unused size_t blocksize)
+{
+	ctx->fd = open(name, O_RDONLY);
+	if (ctx->fd < 0)
+		return -1;
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return read(ctx->fd, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	close(ctx->fd);
+	ctx->fd = -1;
+}
+
+#endif /* !RTE_HAS_LIBARCHIVE */
+
+static int
+firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	const size_t blocksize = 4096;
+	struct firmware_read_ctx ctx;
+	int ret = -1;
+	int err;
+
+	*buf = NULL;
+	*bufsz = 0;
+
+	if (firmware_open(&ctx, name, blocksize) < 0)
+		return -1;
+
+	do {
+		void *tmp;
+
+		tmp = realloc(*buf, *bufsz + blocksize);
+		if (tmp == NULL) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*buf = tmp;
+
+		err = firmware_read_block(&ctx, RTE_PTR_ADD(*buf, *bufsz), blocksize);
+		if (err < 0) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*bufsz += err;
+
+	} while (err != 0);
+
+	ret = 0;
+out:
+	firmware_close(&ctx);
+	return ret;
+}
+
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	char path[PATH_MAX];
+	int ret;
+
+	ret = firmware_read(name, buf, bufsz);
+	if (ret < 0) {
+		snprintf(path, sizeof(path), "%s.xz", name);
+		path[PATH_MAX - 1] = '\0';
+#ifndef RTE_HAS_LIBARCHIVE
+		if (access(path, F_OK) == 0) {
+			RTE_LOG(WARNING, EAL, "libarchive not linked, %s cannot be decompressed\n",
+				path);
+		}
+#else
+		ret = firmware_read(path, buf, bufsz);
+#endif
+	}
+	return ret;
+}
diff --git a/lib/eal/unix/meson.build b/lib/eal/unix/meson.build
index dc711b4240..e3ecd3e956 100644
--- a/lib/eal/unix/meson.build
+++ b/lib/eal/unix/meson.build
@@ -5,5 +5,6 @@ sources += files(
         'eal_file.c',
         'eal_unix_memory.c',
         'eal_unix_timer.c',
+        'eal_firmware.c',
         'rte_thread.c',
 )
diff --git a/lib/eal/version.map b/lib/eal/version.map
index fe5c3dac98..2df65c6903 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -428,6 +428,7 @@ EXPERIMENTAL {
 INTERNAL {
 	global:
 
+	rte_firmware_read;
 	rte_mem_lock;
 	rte_mem_map;
 	rte_mem_page_size;
diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
index 8483f6b02c..6fe2bdd282 100644
--- a/lib/eal/windows/eal.c
+++ b/lib/eal/windows/eal.c
@@ -21,6 +21,7 @@
 #include <eal_private.h>
 #include <rte_service_component.h>
 #include <rte_vfio.h>
+#include <rte_firmware.h>
 
 #include "eal_hugepages.h"
 #include "eal_trace.h"
@@ -465,3 +466,11 @@ rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 {
 	return -1;
 }
+
+int
+rte_firmware_read(__rte_unused const char *name,
+			__rte_unused void **buf,
+			__rte_unused size_t *bufsz)
+{
+	return -1;
+}
-- 
2.23.0

[dpdk-dev] [PATCH v5 0/2] Support compressed firmwares

[David Marchand]

Fedora 34 only provides compressed firmwares.

Introduce an internal driver helper to handle transparently compression.

I chose libarchive for decompressing as it seems widely available and
DPDK had used it in the past.

Windows support only matters for net/ice and firmware loading was skipped
in this driver before this series. Since I don't know if/how we want to
load firmwares on Windows, I let an empty stub for this OS.

This series has been compile tested on Linux (I'll trust the CI for
others OSes).
I only tested basic init with a net/ice device (no DCF test).

So please drivers maintainers, check nothing is broken.


-- 
David Marchand

Changes since v4:
- unexport header,
- s/firmwares/firmware/ from Bruce,

Changes since v3:
- add release note update,

Changes since v2:
- update comment on libarchive link dependency,

Changes since v1:
- address comments on patch2,

David Marchand (2):
  net/ice: factorize firmware loading
  eal: handle compressed firmware

 .github/workflows/build.yml            |   1 +
 .travis.yml                            |   1 +
 config/meson.build                     |  10 ++
 doc/guides/rel_notes/release_21_08.rst |   6 +
 drivers/net/bnx2x/bnx2x.c              |  36 ++---
 drivers/net/ice/base/ice_osdep.h       |   6 -
 drivers/net/ice/ice_dcf_parent.c       |  97 ++------------
 drivers/net/ice/ice_ethdev.c           | 176 ++++++++++---------------
 drivers/net/ice/ice_ethdev.h           |   3 +-
 drivers/net/nfp/nfp_net.c              |  58 ++------
 drivers/net/qede/qede_main.c           |  46 +++----
 lib/eal/common/eal_firmware.h          |  32 +++++
 lib/eal/unix/eal_firmware.c            | 150 +++++++++++++++++++++
 lib/eal/unix/meson.build               |   1 +
 lib/eal/version.map                    |   1 +
 lib/eal/windows/eal.c                  |   9 ++
 16 files changed, 335 insertions(+), 298 deletions(-)
 create mode 100644 lib/eal/common/eal_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

-- 
2.23.0

[dpdk-dev] [PATCH v5 1/2] net/ice: factorize firmware loading

[David Marchand]

Both "normal" and "dcf" inits have their copy of some firmware loading
code.

The DSN query is moved in specific parts for the "normal" and "dcf" init.

A common helper ice_load_pkg is then introduced and takes an adapter
pointer as its main input.

This helper takes care of finding the right firmware file and loading
it.
The adapter active_pkg_type field is set by this helper.

The ice_access macro is removed from the osdep.h header: osdep.h should
only hosts wrappers for base driver code.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Acked-by: Haiyue Wang <haiyue.wang@intel.com>
---
 drivers/net/ice/base/ice_osdep.h |   6 --
 drivers/net/ice/ice_dcf_parent.c |  97 ++-----------------
 drivers/net/ice/ice_ethdev.c     | 161 +++++++++++++++----------------
 drivers/net/ice/ice_ethdev.h     |   3 +-
 4 files changed, 88 insertions(+), 179 deletions(-)

diff --git a/drivers/net/ice/base/ice_osdep.h b/drivers/net/ice/base/ice_osdep.h
index 878c5597d4..78093adb00 100644
--- a/drivers/net/ice/base/ice_osdep.h
+++ b/drivers/net/ice/base/ice_osdep.h
@@ -74,12 +74,6 @@ typedef uint64_t        s64;
 #define min(a, b) RTE_MIN(a, b)
 #define max(a, b) RTE_MAX(a, b)
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 #define FIELD_SIZEOF(t, f) RTE_SIZEOF_FIELD(t, f)
 #define ARRAY_SIZE(arr) RTE_DIM(arr)
 
diff --git a/drivers/net/ice/ice_dcf_parent.c b/drivers/net/ice/ice_dcf_parent.c
index c59cd0bef9..020f764671 100644
--- a/drivers/net/ice/ice_dcf_parent.c
+++ b/drivers/net/ice/ice_dcf_parent.c
@@ -372,13 +372,14 @@ static void ice_dcf_uninit_parent_hw(struct ice_hw *hw)
 }
 
 static int
-ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
+ice_dcf_load_pkg(struct ice_adapter *adapter)
 {
 	struct ice_dcf_adapter *dcf_adapter =
-			container_of(hw, struct ice_dcf_adapter, parent.hw);
+			container_of(&adapter->hw, struct ice_dcf_adapter, parent.hw);
 	struct virtchnl_pkg_info pkg_info;
 	struct dcf_virtchnl_cmd vc_cmd;
-	uint64_t dsn;
+	bool use_dsn;
+	uint64_t dsn = 0;
 
 	vc_cmd.v_op = VIRTCHNL_OP_DCF_GET_PKG_INFO;
 	vc_cmd.req_msglen = 0;
@@ -386,90 +387,11 @@ ice_dcf_request_pkg_name(struct ice_hw *hw, char *pkg_name)
 	vc_cmd.rsp_buflen = sizeof(pkg_info);
 	vc_cmd.rsp_msgbuf = (uint8_t *)&pkg_info;
 
-	if (ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd))
-		goto pkg_file_direct;
+	use_dsn = ice_dcf_execute_virtchnl_cmd(&dcf_adapter->real_hw, &vc_cmd) == 0;
+	if (use_dsn)
+		rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
 
-	rte_memcpy(&dsn, pkg_info.dsn, sizeof(dsn));
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_UPDATES "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name, ICE_MAX_PKG_FILENAME_SIZE,
-		 ICE_PKG_FILE_SEARCH_PATH_DEFAULT "ice-%016llx.pkg",
-		 (unsigned long long)dsn);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-pkg_file_direct:
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_UPDATES);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	snprintf(pkg_name,
-		 ICE_MAX_PKG_FILENAME_SIZE, "%s", ICE_PKG_FILE_DEFAULT);
-	if (!ice_access(pkg_name, 0))
-		return 0;
-
-	return -1;
-}
-
-static int
-ice_dcf_load_pkg(struct ice_hw *hw)
-{
-	char pkg_name[ICE_MAX_PKG_FILENAME_SIZE];
-	uint8_t *pkg_buf;
-	uint32_t buf_len;
-	struct stat st;
-	FILE *fp;
-	int err;
-
-	if (ice_dcf_request_pkg_name(hw, pkg_name)) {
-		PMD_INIT_LOG(ERR, "Failed to locate the package file");
-		return -ENOENT;
-	}
-
-	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_name);
-
-	err = stat(pkg_name, &st);
-	if (err) {
-		PMD_INIT_LOG(ERR, "Failed to get file status");
-		return err;
-	}
-
-	buf_len = st.st_size;
-	pkg_buf = rte_malloc(NULL, buf_len, 0);
-	if (!pkg_buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buffer of size %u for package",
-			     buf_len);
-		return -1;
-	}
-
-	fp = fopen(pkg_name, "rb");
-	if (!fp)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s", pkg_name);
-		err = -1;
-		goto ret;
-	}
-
-	err = fread(pkg_buf, buf_len, 1, fp);
-	fclose(fp);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data");
-		err = -1;
-		goto ret;
-	}
-
-	err = ice_copy_and_init_pkg(hw, pkg_buf, buf_len);
-	if (err)
-		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d", err);
-
-ret:
-	rte_free(pkg_buf);
-	return err;
+	return ice_load_pkg(adapter, use_dsn, dsn);
 }
 
 int
@@ -518,13 +440,12 @@ ice_dcf_init_parent_adapter(struct rte_eth_dev *eth_dev)
 		}
 	}
 
-	err = ice_dcf_load_pkg(parent_hw);
+	err = ice_dcf_load_pkg(parent_adapter);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to load package with error %d",
 			     err);
 		goto uninit_hw;
 	}
-	parent_adapter->active_pkg_type = ice_load_pkg_type(parent_hw);
 
 	parent_adapter->pf.main_vsi->idx = hw->num_vfs;
 	ice_dcf_update_pf_vsi_map(parent_hw,
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 63f735d1ff..6c3d2d9156 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -1651,57 +1651,7 @@ ice_pf_setup(struct ice_pf *pf)
 	return 0;
 }
 
-/*
- * Extract device serial number from PCIe Configuration Space and
- * determine the pkg file path according to the DSN.
- */
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int
-ice_pkg_file_search_path(struct rte_pci_device *pci_dev, char *pkg_file)
-{
-	off_t pos;
-	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
-	uint32_t dsn_low, dsn_high;
-	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
-
-	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
-
-	if (pos) {
-		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		if (rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
-			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
-			return -1;
-		}
-		snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
-			 "ice-%08x%08x.pkg", dsn_high, dsn_low);
-	} else {
-		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
-		goto fail_dsn;
-	}
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
-		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
-		return 0;
-
-fail_dsn:
-	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
-		return 0;
-	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	return 0;
-}
-#endif
-
-enum ice_pkg_type
+static enum ice_pkg_type
 ice_load_pkg_type(struct ice_hw *hw)
 {
 	enum ice_pkg_type package_type;
@@ -1725,37 +1675,62 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifndef RTE_EXEC_ENV_WINDOWS
-static int ice_load_pkg(struct rte_eth_dev *dev)
+#ifdef RTE_EXEC_ENV_WINDOWS
+#define ice_access _access
+#else
+#define ice_access access
+#endif
+
+int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
-	struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
+	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
 	int err;
-	uint8_t *buf;
+	uint8_t *buf = NULL;
 	int buf_len;
 	FILE *file;
 	struct stat fstat;
-	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
-	struct ice_adapter *ad =
-		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 
-	err = ice_pkg_file_search_path(pci_dev, pkg_file);
-	if (err) {
+	if (!use_dsn)
+		goto no_dsn;
+
+	memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
+	snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
+		"ice-%016" PRIx64 ".pkg", dsn);
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
+		ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+		goto load_fw;
+
+no_dsn:
+	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
+	if (!ice_access(pkg_file, 0))
+		goto load_fw;
+	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
+	if (ice_access(pkg_file, 0)) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
-		return err;
+		return -1;
 	}
 
+load_fw:
 	file = fopen(pkg_file, "rb");
 	if (!file)  {
 		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
 		return -1;
 	}
 
+	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
+
 	err = stat(pkg_file, &fstat);
 	if (err) {
 		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		fclose(file);
-		return err;
+		goto out;
 	}
 
 	buf_len = fstat.st_size;
@@ -1764,44 +1739,33 @@ static int ice_load_pkg(struct rte_eth_dev *dev)
 	if (!buf) {
 		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
 				buf_len);
-		fclose(file);
-		return -1;
+		err = -1;
+		goto out;
 	}
 
 	err = fread(buf, buf_len, 1, file);
 	if (err != 1) {
 		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		fclose(file);
 		err = -1;
-		goto fail_exit;
+		goto out;
 	}
 
-	fclose(file);
-
 	err = ice_copy_and_init_pkg(hw, buf, buf_len);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
-		goto fail_exit;
+		goto out;
 	}
 
 	/* store the loaded pkg type info */
-	ad->active_pkg_type = ice_load_pkg_type(hw);
+	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
-	err = ice_init_hw_tbls(hw);
-	if (err) {
-		PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", err);
-		goto fail_init_tbls;
-	}
-
-	return 0;
-
-fail_init_tbls:
-	rte_free(hw->pkg_copy);
-fail_exit:
+out:
+	fclose(file);
 	rte_free(buf);
 	return err;
 }
-#endif
+
+#undef ice_access
 
 static void
 ice_base_queue_get(struct ice_pf *pf)
@@ -2056,6 +2020,12 @@ ice_dev_init(struct rte_eth_dev *dev)
 		ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct ice_vsi *vsi;
 	int ret;
+#ifndef RTE_EXEC_ENV_WINDOWS
+	off_t pos;
+	uint32_t dsn_low, dsn_high;
+	uint64_t dsn;
+	bool use_dsn;
+#endif
 
 	dev->dev_ops = &ice_eth_dev_ops;
 	dev->rx_queue_count = ice_rx_queue_count;
@@ -2106,7 +2076,30 @@ ice_dev_init(struct rte_eth_dev *dev)
 	}
 
 #ifndef RTE_EXEC_ENV_WINDOWS
-	ret = ice_load_pkg(dev);
+	use_dsn = false;
+	dsn = 0;
+	pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
+	if (pos) {
+		if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0 ||
+				rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
+			PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
+		} else {
+			use_dsn = true;
+			dsn = (uint64_t)dsn_high << 32 | dsn_low;
+		}
+	} else {
+		PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
+	}
+
+	ret = ice_load_pkg(pf->adapter, use_dsn, dsn);
+	if (ret == 0) {
+		ret = ice_init_hw_tbls(hw);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", ret);
+			rte_free(hw->pkg_copy);
+		}
+	}
+
 	if (ret) {
 		if (ad->devargs.safe_mode_support == 0) {
 			PMD_INIT_LOG(ERR, "Failed to load the DDP package,"
diff --git a/drivers/net/ice/ice_ethdev.h b/drivers/net/ice/ice_ethdev.h
index 7f3c26fb6f..edafdf168b 100644
--- a/drivers/net/ice/ice_ethdev.h
+++ b/drivers/net/ice/ice_ethdev.h
@@ -534,7 +534,8 @@ struct ice_vsi_vlan_pvid_info {
 #define ICE_PF_TO_ETH_DEV(pf) \
 	(((struct ice_pf *)pf)->adapter->eth_dev)
 
-enum ice_pkg_type ice_load_pkg_type(struct ice_hw *hw);
+int
+ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn);
 struct ice_vsi *
 ice_setup_vsi(struct ice_pf *pf, enum ice_vsi_type type);
 int
-- 
2.23.0

[dpdk-dev] [PATCH v5 2/2] eal: handle compressed firmware

[David Marchand]

Introduce an internal firmware loading helper to remove code duplication
in our drivers and handle xz compressed firmware by calling libarchive.

This helper tries to look for .xz suffixes so that drivers are not aware
the firmware has been compressed.

libarchive is set as an optional dependency: without libarchive, a
runtime warning is emitted so that users know there is a compressed
firmware.

Windows implementation is left as an empty stub.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
Acked-by: Aaron Conole <aconole@redhat.com>
Tested-by: Haiyue Wang <haiyue.wang@intel.com>
---
Changes since v4:
- moved header to internal,
- s/firmwares/firmware/g,

Changes since v3:
- added release note update,
- updated warning message when uncompressing is unavailable,

Changes since v2:
- added a comment on libarchive link dependency,

Changes since v1:
- used pkg-config for libarchive detection,
- updated doxygen annotations,
- added internal helpers in eal_firmware.c to enhance readability,
- dropped whitespace damage in version.map,

---
 .github/workflows/build.yml            |   1 +
 .travis.yml                            |   1 +
 config/meson.build                     |  10 ++
 doc/guides/rel_notes/release_21_08.rst |   6 +
 drivers/net/bnx2x/bnx2x.c              |  36 +++---
 drivers/net/ice/ice_ethdev.c           |  61 +++-------
 drivers/net/nfp/nfp_net.c              |  58 +++-------
 drivers/net/qede/qede_main.c           |  46 ++++----
 lib/eal/common/eal_firmware.h          |  32 ++++++
 lib/eal/unix/eal_firmware.c            | 150 +++++++++++++++++++++++++
 lib/eal/unix/meson.build               |   1 +
 lib/eal/version.map                    |   1 +
 lib/eal/windows/eal.c                  |   9 ++
 13 files changed, 270 insertions(+), 142 deletions(-)
 create mode 100644 lib/eal/common/eal_firmware.h
 create mode 100644 lib/eal/unix/eal_firmware.c

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7c4d6dcdbf..7dac20ddeb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -93,6 +93,7 @@ jobs:
       run: sudo apt install -y ccache libnuma-dev python3-setuptools
         python3-wheel python3-pip python3-pyelftools ninja-build libbsd-dev
         libpcap-dev libibverbs-dev libcrypto++-dev libfdt-dev libjansson-dev
+        libarchive-dev
     - name: Install libabigail build dependencies if no cache is available
       if: env.ABI_CHECKS == 'true' && steps.libabigail-cache.outputs.cache-hit != 'true'
       run: sudo apt install -y autoconf automake libtool pkg-config libxml2-dev
diff --git a/.travis.yml b/.travis.yml
index 5b702cc9bb..23067d9e3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,7 @@ addons:
     packages: &required_packages
       - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
       - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
+      - [libarchive-dev]
 
 _aarch64_packages: &aarch64_packages
   - *required_packages
diff --git a/config/meson.build b/config/meson.build
index 017bb2efbb..639aa74e12 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -172,6 +172,16 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
     dpdk_extra_ldflags += '-lexecinfo'
 endif
 
+libarchive = dependency('libarchive', required: false, method: 'pkg-config')
+if libarchive.found()
+    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
+    # Push libarchive link dependency at the project level to support
+    # statically linking dpdk apps. Details at:
+    # https://inbox.dpdk.org/dev/20210605004024.660267a1@sovereign/
+    add_project_link_arguments('-larchive', language: 'c')
+    dpdk_extra_ldflags += '-larchive'
+endif
+
 # check for libbsd
 libbsd = dependency('libbsd', required: false, method: 'pkg-config')
 if libbsd.found()
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index cd02820e68..96bd907e53 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -61,6 +61,12 @@ New Features
   representing sub-domains of functionality. Each auxiliary device
   represents a part of its parent functionality.
 
+* **Added XZ compressed firmware support.**
+
+  Using ``rte_firmware_read``, a driver can now handle XZ compressed firmware
+  in a transparent way, with EAL uncompressing using libarchive if this library
+  is available when building DPDK.
+
 * **Updated Intel iavf driver.**
 
   * Added Tx QoS VF queue TC mapping.
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c
index 654878d9de..7ee805bd0d 100644
--- a/drivers/net/bnx2x/bnx2x.c
+++ b/drivers/net/bnx2x/bnx2x.c
@@ -26,9 +26,12 @@
 #include <arpa/inet.h>
 #include <fcntl.h>
 #include <zlib.h>
+
 #include <rte_bitops.h>
 #include <rte_string_fns.h>
 
+#include "eal_firmware.h"
+
 #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
 #define BNX2X_PMD_VERSION_MAJOR 1
 #define BNX2X_PMD_VERSION_MINOR 1
@@ -9655,44 +9658,33 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc)
 void bnx2x_load_firmware(struct bnx2x_softc *sc)
 {
 	const char *fwname;
-	int f;
-	struct stat st;
+	void *buf;
+	size_t bufsz;
 
 	fwname = sc->devinfo.device_id == CHIP_NUM_57711
 		? FW_NAME_57711 : FW_NAME_57810;
-	f = open(fwname, O_RDONLY);
-	if (f < 0) {
+	if (rte_firmware_read(fwname, &buf, &bufsz) != 0) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
 		return;
 	}
 
-	if (fstat(f, &st) < 0) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
-		close(f);
-		return;
-	}
-
-	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
+	sc->firmware = rte_zmalloc("bnx2x_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!sc->firmware) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
-		close(f);
-		return;
+		goto out;
 	}
 
-	if (read(f, sc->firmware, st.st_size) != st.st_size) {
-		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
-		close(f);
-		return;
-	}
-	close(f);
-
-	sc->fw_len = st.st_size;
+	sc->fw_len = bufsz;
 	if (sc->fw_len < FW_HEADER_LEN) {
 		PMD_DRV_LOG(NOTICE, sc,
 			    "Invalid fw size: %" PRIu64, sc->fw_len);
-		return;
+		goto out;
 	}
+
+	memcpy(sc->firmware, buf, sc->fw_len);
 	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
+out:
+	free(buf);
 }
 
 static void
diff --git a/drivers/net/ice/ice_ethdev.c b/drivers/net/ice/ice_ethdev.c
index 6c3d2d9156..aec19b6521 100644
--- a/drivers/net/ice/ice_ethdev.c
+++ b/drivers/net/ice/ice_ethdev.c
@@ -12,6 +12,8 @@
 
 #include <rte_tailq.h>
 
+#include "eal_firmware.h"
+
 #include "base/ice_sched.h"
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
@@ -1675,22 +1677,14 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
 	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
 	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
+	void *buf;
+	size_t bufsz;
 	int err;
-	uint8_t *buf = NULL;
-	int buf_len;
-	FILE *file;
-	struct stat fstat;
 
 	if (!use_dsn)
 		goto no_dsn;
@@ -1700,57 +1694,31 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 		"ice-%016" PRIx64 ".pkg", dsn);
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 no_dsn:
 	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
+
 	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	if (ice_access(pkg_file, 0)) {
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
 		return -1;
 	}
 
 load_fw:
-	file = fopen(pkg_file, "rb");
-	if (!file)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
-		return -1;
-	}
-
 	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
 
-	err = stat(pkg_file, &fstat);
-	if (err) {
-		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		goto out;
-	}
-
-	buf_len = fstat.st_size;
-	buf = rte_malloc(NULL, buf_len, 0);
-
-	if (!buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
-				buf_len);
-		err = -1;
-		goto out;
-	}
-
-	err = fread(buf, buf_len, 1, file);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		err = -1;
-		goto out;
-	}
-
-	err = ice_copy_and_init_pkg(hw, buf, buf_len);
+	err = ice_copy_and_init_pkg(hw, buf, bufsz);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
 		goto out;
@@ -1760,13 +1728,10 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
 out:
-	fclose(file);
-	rte_free(buf);
+	free(buf);
 	return err;
 }
 
-#undef ice_access
-
 static void
 ice_base_queue_get(struct ice_pf *pf)
 {
diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c
index 2ee88fbfc7..6950d6f714 100644
--- a/drivers/net/nfp/nfp_net.c
+++ b/drivers/net/nfp/nfp_net.c
@@ -30,6 +30,8 @@
 #include <rte_spinlock.h>
 #include <rte_service_component.h>
 
+#include "eal_firmware.h"
+
 #include "nfpcore/nfp_cpp.h"
 #include "nfpcore/nfp_nffw.h"
 #include "nfpcore/nfp_hwinfo.h"
@@ -3366,12 +3368,10 @@ static int
 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 {
 	struct nfp_cpp *cpp = nsp->cpp;
-	int fw_f;
-	char *fw_buf;
+	void *fw_buf;
 	char fw_name[125];
 	char serial[40];
-	struct stat file_stat;
-	off_t fsize, bytes;
+	size_t fsize;
 
 	/* Looking for firmware file in order of priority */
 
@@ -3384,66 +3384,34 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 
 	snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 			serial);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Then try the PCI name */
 	snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 			dev->device.name);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f >= 0)
-		goto read_fw;
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
+		goto load_fw;
 
 	/* Finally try the card type and media */
 	snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
-	if (fw_f < 0) {
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
 		PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 		return -ENOENT;
 	}
 
-read_fw:
-	if (fstat(fw_f, &file_stat) < 0) {
-		PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
-		close(fw_f);
-		return -ENOENT;
-	}
-
-	fsize = file_stat.st_size;
-	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
-			    fw_name, (uint64_t)fsize);
-
-	fw_buf = malloc((size_t)fsize);
-	if (!fw_buf) {
-		PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
-		close(fw_f);
-		return -ENOMEM;
-	}
-	memset(fw_buf, 0, fsize);
-
-	bytes = read(fw_f, fw_buf, fsize);
-	if (bytes != fsize) {
-		PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
-				   "Just %" PRIu64 " of %" PRIu64 " bytes read",
-				   (uint64_t)bytes, (uint64_t)fsize);
-		free(fw_buf);
-		close(fw_f);
-		return -EIO;
-	}
+load_fw:
+	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
+		fw_name, fsize);
 
 	PMD_DRV_LOG(INFO, "Uploading the firmware ...");
-	nfp_nsp_load_fw(nsp, fw_buf, bytes);
+	nfp_nsp_load_fw(nsp, fw_buf, fsize);
 	PMD_DRV_LOG(INFO, "Done");
 
 	free(fw_buf);
-	close(fw_f);
-
 	return 0;
 }
 
diff --git a/drivers/net/qede/qede_main.c b/drivers/net/qede/qede_main.c
index caa9d1d4f6..2d1f70693a 100644
--- a/drivers/net/qede/qede_main.c
+++ b/drivers/net/qede/qede_main.c
@@ -5,9 +5,12 @@
  */
 
 #include <limits.h>
+
 #include <rte_alarm.h>
 #include <rte_string_fns.h>
 
+#include "eal_firmware.h"
+
 #include "qede_ethdev.h"
 /* ######### DEBUG ###########*/
 #include "qede_debug.h"
@@ -127,51 +130,40 @@ static void qed_free_stream_mem(struct ecore_dev *edev)
 #ifdef CONFIG_ECORE_BINARY_FW
 static int qed_load_firmware_data(struct ecore_dev *edev)
 {
-	int fd;
-	struct stat st;
 	const char *fw = RTE_LIBRTE_QEDE_FW;
+	void *buf;
+	size_t bufsz;
+	int ret;
 
 	if (strcmp(fw, "") == 0)
 		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
 	else
 		strcpy(qede_fw_file, fw);
 
-	fd = open(qede_fw_file, O_RDONLY);
-	if (fd < 0) {
-		DP_ERR(edev, "Can't open firmware file\n");
-		return -ENOENT;
-	}
-
-	if (fstat(fd, &st) < 0) {
-		DP_ERR(edev, "Can't stat firmware file\n");
-		close(fd);
+	if (rte_firmware_read(qede_fw_file, &buf, &bufsz) < 0) {
+		DP_ERR(edev, "Can't read firmware data: %s\n", qede_fw_file);
 		return -1;
 	}
 
-	edev->firmware = rte_zmalloc("qede_fw", st.st_size,
-				    RTE_CACHE_LINE_SIZE);
+	edev->firmware = rte_zmalloc("qede_fw", bufsz, RTE_CACHE_LINE_SIZE);
 	if (!edev->firmware) {
 		DP_ERR(edev, "Can't allocate memory for firmware\n");
-		close(fd);
-		return -ENOMEM;
-	}
-
-	if (read(fd, edev->firmware, st.st_size) != st.st_size) {
-		DP_ERR(edev, "Can't read firmware data\n");
-		close(fd);
-		return -1;
+		ret = -ENOMEM;
+		goto out;
 	}
 
-	edev->fw_len = st.st_size;
+	memcpy(edev->firmware, buf, bufsz);
+	edev->fw_len = bufsz;
 	if (edev->fw_len < 104) {
 		DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n",
 			  edev->fw_len);
-		close(fd);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
-
-	close(fd);
-	return 0;
+	ret = 0;
+out:
+	free(buf);
+	return ret;
 }
 #endif
 
diff --git a/lib/eal/common/eal_firmware.h b/lib/eal/common/eal_firmware.h
new file mode 100644
index 0000000000..96ec98458f
--- /dev/null
+++ b/lib/eal/common/eal_firmware.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifndef _EAL_FIRMWARE_H_
+#define _EAL_FIRMWARE_H_
+
+#include <sys/types.h>
+
+#include <rte_compat.h>
+
+/**
+ * Load a firmware in a dynamically allocated buffer, dealing with compressed
+ * files if libarchive is available.
+ *
+ * @param[in] name
+ *      Firmware filename to load.
+ * @param[out] buf
+ *      Buffer allocated by this function. If this function succeeds, the
+ *      caller is responsible for calling free() on this buffer.
+ * @param[out] bufsz
+ *      Size of the data in the buffer.
+ *
+ * @return
+ *      0 if successful.
+ *      Negative otherwise, buf and bufsize contents are invalid.
+ */
+__rte_internal
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz);
+
+#endif /* _EAL_FIRMWARE_H_ */
diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
new file mode 100644
index 0000000000..d1616b0bd9
--- /dev/null
+++ b/lib/eal/unix/eal_firmware.c
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifdef RTE_HAS_LIBARCHIVE
+#include <archive.h>
+#endif
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+
+#include "eal_firmware.h"
+
+#ifdef RTE_HAS_LIBARCHIVE
+
+struct firmware_read_ctx {
+	struct archive *a;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
+{
+	struct archive_entry *e;
+
+	ctx->a = archive_read_new();
+	if (ctx->a == NULL)
+		return -1;
+	if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK ||
+			archive_read_support_filter_xz(ctx->a) != ARCHIVE_OK ||
+			archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK ||
+			archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) {
+		archive_read_free(ctx->a);
+		ctx->a = NULL;
+		return -1;
+	}
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return archive_read_data(ctx->a, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	archive_read_free(ctx->a);
+	ctx->a = NULL;
+}
+
+#else /* !RTE_HAS_LIBARCHIVE */
+
+struct firmware_read_ctx {
+	int fd;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name,
+	__rte_unused size_t blocksize)
+{
+	ctx->fd = open(name, O_RDONLY);
+	if (ctx->fd < 0)
+		return -1;
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return read(ctx->fd, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	close(ctx->fd);
+	ctx->fd = -1;
+}
+
+#endif /* !RTE_HAS_LIBARCHIVE */
+
+static int
+firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	const size_t blocksize = 4096;
+	struct firmware_read_ctx ctx;
+	int ret = -1;
+	int err;
+
+	*buf = NULL;
+	*bufsz = 0;
+
+	if (firmware_open(&ctx, name, blocksize) < 0)
+		return -1;
+
+	do {
+		void *tmp;
+
+		tmp = realloc(*buf, *bufsz + blocksize);
+		if (tmp == NULL) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*buf = tmp;
+
+		err = firmware_read_block(&ctx, RTE_PTR_ADD(*buf, *bufsz), blocksize);
+		if (err < 0) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*bufsz += err;
+
+	} while (err != 0);
+
+	ret = 0;
+out:
+	firmware_close(&ctx);
+	return ret;
+}
+
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	char path[PATH_MAX];
+	int ret;
+
+	ret = firmware_read(name, buf, bufsz);
+	if (ret < 0) {
+		snprintf(path, sizeof(path), "%s.xz", name);
+		path[PATH_MAX - 1] = '\0';
+#ifndef RTE_HAS_LIBARCHIVE
+		if (access(path, F_OK) == 0) {
+			RTE_LOG(WARNING, EAL, "libarchive not linked, %s cannot be decompressed\n",
+				path);
+		}
+#else
+		ret = firmware_read(path, buf, bufsz);
+#endif
+	}
+	return ret;
+}
diff --git a/lib/eal/unix/meson.build b/lib/eal/unix/meson.build
index dc711b4240..e3ecd3e956 100644
--- a/lib/eal/unix/meson.build
+++ b/lib/eal/unix/meson.build
@@ -5,5 +5,6 @@ sources += files(
         'eal_file.c',
         'eal_unix_memory.c',
         'eal_unix_timer.c',
+        'eal_firmware.c',
         'rte_thread.c',
 )
diff --git a/lib/eal/version.map b/lib/eal/version.map
index fe5c3dac98..2df65c6903 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -428,6 +428,7 @@ EXPERIMENTAL {
 INTERNAL {
 	global:
 
+	rte_firmware_read;
 	rte_mem_lock;
 	rte_mem_map;
 	rte_mem_page_size;
diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
index 8483f6b02c..5413d4d87f 100644
--- a/lib/eal/windows/eal.c
+++ b/lib/eal/windows/eal.c
@@ -22,6 +22,7 @@
 #include <rte_service_component.h>
 #include <rte_vfio.h>
 
+#include "eal_firmware.h"
 #include "eal_hugepages.h"
 #include "eal_trace.h"
 #include "eal_log.h"
@@ -465,3 +466,11 @@ rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 {
 	return -1;
 }
+
+int
+rte_firmware_read(__rte_unused const char *name,
+			__rte_unused void **buf,
+			__rte_unused size_t *bufsz)
+{
+	return -1;
+}
-- 
2.23.0

Re: [dpdk-dev] [PATCH v5 0/2] Support compressed firmwares

[David Marchand]

On Wed, Jul 7, 2021 at 2:08 PM David Marchand <david.marchand@redhat.com> wrote:
>
> Fedora 34 only provides compressed firmwares.
>
> Introduce an internal driver helper to handle transparently compression.
>
> I chose libarchive for decompressing as it seems widely available and
> DPDK had used it in the past.
>
> Windows support only matters for net/ice and firmware loading was skipped
> in this driver before this series. Since I don't know if/how we want to
> load firmwares on Windows, I let an empty stub for this OS.
>
> This series has been compile tested on Linux (I'll trust the CI for
> others OSes).
> I only tested basic init with a net/ice device (no DCF test).
>
> So please drivers maintainers, check nothing is broken.
>
>
> --
> David Marchand
>
> Changes since v4:
> - unexport header,
> - s/firmwares/firmware/ from Bruce,
>
> Changes since v3:
> - add release note update,
>
> Changes since v2:
> - update comment on libarchive link dependency,
>
> Changes since v1:
> - address comments on patch2,
>
> David Marchand (2):
>   net/ice: factorize firmware loading
>   eal: handle compressed firmware

Series applied.


-- 
David Marchand