All of lore.kernel.org
 help / color / mirror / Atom feed
* [Buildroot] [PATCH 1/1] core: add option to override CPE and NVD base url
@ 2021-09-28 20:35 Ankur Tyagi
  2022-07-25 16:56 ` Arnout Vandecappelle
  0 siblings, 1 reply; 2+ messages in thread
From: Ankur Tyagi @ 2021-09-28 20:35 UTC (permalink / raw)
  To: buildroot; +Cc: Ankur Tyagi

It may be desirable to integrate "pkg-stats" as part of build pipeline
and usually build machines don't have internet access.
In those scenarios, private network can be used to fetch CPE dictionary
and NVD data (which are a pre-downloaded artifact).

This commit introduces config options BR2_CPEDB_BASE_URL and
BR2_NVD_BASE_URL which can point to those internal/private network url.
By default, they refer to NIST official url.

It also adds CPE and NVD base url arguments to pkg-stats which
are then passed to CVE and CPE class methods to fetch CPE dictionary
and NVD data.

Signed-off-by: Ankur Tyagi <ankur.tyagi@gallagher.com>
---
 Config.in                 | 14 ++++++++++++++
 Makefile                  |  4 +++-
 support/scripts/cpedb.py  | 12 +++++++-----
 support/scripts/cve.py    | 11 +++++------
 support/scripts/pkg-stats | 16 ++++++++++------
 5 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/Config.in b/Config.in
index 3db2c8dcab..a4224fce38 100644
--- a/Config.in
+++ b/Config.in
@@ -290,6 +290,20 @@ config BR2_CPAN_MIRROR
 	  The list of mirrors is available at:
 	  http://search.cpan.org/mirror
 
+config BR2_CPEDB_BASE_URL
+	string "CPE database download site base url"
+	default "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary"
+	help
+	  The following allows you to select your preferred
+	  mirror. By default, NIST official URL is used
+
+config BR2_NVD_BASE_URL
+	string "NVD download site base url"
+	default "https://nvd.nist.gov/feeds/json/cve"
+	help
+	  The following allows you to select your preferred
+	  mirror. By default, NIST official URL is used
+
 endif
 
 endmenu
diff --git a/Makefile b/Makefile
index c960b53a6d..acb2c76aea 100644
--- a/Makefile
+++ b/Makefile
@@ -930,7 +930,9 @@ pkg-stats:
 	$(TOPDIR)/support/scripts/pkg-stats -c \
 		--json $(O)/pkg-stats.json \
 		--html $(O)/pkg-stats.html \
-		--nvd-path $(DL_DIR)/buildroot-nvd
+		--nvd-path $(DL_DIR)/buildroot-nvd \
+		--cpedb-base-url $(BR2_CPEDB_BASE_URL) \
+		--nvd-base-url $(BR2_NVD_BASE_URL)
 
 .PHONY: missing-cpe
 missing-cpe:
diff --git a/support/scripts/cpedb.py b/support/scripts/cpedb.py
index f4daf56124..36846621c3 100644
--- a/support/scripts/cpedb.py
+++ b/support/scripts/cpedb.py
@@ -10,7 +10,7 @@ from xml.dom import minidom
 
 VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY']
 
-CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
+CPE_DICTIONARY = "official-cpe-dictionary_v2.3.xml.gz"
 
 ns = {
     '': 'http://cpe.mitre.org/dictionary/2.0',
@@ -88,20 +88,22 @@ class CPE:
 
 
 class CPEDB:
-    def __init__(self, nvd_path):
+    def __init__(self, nvd_path, cpedb_base_url):
         self.all_cpes = dict()
         self.all_cpes_no_version = dict()
         self.nvd_path = nvd_path
+        self.cpedb_base_url = cpedb_base_url
 
     def get_xml_dict(self):
         print("CPE: Setting up NIST dictionary")
         if not os.path.exists(os.path.join(self.nvd_path, "cpe")):
             os.makedirs(os.path.join(self.nvd_path, "cpe"))
 
-        cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL))
+        url = "%s/%s" % (self.cpedb_base_url, CPE_DICTIONARY)
+        cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(url))
         if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
-            print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
-            cpe_dict = requests.get(CPEDB_URL)
+            print("CPE: Fetching xml manifest from [" + url + "]")
+            cpe_dict = requests.get(url)
             open(cpe_dict_local, "wb").write(cpe_dict.content)
 
         print("CPE: Unzipping xml manifest...")
diff --git a/support/scripts/cve.py b/support/scripts/cve.py
index 13c29fabe0..0ff5fbe823 100755
--- a/support/scripts/cve.py
+++ b/support/scripts/cve.py
@@ -42,7 +42,6 @@ sys.path.append('utils/')
 
 NVD_START_YEAR = 2002
 NVD_JSON_VERSION = "1.1"
-NVD_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/" + NVD_JSON_VERSION
 
 ops = {
     '>=': operator.ge,
@@ -82,7 +81,7 @@ class CVE:
         self.nvd_cve = nvd_cve
 
     @staticmethod
-    def download_nvd_year(nvd_path, year):
+    def download_nvd_year(nvd_path, nvd_base_url, year):
         metaf = "nvdcve-%s-%s.meta" % (NVD_JSON_VERSION, year)
         path_metaf = os.path.join(nvd_path, metaf)
         jsonf_gz = "nvdcve-%s-%s.json.gz" % (NVD_JSON_VERSION, year)
@@ -94,7 +93,7 @@ class CVE:
             return path_jsonf_gz
 
         # If not, we download the meta file
-        url = "%s/%s" % (NVD_BASE_URL, metaf)
+        url = "%s/%s/%s" % (nvd_base_url, NVD_JSON_VERSION, metaf)
         print("Getting %s" % url)
         page_meta = requests.get(url)
         page_meta.raise_for_status()
@@ -110,7 +109,7 @@ class CVE:
                 return path_jsonf_gz
 
         # Grab the compressed JSON NVD, and write files to disk
-        url = "%s/%s" % (NVD_BASE_URL, jsonf_gz)
+        url = "%s/%s/%s" % (nvd_base_url, NVD_JSON_VERSION, jsonf_gz)
         print("Getting %s" % url)
         page_json = requests.get(url)
         page_json.raise_for_status()
@@ -119,14 +118,14 @@ class CVE:
         return path_jsonf_gz
 
     @classmethod
-    def read_nvd_dir(cls, nvd_dir):
+    def read_nvd_dir(cls, nvd_dir, nvd_base_url):
         """
         Iterate over all the CVEs contained in NIST Vulnerability Database
         feeds since NVD_START_YEAR. If the files are missing or outdated in
         nvd_dir, a fresh copy will be downloaded, and kept in .json.gz
         """
         for year in range(NVD_START_YEAR, datetime.datetime.now().year + 1):
-            filename = CVE.download_nvd_year(nvd_dir, year)
+            filename = CVE.download_nvd_year(nvd_dir, nvd_base_url, year)
             try:
                 content = ijson.items(gzip.GzipFile(filename), 'CVE_Items.item')
             except:  # noqa: E722
diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats
index cc91d13167..5192d36857 100755
--- a/support/scripts/pkg-stats
+++ b/support/scripts/pkg-stats
@@ -583,7 +583,7 @@ def check_package_cve_affects(cve, cpe_product_pkgs):
                 pkg.cves.append(cve.identifier)
 
 
-def check_package_cves(nvd_path, packages):
+def check_package_cves(nvd_path, nvd_base_url, packages):
     if not os.path.isdir(nvd_path):
         os.makedirs(nvd_path)
 
@@ -601,7 +601,7 @@ def check_package_cves(nvd_path, packages):
         else:
             cpe_product_pkgs[pkg.name].append(pkg)
 
-    for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
+    for cve in cvecheck.CVE.read_nvd_dir(nvd_path, nvd_base_url):
         check_package_cve_affects(cve, cpe_product_pkgs)
 
     for pkg in packages:
@@ -612,8 +612,8 @@ def check_package_cves(nvd_path, packages):
                 pkg.status['cve'] = ("ok", "not affected by CVEs")
 
 
-def check_package_cpes(nvd_path, packages):
-    cpedb = CPEDB(nvd_path)
+def check_package_cpes(nvd_path, cpedb_base_url, packages):
+    cpedb = CPEDB(nvd_path, cpedb_base_url)
     cpedb.get_xml_dict()
     for p in packages:
         if not p.cpeid:
@@ -1101,6 +1101,10 @@ def parse_args():
                           help='List of packages (comma separated)')
     parser.add_argument('--nvd-path', dest='nvd_path',
                         help='Path to the local NVD database', type=resolvepath)
+    parser.add_argument('--cpedb-base-url', dest='cpedb_base_url',
+                          help='CPE database wensite base url')
+    parser.add_argument('--nvd-base-url', dest='nvd_base_url',
+                          help='NVD website base url')
     args = parser.parse_args()
     if not args.html and not args.json:
         parser.error('at least one of --html or --json (or both) is required')
@@ -1155,8 +1159,8 @@ def __main__():
     loop.run_until_complete(check_package_latest_version(packages))
     if args.nvd_path:
         print("Checking packages CVEs")
-        check_package_cves(args.nvd_path, packages)
-        check_package_cpes(args.nvd_path, packages)
+        check_package_cves(args.nvd_path, args.nvd_base_url, packages)
+        check_package_cpes(args.nvd_path, args.cpedb_base_url, packages)
     print("Calculate stats")
     stats = calculate_stats(packages)
     if args.html:
-- 
2.25.1

###########################################################################
This email is confidential and may contain information subject to legal 
privilege.  If you are not the intended recipient please advise us of our
error by return e-mail then delete this email and any attached files.  
You may not copy, disclose or use the contents in any way.  

The views expressed in this email may not be those of Gallagher Group 
Ltd or subsidiary companies thereof.
###########################################################################
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply related	[flat|nested] 2+ messages in thread
* Re: [Buildroot] [PATCH 1/1] core: add option to override CPE and NVD base url
  2021-09-28 20:35 [Buildroot] [PATCH 1/1] core: add option to override CPE and NVD base url Ankur Tyagi
@ 2022-07-25 16:56 ` Arnout Vandecappelle
  0 siblings, 0 replies; 2+ messages in thread
From: Arnout Vandecappelle @ 2022-07-25 16:56 UTC (permalink / raw)
  To: Ankur Tyagi, buildroot



On 28/09/2021 22:35, Ankur Tyagi wrote:
> It may be desirable to integrate "pkg-stats" as part of build pipeline
> and usually build machines don't have internet access.
> In those scenarios, private network can be used to fetch CPE dictionary
> and NVD data (which are a pre-downloaded artifact).

  I completely understand the use case.


> This commit introduces config options BR2_CPEDB_BASE_URL and
> BR2_NVD_BASE_URL which can point to those internal/private network url.

  However, I think this unnecessarily complicates how to use it. Normally you 
can just copy download dir from a build which does have internet access and use 
that in your CI builds. With these additional URLs, you have to find out 
yourself exactly which URLs need to be downloaded from NIST, and in which exact 
location (directory structure) to put them.

  A much simpler approach (from a user perspective) is to leverage the already 
existing BR2_PRIMARY_SITE (and BR2_PRIMARY_SITE_ONLY) to implement this.

> By default, they refer to NIST official url.
> 
> It also adds CPE and NVD base url arguments to pkg-stats which
> are then passed to CVE and CPE class methods to fetch CPE dictionary
> and NVD data.
> 
> Signed-off-by: Ankur Tyagi <ankur.tyagi@gallagher.com>
[snip]
> diff --git a/Makefile b/Makefile
> index c960b53a6d..acb2c76aea 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -930,7 +930,9 @@ pkg-stats:
>   	$(TOPDIR)/support/scripts/pkg-stats -c \
>   		--json $(O)/pkg-stats.json \
>   		--html $(O)/pkg-stats.html \
> -		--nvd-path $(DL_DIR)/buildroot-nvd
> +		--nvd-path $(DL_DIR)/buildroot-nvd \
> +		--cpedb-base-url $(BR2_CPEDB_BASE_URL) \
> +		--nvd-base-url $(BR2_NVD_BASE_URL)

  Here basically you'd add something like

		$(if $(BR2_PRIMARY_SITE),--primary-site $(BR2_PRIMARY_SITE)) \
		$(if $(BR2_PRIMARY_SITE_ONLY),--primary-site-only)

>   .PHONY: missing-cpe
>   missing-cpe:
> diff --git a/support/scripts/cpedb.py b/support/scripts/cpedb.py
> index f4daf56124..36846621c3 100644
> --- a/support/scripts/cpedb.py
> +++ b/support/scripts/cpedb.py
> @@ -10,7 +10,7 @@ from xml.dom import minidom
>   
>   VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY']
>   
> -CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
> +CPE_DICTIONARY = "official-cpe-dictionary_v2.3.xml.gz"
>   
>   ns = {
>       '': 'http://cpe.mitre.org/dictionary/2.0',
> @@ -88,20 +88,22 @@ class CPE:
>   
>   
>   class CPEDB:
> -    def __init__(self, nvd_path):
> +    def __init__(self, nvd_path, cpedb_base_url):
>           self.all_cpes = dict()
>           self.all_cpes_no_version = dict()
>           self.nvd_path = nvd_path
> +        self.cpedb_base_url = cpedb_base_url
>   
>       def get_xml_dict(self):
>           print("CPE: Setting up NIST dictionary")
>           if not os.path.exists(os.path.join(self.nvd_path, "cpe")):
>               os.makedirs(os.path.join(self.nvd_path, "cpe"))
>   
> -        cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL))
> +        url = "%s/%s" % (self.cpedb_base_url, CPE_DICTIONARY)

  Here, you do need to construct the URL differently if BR2_PRIMARY_SITE is 
used, since it will be under <BR2_PRIMARY_SITE>/buildroot-nvd/<CPE_DICTIONARY>.

  In general, you'd also want to use the backup mechanism, i.e. if it's not 
found on BR2_PRIMARY_SITE, then it's downloaded directly from NIST (unless 
primary-only is set). That makes it easy to populate it without having to change 
the .config.

  It may be easier to implement that by calling into the download helper 
support/download/dl-wrapper. This already has also the infrastructure to handle 
multiple URLs to try, and to avoid download if the file is already there. It 
also supports URLs with file:// and scp:// in addition to http(s)://, which may 
be useful for some users.


  For now, I've marked the patch as Changes Requested in patchwork. Sorry that 
it took so long before anyone looked at it. Somewhat complicated things that add 
fringe features tend to have that effect.


  Regards,
  Arnout


> +        cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(url))
>           if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
> -            print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
> -            cpe_dict = requests.get(CPEDB_URL)
> +            print("CPE: Fetching xml manifest from [" + url + "]")
> +            cpe_dict = requests.get(url)
>               open(cpe_dict_local, "wb").write(cpe_dict.content)
>   
>           print("CPE: Unzipping xml manifest...")
> diff --git a/support/scripts/cve.py b/support/scripts/cve.py
> index 13c29fabe0..0ff5fbe823 100755
> --- a/support/scripts/cve.py
> +++ b/support/scripts/cve.py
> @@ -42,7 +42,6 @@ sys.path.append('utils/')
>   
>   NVD_START_YEAR = 2002
>   NVD_JSON_VERSION = "1.1"
> -NVD_BASE_URL = "https://nvd.nist.gov/feeds/json/cve/" + NVD_JSON_VERSION
>   
>   ops = {
>       '>=': operator.ge,
> @@ -82,7 +81,7 @@ class CVE:
>           self.nvd_cve = nvd_cve
>   
>       @staticmethod
> -    def download_nvd_year(nvd_path, year):
> +    def download_nvd_year(nvd_path, nvd_base_url, year):
>           metaf = "nvdcve-%s-%s.meta" % (NVD_JSON_VERSION, year)
>           path_metaf = os.path.join(nvd_path, metaf)
>           jsonf_gz = "nvdcve-%s-%s.json.gz" % (NVD_JSON_VERSION, year)
> @@ -94,7 +93,7 @@ class CVE:
>               return path_jsonf_gz
>   
>           # If not, we download the meta file
> -        url = "%s/%s" % (NVD_BASE_URL, metaf)
> +        url = "%s/%s/%s" % (nvd_base_url, NVD_JSON_VERSION, metaf)
>           print("Getting %s" % url)
>           page_meta = requests.get(url)
>           page_meta.raise_for_status()
> @@ -110,7 +109,7 @@ class CVE:
>                   return path_jsonf_gz
>   
>           # Grab the compressed JSON NVD, and write files to disk
> -        url = "%s/%s" % (NVD_BASE_URL, jsonf_gz)
> +        url = "%s/%s/%s" % (nvd_base_url, NVD_JSON_VERSION, jsonf_gz)
>           print("Getting %s" % url)
>           page_json = requests.get(url)
>           page_json.raise_for_status()
> @@ -119,14 +118,14 @@ class CVE:
>           return path_jsonf_gz
>   
>       @classmethod
> -    def read_nvd_dir(cls, nvd_dir):
> +    def read_nvd_dir(cls, nvd_dir, nvd_base_url):
>           """
>           Iterate over all the CVEs contained in NIST Vulnerability Database
>           feeds since NVD_START_YEAR. If the files are missing or outdated in
>           nvd_dir, a fresh copy will be downloaded, and kept in .json.gz
>           """
>           for year in range(NVD_START_YEAR, datetime.datetime.now().year + 1):
> -            filename = CVE.download_nvd_year(nvd_dir, year)
> +            filename = CVE.download_nvd_year(nvd_dir, nvd_base_url, year)
>               try:
>                   content = ijson.items(gzip.GzipFile(filename), 'CVE_Items.item')
>               except:  # noqa: E722
> diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats
> index cc91d13167..5192d36857 100755
> --- a/support/scripts/pkg-stats
> +++ b/support/scripts/pkg-stats
> @@ -583,7 +583,7 @@ def check_package_cve_affects(cve, cpe_product_pkgs):
>                   pkg.cves.append(cve.identifier)
>   
>   
> -def check_package_cves(nvd_path, packages):
> +def check_package_cves(nvd_path, nvd_base_url, packages):
>       if not os.path.isdir(nvd_path):
>           os.makedirs(nvd_path)
>   
> @@ -601,7 +601,7 @@ def check_package_cves(nvd_path, packages):
>           else:
>               cpe_product_pkgs[pkg.name].append(pkg)
>   
> -    for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
> +    for cve in cvecheck.CVE.read_nvd_dir(nvd_path, nvd_base_url):
>           check_package_cve_affects(cve, cpe_product_pkgs)
>   
>       for pkg in packages:
> @@ -612,8 +612,8 @@ def check_package_cves(nvd_path, packages):
>                   pkg.status['cve'] = ("ok", "not affected by CVEs")
>   
>   
> -def check_package_cpes(nvd_path, packages):
> -    cpedb = CPEDB(nvd_path)
> +def check_package_cpes(nvd_path, cpedb_base_url, packages):
> +    cpedb = CPEDB(nvd_path, cpedb_base_url)
>       cpedb.get_xml_dict()
>       for p in packages:
>           if not p.cpeid:
> @@ -1101,6 +1101,10 @@ def parse_args():
>                             help='List of packages (comma separated)')
>       parser.add_argument('--nvd-path', dest='nvd_path',
>                           help='Path to the local NVD database', type=resolvepath)
> +    parser.add_argument('--cpedb-base-url', dest='cpedb_base_url',
> +                          help='CPE database wensite base url')
> +    parser.add_argument('--nvd-base-url', dest='nvd_base_url',
> +                          help='NVD website base url')
>       args = parser.parse_args()
>       if not args.html and not args.json:
>           parser.error('at least one of --html or --json (or both) is required')
> @@ -1155,8 +1159,8 @@ def __main__():
>       loop.run_until_complete(check_package_latest_version(packages))
>       if args.nvd_path:
>           print("Checking packages CVEs")
> -        check_package_cves(args.nvd_path, packages)
> -        check_package_cpes(args.nvd_path, packages)
> +        check_package_cves(args.nvd_path, args.nvd_base_url, packages)
> +        check_package_cpes(args.nvd_path, args.cpedb_base_url, packages)
>       print("Calculate stats")
>       stats = calculate_stats(packages)
>       if args.html:
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2022-07-25 16:56 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-28 20:35 [Buildroot] [PATCH 1/1] core: add option to override CPE and NVD base url Ankur Tyagi
2022-07-25 16:56 ` Arnout Vandecappelle

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.