From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-io0-f172.google.com (mail-io0-f172.google.com [209.85.223.172]) by mail.openembedded.org (Postfix) with ESMTP id 6A668771B2 for ; Mon, 29 Feb 2016 14:50:57 +0000 (UTC) Received: by mail-io0-f172.google.com with SMTP id l127so188176410iof.3 for ; Mon, 29 Feb 2016 06:50:59 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=intel-com.20150623.gappssmtp.com; s=20150623; h=mime-version:in-reply-to:references:from:date:message-id:subject:to :cc; bh=5V+WVu4kxDUYRDSpFvb7rRM7XiWUbwbd/E4GG2mMx9M=; b=0/7J5Wj/drj0SKS8bDGkIQkAPfKAx/3IyM20FHUyxAu1yFj2ssUM9zEogRwJIRZait 9PPiJY26OiPcLgUdtXzCWBZgcjVxlbXOzIEdPutJzGrOrMa8pSEirNtnohtBUpL1HoIY Glegi8QlkVaaDy/rddgR5kErUlmrZ4RWMXUgL6pBMJprJxqnL4JIW7KniXaYAjC7ReFO uaL0j3IeX5f9ePJTTPptA/s1hlO0RJteJIIrO9MWNiEBNzMdhHMCRO6QRkWDVMyOiBgG BgCkAEG962WMu/FbUhkmaKn5FIJ0dDP97uIA6+jQkckZ5nLVSUDmJcizChS//14d0ZkB wSgw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:mime-version:in-reply-to:references:from:date :message-id:subject:to:cc; bh=5V+WVu4kxDUYRDSpFvb7rRM7XiWUbwbd/E4GG2mMx9M=; b=kobaTiqlL9q7qXrzNYAZ1hjuSGsaytcE9+xw1n1iPySmqNQ6OfwmwpDfpkZoZK0QW2 ysXGXaZITuRNclh9ByXwW4v3GdpFTuNFss15XPq424o5mL9g1bmVpiX30QUU6uJ+6qqU IO3q0AdPhWz92rSf4CidnuNtvCI6XvkotFjMjIbdDzPdNuSZRFGC+rUxkHObZzglYmyw mnGvKaRjoEGrNcHE6vN3N73P/c12PaJMQPAOhU59mUi8ywyxNsE74fCAg3WTh4erLbJO 9Rfapqns4vVxj35FOksQtOYGhF43KGWgEXukY2d+bN5yBlqBlP7lexxol/oYQUikUm61 mBZg== X-Gm-Message-State: AG10YOR2I4QQ2+rkDYdRU3qj+f/G3NvBrSm36DkGrozXy1mc309K3KYEXq+UPjOux9kFoa8VHLeKVxFrb1WKBCdd X-Received: by 10.107.3.220 with SMTP id e89mr18130523ioi.99.1456757458689; Mon, 29 Feb 2016 06:50:58 -0800 (PST) MIME-Version: 1.0 Received: by 10.107.162.70 with HTTP; Mon, 29 Feb 2016 06:50:39 -0800 (PST) In-Reply-To: <5648c9e910e63de9aa29e1625ddab5e3804f179e.1456327117.git.mariano.lopez@linux.intel.com> References: <5648c9e910e63de9aa29e1625ddab5e3804f179e.1456327117.git.mariano.lopez@linux.intel.com> From: "Burton, Ross" Date: Mon, 29 Feb 2016 14:50:39 +0000 Message-ID: To: Mariano Lopez Cc: OE-core Subject: Re: [PATCH 3/3] cve-check.bbclass: Add class X-BeenThere: openembedded-core@lists.openembedded.org X-Mailman-Version: 2.1.12 Precedence: list List-Id: Patches and discussions about the oe-core layer List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 29 Feb 2016 14:50:58 -0000 Content-Type: multipart/alternative; boundary=001a113f0e7cd98d27052ce9c6eb --001a113f0e7cd98d27052ce9c6eb Content-Type: text/plain; charset=UTF-8 On 24 February 2016 at 15:27, wrote: > +# Whitelist for packages (PN) > +cve_check_pn_whitelist () { > + glibc-locale > +} Why is this a shell function? CVE_CHECK_PN_WHITELIST = "glibc-locale" please. +# Whitelist for CVE and version of package > +python cve_check_cve_whitelist () { > + {"CVE-2014-2524": ("6.3",), \ > + } > +} Why is this a Python function? Make it a bare string with implied formatting and it can be extended from outside this class, where as this can't. > +addtask cve_check before do_build > If you're expecting to look at the sources, you'll want to be after do_fetch too. > +do_cve_check[depends] = "cve-check-tool-native:do_populate_cve_db" > And cve-check-tool-native:do_populate_sysroot. > +def get_patches_cves(d): > + """ > + Get patches that solve CVEs using the "CVE: " tag. > + """ > + > + import re > + > + pn = d.getVar("PN", True) > + cve_match = re.compile("CVE:( CVE\-\d+\-\d+)+") > How does this work as the backslashes are escaping the - and d and d? Use r"" strings. + patched_cves = set() > + for url in src_patches(d): > + patch_file = bb.fetch.decodeurl(url)[2] > + with open(patch_file, "r") as f: > + patch_text = f.read() > + > + # Search for the "CVE: " line > + match = cve_match.search(patch_text) > + if match: > + # Get only the CVEs without the "CVE: " tag > + cves = patch_text[match.start()+5:match.end()] > + for cve in cves.split(): > + patched_cves.add(cve) > > Breaks for patches such as this in glibc: meta/recipes-core/glibc/glibc/CVE-2015-9761_1.patch:CVE: CVE-2015-9761 patch #1 I'd probably look for a line that starts with "CVE:" and the use re.findall to find all strings matching r"CVE-\d{4}-\d+" > + # It is needed to export the proxies to download the database using > HTTP > + export_proxies(d) > The database has already been downloaded hasn't it? > + # Write the faux CSV file to be used with cve-check-tool > + fd, faux = tempfile.mkstemp(prefix="cve-faux-") > + with os.fdopen(fd, "w") as f: > + f.write("%s,%s,%s," % (bpn, pv, cves)) > > Put this inside the try incase the write fails so the file will still be deleted. > + cmd += " %s" % faux > + try: > + popen = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, > stderr=subprocess.PIPE) > + output, error = popen.communicate() > Don't use the low-level function, the higher level helpers are clearer. Always write cmd as a list unless you *need* the shell. try: subprocess.check_output(cmd, stderr=subprocess.STDOUT) except CommandCalledException as e: bb.warn("Couldn't check for CVEs: %s (output %s)" % (e, e.output)) +def get_cve_info(d, cves): > + """ > + Get CVE information from the database used by cve-check-tool. > + """ > + > + try: > + import sqlite3 > + except ImportError: > + from pysqlite2 import dbapi2 as sqlite3 > Isn't the output from cve-check-tool good enough? Would it be nicer to extend the log instead of assuming that the database format won't ever change? +def cve_write_data(d, patched, unpatched, cve_data): > + """ > + Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and > + CVE manifest if enabled. > + """ > + > + from bb.utils import mkdirhier > + > + cve_file = d.getVar("CVE_CHECK_LOCAL_FILE", True) > + nvd_link = "https://web.nvd.nist.gov/view/vuln/detail?vulnId=" > + write_string = "" > + mkdirhier(d.getVar("CVE_CHECK_LOCAL_DIR", True)) > + > + for cve in sorted(cve_data): > + write_string += "PACKAGE NAME: %s\n" % d.getVar("PN", True) > + write_string += "PACKAGE VERSION: %s\n" % d.getVar("PV", True) > + write_string += "CVE: %s\n" % cve > + if cve in patched: > + write_string += "CVE STATUS: Patched\n" > + else: > + write_string += "CVE STATUS: Unpatched\n" > + bb.warn("Found unpatched CVE, for more information check %s" > % cve_file) > + write_string += "CVE SUMMARY: %s\n" % cve_data[cve]["summary"] > + write_string += "CVSS v2 BASE SCORE: %s\n" % > cve_data[cve]["score"] > + write_string += "VECTOR: %s\n" % cve_data[cve]["vector"] > + write_string += "MORE INFORMATION: %s%s\n\n" % (nvd_link, cve) > + > + with open(cve_file, "w") as f: > + f.write(write_string) > Just write to the file instead of to a temporary string. Ross --001a113f0e7cd98d27052ce9c6eb Content-Type: text/html; charset=UTF-8 Content-Transfer-Encoding: quoted-printable
On 24 February 2016 at 15:27, <mariano.lopez= @linux.intel.com> wrote:
+# Whitelist for packages (PN)
+cve_check_pn_whitelist () {
+=C2=A0 =C2=A0 glibc-locale
+}

Why is this a shell function?=C2=A0 CVE_= CHECK_PN_WHITELIST =3D "glibc-locale" please.=C2=A0
+# Whitelist for CVE and version of package
+python cve_check_cve_whitelist () {
+=C2=A0 =C2=A0 {"CVE-2014-2524": ("6.3",), \
+=C2=A0 =C2=A0 }
+}

Why is this a Python function?=C2=A0 Mak= e it a bare string with implied formatting and it can be extended from outs= ide this class, where as this can't.
=C2=A0=C2=A0
+addtask cve_check before do_build

If you're expecting to look at the sources, you'll want to b= e after do_fetch too.
=C2=A0
+do_cve_check[depends] =3D "cve-check-tool-native:do_populate_cve_db&q= uot;


And cve-check-tool-= native:do_populate_sysroot.
=C2=A0
+def get= _patches_cves(d):
+=C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 Get patches that solve CVEs using the "CVE: " tag.=
+=C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 import re
+
+=C2=A0 =C2=A0 pn =3D d.getVar("PN", True)
+=C2=A0 =C2=A0 cve_match =3D re.compile("CVE:( CVE\-\d+\-\d+)+")<= br>

How does this work as the backslashes a= re escaping the - and d and d?=C2=A0 Use r"" strings.
<= br>
+=C2=A0 =C2=A0 patched_cves =3D set()
+=C2=A0 =C2=A0 for url in src_patches(d):
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 patch_file =3D bb.fetch.decodeurl(url)[2]
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 with open(patch_file, "r") as f:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 patch_text =3D f.read()
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # Search for the "CVE: " line
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 match =3D cve_match.search(patch_text)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if match:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # Get only the CVEs without the = "CVE: " tag
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cves =3D patch_text[match.start(= )+5:match.end()]
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for cve in cves.split():
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 patched_cves.add(c= ve)


Breaks for patches such as this= in glibc:

meta/recipes-core/glibc/glibc/CVE-2015-= 9761_1.patch:CVE: CVE-2015-9761 patch #1

I'd p= robably look for a line that starts with "CVE:" and the use re.fi= ndall to find all strings matching r"CVE-\d{4}-\d+"
=C2= =A0
+=C2=A0 =C2=A0 # It is needed to export the proxi= es to download the database using HTTP
+=C2=A0 =C2=A0 export_proxies(d)

The da= tabase has already been downloaded hasn't it?
=C2=A0
+=C2=A0 =C2=A0 # Write the faux CSV file to be used with cve-check-tool
+=C2=A0 =C2=A0 fd, faux =3D tempfile.mkstemp(prefix=3D"cve-faux-"= )
+=C2=A0 =C2=A0 with os.fdopen(fd, "w") as f:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 f.write("%s,%s,%s," % (bpn, pv, cves= ))


Put this inside the try incase t= he write fails so the file will still be deleted.
=C2=A0
+=C2=A0 =C2=A0 cmd +=3D " %s" % faux
+=C2=A0 =C2=A0 try:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 popen =3D subprocess.Popen(cmd, shell=3DTrue, = stdout=3Dsubprocess.PIPE, stderr=3Dsubprocess.PIPE)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 output, error =3D popen.communicate()

Don't use the low-level function, the highe= r level helpers are clearer.

Always write cmd as a= list unless you *need* the shell.

try:
= =C2=A0 =C2=A0 subprocess.check_output(cmd, stderr=3Dsubprocess.STDOUT)
except CommandCalledException as e:
=C2=A0 =C2=A0 bb.warn(&= quot;Couldn't check for CVEs: %s (output %s)" % (e, e.output))

+def get_cve_info(d, cves):
+=C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 Get CVE information from the database used by cve-check-tool= .
+=C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 try:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 import sqlite3
+=C2=A0 =C2=A0 except ImportError:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 from pysqlite2 import dbapi2 as sqlite3

Isn't the output from cve-check-tool g= ood enough?=C2=A0 Would it be nicer to extend the log instead of assuming t= hat the database format won't ever change?

+def cve_write_data(d, patched, unpatched, cve_data):
+=C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and<= br> +=C2=A0 =C2=A0 CVE manifest if enabled.
+=C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 from bb.utils import mkdirhier
+
+=C2=A0 =C2=A0 cve_file =3D d.getVar("CVE_CHECK_LOCAL_FILE", True= )
+=C2=A0 =C2=A0 nvd_link =3D "https://web.nvd= .nist.gov/view/vuln/detail?vulnId=3D"
+=C2=A0 =C2=A0 write_string =3D ""
+=C2=A0 =C2=A0 mkdirhier(d.getVar("CVE_CHECK_LOCAL_DIR", True)) +
+=C2=A0 =C2=A0 for cve in sorted(cve_data):
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "PACKAGE NAME: %s\n&quo= t; % d.getVar("PN", True)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "PACKAGE VERSION: %s\n&= quot; % d.getVar("PV", True)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "CVE: %s\n" % cve<= br> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 if cve in patched:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "CVE STAT= US: Patched\n"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "CVE STAT= US: Unpatched\n"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 bb.warn("Found unpatched CV= E, for more information check %s" % cve_file)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "CVE SUMMARY: %s\n"= ; % cve_data[cve]["summary"]
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "CVSS v2 BASE SCORE: %s= \n" % cve_data[cve]["score"]
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "VECTOR: %s\n" % c= ve_data[cve]["vector"]
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 write_string +=3D "MORE INFORMATION: %s%s= \n\n" % (nvd_link, cve)
+
+=C2=A0 =C2=A0 with open(cve_file, "w") as f:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 f.write(write_string)
Just write to the file instead of to a temporary string.
<= div>
Ross
--001a113f0e7cd98d27052ce9c6eb--