* [PATCH 2/6] scripts/machine-summary: harvest more patch data
2022-01-19 21:20 [PATCH 1/6] scripts/layer-overview: improve layer detection Ross Burton
@ 2022-01-19 21:20 ` Ross Burton
2022-01-19 21:20 ` [PATCH 3/6] scripts/machine-summary: refactor output to classes Ross Burton
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Ross Burton @ 2022-01-19 21:20 UTC (permalink / raw)
To: meta-arm
Instead of just putting whether there are patches or not into the
context, store the list of patches, the layer they came from, and the
upstream status.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
scripts/machine-summary.py | 30 ++++++++++++++++++++++++++----
1 file changed, 26 insertions(+), 4 deletions(-)
diff --git a/scripts/machine-summary.py b/scripts/machine-summary.py
index 7d2eb53f..e0c7870d 100755
--- a/scripts/machine-summary.py
+++ b/scripts/machine-summary.py
@@ -1,9 +1,10 @@
#! /usr/bin/env python3
-import os
-import sys
import argparse
import datetime
+import os
+import re
+import sys
import jinja2
@@ -42,8 +43,28 @@ def layer_path(layername, d):
return path
return None
+def extract_patch_info(src_uri, d):
+ """
+ Parse the specified patch entry from a SRC_URI and return (base name, layer name, status) tuple
+ """
+ import bb.fetch, bb.utils
+
+ info = {}
+ localpath = bb.fetch.decodeurl(src_uri)[2]
+ info["name"] = os.path.basename(localpath)
+ info["layer"] = bb.utils.get_file_layer(localpath, d)
+
+ status = "Unknown"
+ with open(localpath, errors="ignore") as f:
+ m = re.search(r"^[\t ]*Upstream[-_ ]Status:?[\t ]*(\w*)", f.read(), re.IGNORECASE | re.MULTILINE)
+ if m:
+ # TODO: validate
+ status = m.group(1)
+ info["status"] = status
+ return info
+
def harvest_data(machines, recipes):
- import bb.tinfoil, bb.utils
+ import bb.tinfoil
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.prepare(config_only=True)
corepath = layer_path("core", tinfoil.config_data)
@@ -83,7 +104,8 @@ def harvest_data(machines, recipes):
details = versions[machine][recipe] = {}
details["recipe"] = d.getVar("PN")
details["version"] = trim_pv(d.getVar("PV"))
- details["patched"] = bool(oe.patch.src_patches(d))
+ details["patches"] = [extract_patch_info(p, d) for p in oe.patch.src_patches(d)]
+ details["patched"] = bool(details["patches"])
# Now backfill the upstream versions
for machine in versions:
--
2.25.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 3/6] scripts/machine-summary: refactor output to classes
2022-01-19 21:20 [PATCH 1/6] scripts/layer-overview: improve layer detection Ross Burton
2022-01-19 21:20 ` [PATCH 2/6] scripts/machine-summary: harvest more patch data Ross Burton
@ 2022-01-19 21:20 ` Ross Burton
2022-01-19 21:20 ` [PATCH 4/6] scripts/machine-summary: put more version information into the context Ross Burton
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Ross Burton @ 2022-01-19 21:20 UTC (permalink / raw)
To: meta-arm
To prepare for future expansion, refactor the output code to be delegated
to Format subclasses.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
scripts/machine-summary.py | 75 +++++++++++++++++++++++++++-----------
1 file changed, 53 insertions(+), 22 deletions(-)
diff --git a/scripts/machine-summary.py b/scripts/machine-summary.py
index e0c7870d..e5161440 100755
--- a/scripts/machine-summary.py
+++ b/scripts/machine-summary.py
@@ -3,28 +3,12 @@
import argparse
import datetime
import os
+import pathlib
import re
import sys
import jinja2
-def get_template(name):
- template_dir = os.path.dirname(os.path.abspath(__file__))
- env = jinja2.Environment(
- loader=jinja2.FileSystemLoader(template_dir),
- autoescape=jinja2.select_autoescape(),
- trim_blocks=True,
- lstrip_blocks=True
- )
- def is_old(version, upstream):
- if "+git" in version:
- # strip +git and see if this is a post-release snapshot
- version = version.replace("+git", "")
- return version != upstream
- env.tests["old"] = is_old
-
- return env.get_template(f"machine-summary-{name}.jinja")
-
def trim_pv(pv):
"""
Strip anything after +git from the PV
@@ -126,19 +110,66 @@ recipes = ("virtual/kernel",
"gcc-aarch64-none-elf-native",
"gcc-arm-none-eabi-native")
+
+class Format:
+ """
+ The name of this format
+ """
+ name = None
+ """
+ Registry of names to classes
+ """
+ registry = {}
+
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ assert cls.name
+ cls.registry[cls.name] = cls
+
+ @classmethod
+ def get_format(cls, name):
+ return cls.registry[name]()
+
+ def render(self, context, output: pathlib.Path):
+ # Default implementation for convenience
+ with open(output, "wt") as f:
+ f.write(self.get_template(f"machine-summary-{self.name}.jinja").render(context))
+
+ def get_template(self, name):
+ template_dir = os.path.dirname(os.path.abspath(__file__))
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(template_dir),
+ autoescape=jinja2.select_autoescape(),
+ trim_blocks=True,
+ lstrip_blocks=True
+ )
+ def is_old(version, upstream):
+ if "+git" in version:
+ # strip +git and see if this is a post-release snapshot
+ version = version.replace("+git", "")
+ return version != upstream
+ env.tests["old"] = is_old
+
+ return env.get_template(name)
+
+class TextOverview(Format):
+ name = "overview.txt"
+
+class HtmlUpdates(Format):
+ name = "updates.html"
+
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="machine-summary")
parser.add_argument("machines", nargs="+", help="machine names", metavar="MACHINE")
- parser.add_argument("-t", "--template", required=True)
- parser.add_argument("-o", "--output", required=True, type=argparse.FileType('w', encoding='UTF-8'))
+ parser.add_argument("-t", "--type", required=True, choices=Format.registry.keys())
+ parser.add_argument("-o", "--output", type=pathlib.Path, required=True)
args = parser.parse_args()
- template = get_template(args.template)
-
context = {}
# TODO: include git describe for meta-arm
context["timestamp"] = str(datetime.datetime.now().strftime("%c"))
context["recipes"] = sorted(recipes)
context["releases"], context["data"] = harvest_data(args.machines, recipes)
- args.output.write(template.render(context))
+ formatter = Format.get_format(args.type)
+ formatter.render(context, args.output)
--
2.25.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 4/6] scripts/machine-summary: put more version information into the context
2022-01-19 21:20 [PATCH 1/6] scripts/layer-overview: improve layer detection Ross Burton
2022-01-19 21:20 ` [PATCH 2/6] scripts/machine-summary: harvest more patch data Ross Burton
2022-01-19 21:20 ` [PATCH 3/6] scripts/machine-summary: refactor output to classes Ross Burton
@ 2022-01-19 21:20 ` Ross Burton
2022-01-19 21:20 ` [PATCH 5/6] scripts/machine-summary: write per-machine reports with more details Ross Burton
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Ross Burton @ 2022-01-19 21:20 UTC (permalink / raw)
To: meta-arm
As well as storing the truncated PV, also store the original PV and
whether the recipe needs updating, to avoid the templates needing to
do that logic.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
scripts/machine-summary.py | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/scripts/machine-summary.py b/scripts/machine-summary.py
index e5161440..2fef8491 100755
--- a/scripts/machine-summary.py
+++ b/scripts/machine-summary.py
@@ -15,6 +15,15 @@ def trim_pv(pv):
"""
return "".join(pv.partition("+git")[:2])
+def needs_update(version, upstream):
+ """
+ Do a dumb comparison to determine if the version needs to be updated.
+ """
+ if "+git" in version:
+ # strip +git and see if this is a post-release snapshot
+ version = version.replace("+git", "")
+ return version != upstream
+
def layer_path(layername, d):
"""
Return the path to the specified layer, or None if the layer isn't present.
@@ -88,14 +97,16 @@ def harvest_data(machines, recipes):
details = versions[machine][recipe] = {}
details["recipe"] = d.getVar("PN")
details["version"] = trim_pv(d.getVar("PV"))
+ details["fullversion"] = d.getVar("PV")
details["patches"] = [extract_patch_info(p, d) for p in oe.patch.src_patches(d)]
details["patched"] = bool(details["patches"])
# Now backfill the upstream versions
for machine in versions:
for recipe in versions[machine]:
- versions[machine][recipe]["upstream"] = upstreams[recipe]
-
+ data = versions[machine][recipe]
+ data["upstream"] = upstreams[recipe]
+ data["needs_update"] = needs_update(data["version"], data["upstream"])
return upstreams, versions
# TODO can this be inferred from the list of recipes in the layer
--
2.25.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 5/6] scripts/machine-summary: write per-machine reports with more details
2022-01-19 21:20 [PATCH 1/6] scripts/layer-overview: improve layer detection Ross Burton
` (2 preceding siblings ...)
2022-01-19 21:20 ` [PATCH 4/6] scripts/machine-summary: put more version information into the context Ross Burton
@ 2022-01-19 21:20 ` Ross Burton
2022-01-19 21:20 ` [PATCH 6/6] CI: update for changes to machine-summary Ross Burton
2022-01-20 3:50 ` [PATCH 1/6] scripts/layer-overview: improve layer detection Jon Mason
5 siblings, 0 replies; 7+ messages in thread
From: Ross Burton @ 2022-01-19 21:20 UTC (permalink / raw)
To: meta-arm
Rename the updates.html format to just 'report'.
This report has the existing overview as the index.html, and then
per-machine files are written with the patch breakdown.
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
scripts/machine-summary-updates.html.jinja | 47 -----------------
scripts/machine-summary.py | 21 +++++++-
scripts/report-base.html.jinja | 35 +++++++++++++
scripts/report-details.html.jinja | 61 ++++++++++++++++++++++
scripts/report-index.html.jinja | 44 ++++++++++++++++
5 files changed, 160 insertions(+), 48 deletions(-)
delete mode 100644 scripts/machine-summary-updates.html.jinja
create mode 100644 scripts/report-base.html.jinja
create mode 100644 scripts/report-details.html.jinja
create mode 100644 scripts/report-index.html.jinja
diff --git a/scripts/machine-summary-updates.html.jinja b/scripts/machine-summary-updates.html.jinja
deleted file mode 100644
index d3ac2ff6..00000000
--- a/scripts/machine-summary-updates.html.jinja
+++ /dev/null
@@ -1,47 +0,0 @@
-<!DOCTYPE html>
-<html>
- <head>
- <title>Pending Machine Upgrades Report</title>
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.1/css/bulma.min.css">
- </head>
- <body>
- <section class="section">
- <div class="content">
- <h1 class="title">Pending Machine Upgrades Report</h1>
- <p>Generated at {{ timestamp }}.</p>
- </div>
-
- <table class="table is-striped">
- <thead>
- <tr>
- <th>Machine</th>
- {% for recipe in recipes|sort %}
- <th>{{ recipe }} ({{releases[recipe]|default("?")}})</th>
- {% endfor %}
- </tr>
- </thead>
- <tbody>
- {% for machine, data in data|dictsort %}
- <tr>
- <th>{{ machine }}</th>
- {% for recipe in recipes|sort %}
- {% if recipe in data %}
- {% set details = data[recipe] %}
- {% set is_old = details.version is old(details.upstream) %}
- <td class="{% if is_old %}has-text-weight-bold{% endif %}">
- {{ details.recipe if details.recipe != recipe}}
- {{ details.version }}
- {{ "(patched)" if details.patched }}
- </td>
- {% else %}
- <td>-</td>
- {% endif %}
- {% endfor %}
- </tr>
- {% endfor %}
- </tbody>
- </table>
- </section>
- </body>
-</html>
diff --git a/scripts/machine-summary.py b/scripts/machine-summary.py
index 2fef8491..1f98f247 100755
--- a/scripts/machine-summary.py
+++ b/scripts/machine-summary.py
@@ -167,7 +167,26 @@ class TextOverview(Format):
name = "overview.txt"
class HtmlUpdates(Format):
- name = "updates.html"
+ name = "report"
+
+ def render(self, context, output: pathlib.Path):
+ if output.exists() and not output.is_dir():
+ print(f"{output} is not a directory", file=sys.stderr)
+ sys.exit(1)
+
+ if not output.exists():
+ output.mkdir(parents=True)
+
+ with open(output / "index.html", "wt") as f:
+ f.write(self.get_template(f"report-index.html.jinja").render(context))
+
+ subcontext = context.copy()
+ del subcontext["data"]
+ for machine, subdata in context["data"].items():
+ subcontext["machine"] = machine
+ subcontext["data"] = subdata
+ with open(output / f"{machine}.html", "wt") as f:
+ f.write(self.get_template(f"report-details.html.jinja").render(subcontext))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="machine-summary")
diff --git a/scripts/report-base.html.jinja b/scripts/report-base.html.jinja
new file mode 100644
index 00000000..be081251
--- /dev/null
+++ b/scripts/report-base.html.jinja
@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>{% block title %}{% endblock %}</title>
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.1/css/bulma.min.css">
+ </head>
+ <body>
+ <section class="section">
+ {# TODO use position: sticky to glue this to the top #}
+ <nav class="breadcrumb is-large">
+ <ul>
+ <li class="{{ "is-active" if machine is undefined }}">
+ <a href="index.html">Recipe Report</a>
+ </li>
+ {% if machine is defined %}
+ <li class="is-active">
+ <a href="#">{{machine}}</a>
+ </li>
+ {% endif %}
+ </ul>
+ </nav>
+
+ <div class="content">
+ {% block content %}{% endblock %}
+ </div>
+ </section>
+
+ <footer class="footer">
+ <div class="content has-text-centered">
+ Generated by <code>machine-summary</code> at {{ timestamp }}.
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/scripts/report-details.html.jinja b/scripts/report-details.html.jinja
new file mode 100644
index 00000000..ba66fcf0
--- /dev/null
+++ b/scripts/report-details.html.jinja
@@ -0,0 +1,61 @@
+{% extends "report-base.html.jinja" %}
+{% block title %}Recipe Report for {{ machine }}{% endblock %}
+
+{# Write a tag element using the Upstream-Status to determine the class. #}
+{% macro make_patch_tag(status) -%}
+ {% set status = status.split()[0] %}
+ {% if status in ("Unknown", "Pending") %}
+ {% set class = "is-danger" %}
+ {% elif status in ("Backport", "Accepted", "Inappropriate") %}
+ {% set class = "is-success" %}
+ {% elif status in ("Submitted", "Denied") %}
+ {% set class = "is-info" %}
+ {% else %}
+ {% set class = "is-info" %}
+ {% endif %}
+ <span class="tag {{ class }}">{{ status }}</span>
+{%- endmacro %}
+
+{% block content %}
+ <!-- TODO table of contents -->
+
+ {% for name, data in data|dictsort if data.needs_update or data.patched %}
+ <h2 class="title is-4">
+ {{ data.recipe }} {{ data.fullversion }}
+ {% if name != data.recipe %}
+ (provides {{ name }})
+ {% endif %}
+ {% if data.needs_update %}<span class="tag is-danger">Upgrade Needed</span>{% endif %}
+ <a id="recipe-{{ data.recipe }}" class="has-text-grey-lighter">#</a>
+ </h2>
+
+ {% if data.needs_update %}
+ <p>
+ Recipe is version {{ data.fullversion }}, latest upstream release is <strong>{{ data.upstream }}</strong>.
+ </p>
+ {% endif%}
+
+ {% if data.patched %}
+ <table class="table is-striped is-bordered">
+ <thead>
+ <tr>
+ <th>Patch</th>
+ <th style="width: 20em">Layer</th>
+ <th style="width: 10em">Status</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for pinfo in data.patches %}
+ <tr>
+ <!-- TODO links to cgit -->
+ <td>{{ pinfo.name }}</td>
+ <td>{{ pinfo.layer }}</td>
+ <!-- TODO: tooltip with full status? -->
+ <td class="has-text-centered">{{ make_patch_tag(pinfo.status)}}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ {% endif %}
+ {% endfor %}
+{% endblock %}
diff --git a/scripts/report-index.html.jinja b/scripts/report-index.html.jinja
new file mode 100644
index 00000000..3e39174f
--- /dev/null
+++ b/scripts/report-index.html.jinja
@@ -0,0 +1,44 @@
+{% extends "report-base.html.jinja" %}
+{% block title %}Recipe Report{% endblock %}
+
+{% block content %}
+ <table class="table is-striped">
+ <thead>
+ <tr>
+ <th>Machine</th>
+ {% for recipe in recipes|sort %}
+ <th>{{ recipe }} ({{releases[recipe]|default("?")}})</th>
+ {% endfor %}
+ </tr>
+ </thead>
+ <tbody>
+ {% for machine, data in data|dictsort %}
+ <tr>
+ <th><a href="{{machine}}.html">{{ machine }}</a></th>
+ {% for recipe in recipes|sort %}
+ {% if recipe in data %}
+ {% set details = data[recipe] %}
+ <td style="text-align: center">
+ <a href="{{machine}}.html#recipe-{{details.recipe}}">
+ {{ details.recipe if details.recipe != recipe}}
+ {{ details.version }}
+ </a>
+ {% if details.patches or details.needs_update %}
+ <br>
+ {% if details.patches %}
+ <span class="tag is-info">Patched</span>
+ {% endif %}
+ {% if details.needs_update %}
+ <span class="tag is-danger">Upgrade</span>
+ {% endif %}
+ {% endif %}
+ </td>
+ {% else %}
+ <td>-</td>
+ {% endif %}
+ {% endfor %}
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+{% endblock %}
--
2.25.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 6/6] CI: update for changes to machine-summary
2022-01-19 21:20 [PATCH 1/6] scripts/layer-overview: improve layer detection Ross Burton
` (3 preceding siblings ...)
2022-01-19 21:20 ` [PATCH 5/6] scripts/machine-summary: write per-machine reports with more details Ross Burton
@ 2022-01-19 21:20 ` Ross Burton
2022-01-20 3:50 ` [PATCH 1/6] scripts/layer-overview: improve layer detection Jon Mason
5 siblings, 0 replies; 7+ messages in thread
From: Ross Burton @ 2022-01-19 21:20 UTC (permalink / raw)
To: meta-arm
Signed-off-by: Ross Burton <ross.burton@arm.com>
---
.gitlab-ci.yml | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bec61400..266e0868 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -86,9 +86,11 @@ pending-updates:
extends: .setup
artifacts:
paths:
- - update-report.html
+ - update-report
script:
- - kas shell ci/qemuarm64.yml:ci/meta-openembedded.yml -c "$CI_PROJECT_DIR/scripts/machine-summary.py -t updates.html -o $CI_PROJECT_DIR/update-report.html $($CI_PROJECT_DIR/ci/listmachines.py meta-arm meta-arm-bsp)"
+ - rm -fr update-report
+ - kas shell ci/qemuarm64.yml:ci/meta-openembedded.yml --command \
+ "$CI_PROJECT_DIR/scripts/machine-summary.py -t report -o $CI_PROJECT_DIR/update-report $($CI_PROJECT_DIR/ci/listmachines.py meta-arm meta-arm-bsp)"
# Do this on x86 whilst the compilers are x86-only
tags:
- x86_64
--
2.25.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [PATCH 1/6] scripts/layer-overview: improve layer detection
2022-01-19 21:20 [PATCH 1/6] scripts/layer-overview: improve layer detection Ross Burton
` (4 preceding siblings ...)
2022-01-19 21:20 ` [PATCH 6/6] CI: update for changes to machine-summary Ross Burton
@ 2022-01-20 3:50 ` Jon Mason
5 siblings, 0 replies; 7+ messages in thread
From: Jon Mason @ 2022-01-20 3:50 UTC (permalink / raw)
To: meta-arm, Ross Burton
On Wed, 19 Jan 2022 21:20:45 +0000, Ross Burton wrote:
> Refactor the script somewhat, and detect whether the starting directory
> is a single layer, or a collection of layers.
Applied, thanks!
[1/6] scripts/layer-overview: improve layer detection
commit: 02350369574cdcea2091048a04468ee515c7d711
[2/6] scripts/machine-summary: harvest more patch data
commit: cb29175ab4c9a68e26b66aceb97463b32b3639c9
[3/6] scripts/machine-summary: refactor output to classes
commit: 0d2d0fd69102cadb4e699fa41402227f60e56b69
[4/6] scripts/machine-summary: put more version information into the context
commit: b1b0f37da7f4c670216de4225af024560e97248e
[5/6] scripts/machine-summary: write per-machine reports with more details
commit: c4b34740cfc78485d22453d9bc284adc12216dcd
[6/6] CI: update for changes to machine-summary
commit: aaac0541658b6d6e655662d53dcd27e5cd1bf018
Best regards,
--
Jon Mason <jon.mason@arm.com>
^ permalink raw reply [flat|nested] 7+ messages in thread