[PATCH 1/3] cmake-native: Enable ccmake by default and depend on ncurses

[PATCH 1/3] cmake-native: Enable ccmake by default and depend on ncurses

[Nathan Rossi]

Enable the building of the curses based ui for cmake. This depends on
ncurses.

Signed-off-by: Nathan Rossi <nathan@nathanrossi.com>
---
This patch suggests enabling this in the cmake-native recipe, however
this might be undesirable for bootstrapping reasons. Whilst ccmake is
not used normally the added dependency on ncurses is likely required for
other parts of the build so impact on build ordering and time should be
relatively minimal.
---
 meta/recipes-devtools/cmake/cmake-native_3.14.0.bb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/meta/recipes-devtools/cmake/cmake-native_3.14.0.bb b/meta/recipes-devtools/cmake/cmake-native_3.14.0.bb
index fedcf3d4bd..b2952ee5f5 100644
--- a/meta/recipes-devtools/cmake/cmake-native_3.14.0.bb
+++ b/meta/recipes-devtools/cmake/cmake-native_3.14.0.bb
@@ -1,7 +1,7 @@
 require cmake.inc
 inherit native
 
-DEPENDS += "bzip2-replacement-native expat-native xz-native zlib-native curl-native"
+DEPENDS += "bzip2-replacement-native expat-native xz-native zlib-native curl-native ncurses-native"
 
 SRC_URI += "file://OEToolchainConfig.cmake \
             file://environment.d-cmake.sh \
@@ -13,10 +13,9 @@ SRC_URI += "file://OEToolchainConfig.cmake \
 B = "${WORKDIR}/build"
 do_configure[cleandirs] = "${B}"
 
-# Disable ccmake since we don't depend on ncurses
 CMAKE_EXTRACONF = "\
     -DCMAKE_LIBRARY_PATH=${STAGING_LIBDIR_NATIVE} \
-    -DBUILD_CursesDialog=0 \
+    -DBUILD_CursesDialog=1 \
     -DCMAKE_USE_SYSTEM_LIBRARIES=1 \
     -DCMAKE_USE_SYSTEM_LIBRARY_JSONCPP=0 \
     -DCMAKE_USE_SYSTEM_LIBRARY_LIBARCHIVE=0 \
---
2.20.1

[PATCH 2/3] ccmake.bbclass: Create a cml1 style class for the CMake curses UI

[Nathan Rossi]

The ccmake bbclass implements two tasks. The first task 'ccmake'
preserves the configured state of CMakeCache.txt (generated from the
configure task) and invokes the 'ccmake' program within a oe_terminal
execution. The user can then review, select and modify configuration
options and once satisfied with the configuration exit ccmake. Once
ccmake has exited the build can be run and the updated configuration
should be reflected in the output build.

The ccmake bbclass has a second task 'ccmake_diffconfig' to compute the
differences in configuration which was modified by ccmake. Since there
are many ways to persist the configuration changes within recipes and
layer configuration, the differences are emitted as a bitbake recipe
fragment (configuration.inc) using EXTRA_OECMAKE as well as a CMake
script file which can be used as a input to cmake via the '-C' argument.
Both files are generated in the WORKDIR of the build and the paths to
the files are written as output from the build. It is then up to the
user to take this configuration and apply it to the desired location.

Signed-off-by: Nathan Rossi <nathan@nathanrossi.com>
---
 meta/classes/ccmake.bbclass | 97 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 meta/classes/ccmake.bbclass

diff --git a/meta/classes/ccmake.bbclass b/meta/classes/ccmake.bbclass
new file mode 100644
index 0000000000..933e122b35
--- /dev/null
+++ b/meta/classes/ccmake.bbclass
@@ -0,0 +1,97 @@
+inherit terminal
+
+python do_ccmake() {
+    import shutil
+
+    # copy current config for diffing
+    config = os.path.join(d.getVar("B"), "CMakeCache.txt")
+    if os.path.exists(config):
+        shutil.copy(config, config + ".orig")
+
+    oe_terminal(d.expand("ccmake ${OECMAKE_GENERATOR_ARGS} ${OECMAKE_SOURCEPATH} -Wno-dev"),
+        d.getVar("PN") + " - ccmake", d)
+
+    if os.path.exists(config) and os.path.exists(config + ".orig"):
+        if bb.utils.md5_file(config) != bb.utils.md5_file(config + ".orig"):
+            # the cmake class uses cmake --build, which will by default
+            # regenerate configuration, simply mark the compile step as tainted
+            # to ensure it is re-run
+            bb.note("Configuration changed, recompile will be forced")
+            bb.build.write_taint('do_compile', d)
+
+}
+do_ccmake[depends] += "cmake-native:do_populate_sysroot"
+do_ccmake[nostamp] = "1"
+do_ccmake[dirs] = "${B}"
+addtask ccmake after do_configure
+
+def cmake_parse_config_cache(path):
+    with open(path, "r") as f:
+        for i in f:
+            i = i.rstrip("\n")
+            if len(i) == 0 or i.startswith("//") or i.startswith("#"):
+                continue # empty or comment
+            key, value = i.split("=", 1)
+            key, keytype = key.split(":")
+            if keytype in ["INTERNAL", "STATIC"]:
+                continue # skip internal and static config options
+            yield key, keytype, value
+
+def cmake_diff_config_vars(a, b):
+    removed, added = [], []
+
+    for ak, akt, av in a:
+        found = False
+        for bk, bkt, bv in b:
+            if bk == ak:
+                found = True
+                if bkt != akt or bv != av: # changed
+                    removed.append((ak, akt, av))
+                    added.append((bk, bkt, bv))
+                break
+        # remove any missing from b
+        if not found:
+            removed.append((ak, akt, av))
+
+    # add any missing from a
+    for bk, bkt, bv in b:
+        if not any(bk == ak for ak, akt, av in a):
+            added.append((bk, bkt, bv))
+
+    return removed, added
+
+python do_ccmake_diffconfig() {
+    import shutil
+    config = os.path.join(d.getVar("B"), "CMakeCache.txt")
+    if os.path.exists(config) and os.path.exists(config + ".orig"):
+        if bb.utils.md5_file(config) != bb.utils.md5_file(config + ".orig"):
+            # scan the changed options
+            old = list(cmake_parse_config_cache(config + ".orig"))
+            new = list(cmake_parse_config_cache(config))
+            _, added = cmake_diff_config_vars(old, new)
+
+            if len(added) != 0:
+                with open(d.expand("${WORKDIR}/configuration.inc"), "w") as f:
+                    f.write("EXTRA_OECMAKE += \" \\\n")
+                    for k, kt, v in added:
+                        escaped = v if " " not in v else "\"{0}\"".format(v)
+                        f.write("    -D{0}:{1}={2} \\\n".format(k, kt, escaped))
+                    f.write("    \"\n")
+                bb.plain("Configuration recipe fragment written to: {0}".format(d.expand("${WORKDIR}/configuration.inc")))
+
+                with open(d.expand("${WORKDIR}/site-file.cmake"), "w") as f:
+                    for k, kt, v in added:
+                        f.write("SET({0} \"{1}\" CACHE {2} "")\n".format(k, v, kt))
+                bb.plain("Configuration cmake fragment written to: {0}".format(d.expand("${WORKDIR}/site-file.cmake")))
+
+                # restore the original config
+                shutil.copy(config + ".orig", config)
+            else:
+                bb.plain("No configuration differences, skipping configuration fragment generation.")
+    else:
+        bb.fatal("No config files found. Did you run ccmake?\n{0}".format(e))
+}
+do_ccmake_diffconfig[nostamp] = "1"
+do_ccmake_diffconfig[dirs] = "${B}"
+addtask ccmake_diffconfig
+
---
2.20.1

[PATCH 3/3] devtool: standard: Handle exporting generated config fragments

[Nathan Rossi]

The cml1 and ccmake bbclasses generate configuration fragment source
files that must be exported from the WORKDIR as a source file to be
preserved across builds. This change adds detection of the current
recipes inherited classes and for cml1 and ccmake classes checks for the
specific generated configuration fragment files. These files are then
exported by devtool and included as SRC_URI files from within the target
layer.

Signed-off-by: Nathan Rossi <nathan@nathanrossi.com>
---
 scripts/lib/devtool/standard.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index ea09bbff31..0a1e329e61 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -1328,6 +1328,20 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
                 if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
                     os.unlink(os.path.join(local_files_dir, fragment_fn))
 
+    # Special handling for cml1, ccmake, etc bbclasses that generated
+    # configuration fragment files that are consumed as source files
+    for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]:
+        if bb.data.inherits_class(frag_class, rd):
+            srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name)
+            if os.path.exists(srcpath):
+                if frag_name not in new_set:
+                    new_set.append(frag_name)
+                # copy fragment into destdir
+                shutil.copy2(srcpath, destdir)
+                # copy fragment into local files if exists
+                if os.path.isdir(local_files_dir):
+                    shutil.copy2(srcpath, local_files_dir)
+
     if new_set is not None:
         for fname in new_set:
             if fname in existing_files:
---
2.20.1

Re: [PATCH 2/3] ccmake.bbclass: Create a cml1 style class for the CMake curses UI

[Nathan Rossi]

On Tue, 2 Apr 2019 at 18:22, Nathan Rossi <nathan@nathanrossi.com> wrote:
>
> The ccmake bbclass implements two tasks. The first task 'ccmake'
> preserves the configured state of CMakeCache.txt (generated from the
> configure task) and invokes the 'ccmake' program within a oe_terminal
> execution. The user can then review, select and modify configuration
> options and once satisfied with the configuration exit ccmake. Once
> ccmake has exited the build can be run and the updated configuration
> should be reflected in the output build.
>
> The ccmake bbclass has a second task 'ccmake_diffconfig' to compute the
> differences in configuration which was modified by ccmake. Since there
> are many ways to persist the configuration changes within recipes and
> layer configuration, the differences are emitted as a bitbake recipe
> fragment (configuration.inc) using EXTRA_OECMAKE as well as a CMake
> script file which can be used as a input to cmake via the '-C' argument.
> Both files are generated in the WORKDIR of the build and the paths to
> the files are written as output from the build. It is then up to the
> user to take this configuration and apply it to the desired location.
>
> Signed-off-by: Nathan Rossi <nathan@nathanrossi.com>
> ---
>  meta/classes/ccmake.bbclass | 97 +++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 97 insertions(+)
>  create mode 100644 meta/classes/ccmake.bbclass
>
> diff --git a/meta/classes/ccmake.bbclass b/meta/classes/ccmake.bbclass
> new file mode 100644
> index 0000000000..933e122b35
> --- /dev/null
> +++ b/meta/classes/ccmake.bbclass
> @@ -0,0 +1,97 @@
> +inherit terminal
> +
> +python do_ccmake() {
> +    import shutil
> +
> +    # copy current config for diffing
> +    config = os.path.join(d.getVar("B"), "CMakeCache.txt")
> +    if os.path.exists(config):
> +        shutil.copy(config, config + ".orig")
> +
> +    oe_terminal(d.expand("ccmake ${OECMAKE_GENERATOR_ARGS} ${OECMAKE_SOURCEPATH} -Wno-dev"),
> +        d.getVar("PN") + " - ccmake", d)
> +
> +    if os.path.exists(config) and os.path.exists(config + ".orig"):
> +        if bb.utils.md5_file(config) != bb.utils.md5_file(config + ".orig"):
> +            # the cmake class uses cmake --build, which will by default
> +            # regenerate configuration, simply mark the compile step as tainted
> +            # to ensure it is re-run
> +            bb.note("Configuration changed, recompile will be forced")
> +            bb.build.write_taint('do_compile', d)
> +
> +}
> +do_ccmake[depends] += "cmake-native:do_populate_sysroot"
> +do_ccmake[nostamp] = "1"
> +do_ccmake[dirs] = "${B}"
> +addtask ccmake after do_configure
> +
> +def cmake_parse_config_cache(path):
> +    with open(path, "r") as f:
> +        for i in f:
> +            i = i.rstrip("\n")
> +            if len(i) == 0 or i.startswith("//") or i.startswith("#"):
> +                continue # empty or comment
> +            key, value = i.split("=", 1)
> +            key, keytype = key.split(":")
> +            if keytype in ["INTERNAL", "STATIC"]:
> +                continue # skip internal and static config options
> +            yield key, keytype, value
> +
> +def cmake_diff_config_vars(a, b):
> +    removed, added = [], []
> +
> +    for ak, akt, av in a:
> +        found = False
> +        for bk, bkt, bv in b:
> +            if bk == ak:
> +                found = True
> +                if bkt != akt or bv != av: # changed
> +                    removed.append((ak, akt, av))
> +                    added.append((bk, bkt, bv))
> +                break
> +        # remove any missing from b
> +        if not found:
> +            removed.append((ak, akt, av))
> +
> +    # add any missing from a
> +    for bk, bkt, bv in b:
> +        if not any(bk == ak for ak, akt, av in a):
> +            added.append((bk, bkt, bv))
> +
> +    return removed, added
> +
> +python do_ccmake_diffconfig() {
> +    import shutil
> +    config = os.path.join(d.getVar("B"), "CMakeCache.txt")
> +    if os.path.exists(config) and os.path.exists(config + ".orig"):
> +        if bb.utils.md5_file(config) != bb.utils.md5_file(config + ".orig"):
> +            # scan the changed options
> +            old = list(cmake_parse_config_cache(config + ".orig"))
> +            new = list(cmake_parse_config_cache(config))
> +            _, added = cmake_diff_config_vars(old, new)
> +
> +            if len(added) != 0:
> +                with open(d.expand("${WORKDIR}/configuration.inc"), "w") as f:
> +                    f.write("EXTRA_OECMAKE += \" \\\n")
> +                    for k, kt, v in added:
> +                        escaped = v if " " not in v else "\"{0}\"".format(v)
> +                        f.write("    -D{0}:{1}={2} \\\n".format(k, kt, escaped))
> +                    f.write("    \"\n")
> +                bb.plain("Configuration recipe fragment written to: {0}".format(d.expand("${WORKDIR}/configuration.inc")))
> +
> +                with open(d.expand("${WORKDIR}/site-file.cmake"), "w") as f:
> +                    for k, kt, v in added:
> +                        f.write("SET({0} \"{1}\" CACHE {2} "")\n".format(k, v, kt))
> +                bb.plain("Configuration cmake fragment written to: {0}".format(d.expand("${WORKDIR}/site-file.cmake")))
> +
> +                # restore the original config
> +                shutil.copy(config + ".orig", config)
> +            else:
> +                bb.plain("No configuration differences, skipping configuration fragment generation.")
> +    else:
> +        bb.fatal("No config files found. Did you run ccmake?\n{0}".format(e))

I've just noticed I had not cleaned up the debugging format of an
exception here, I will send a v2 with this fixed up.

Regards,
Nathan

> +}
> +do_ccmake_diffconfig[nostamp] = "1"
> +do_ccmake_diffconfig[dirs] = "${B}"
> +addtask ccmake_diffconfig
> +
> ---
> 2.20.1