[meta-virtualization][kirkstone][master][PATCH 1/4] nerdctl: install the binaries in OE standard places

[meta-virtualization][kirkstone][master][PATCH 1/4] nerdctl: install the binaries in OE standard places

[Jose Quaresma]

- The env BIN_PREFIX is there from the beginning but there are no references to it,
also fix a fatal QA errors installed-vs-shipped:

ERROR: nerdctl-v0.18.0-r0 do_package: QA Issue: nerdctl: Files/directories were installed but not shipped in any package:
  /bin
  /bin/nerdctl
Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
nerdctl: 2 installed and not shipped files. [installed-vs-shipped]
ERROR: nerdctl-v0.18.0-r0 do_package: Fatal QA errors were found, failing task.

Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
---
 recipes-containers/nerdctl/nerdctl_git.bb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/recipes-containers/nerdctl/nerdctl_git.bb b/recipes-containers/nerdctl/nerdctl_git.bb
index 5d4d827..ad18f67 100644
--- a/recipes-containers/nerdctl/nerdctl_git.bb
+++ b/recipes-containers/nerdctl/nerdctl_git.bb
@@ -239,8 +239,8 @@ do_compile() {
 }
 
 do_install() {
-        install -d "${D}${BIN_PREFIX}/bin"
-        install -m 755 "${S}/src/import/_output/nerdctl" "${D}${BIN_PREFIX}/bin"
+        install -d ${D}${bindir}
+        install -m 755 ${S}/src/import/_output/nerdctl ${D}${bindir}
 }
 
 INHIBIT_PACKAGE_STRIP = "1"
-- 
2.37.1

[meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Jose Quaresma]

- The env BIN_PREFIX is there from the beginning but there are no references to it,
also fix a fatal QA errors installed-vs-shipped.

ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: QA Issue: kubernetes: Files/directories were installed but not shipped in any package:
  /bin
  /bin/k8s-init
Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: Fatal QA errors were found, failing task.

Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
---
 recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
index e9460d4..82b75b1 100644
--- a/recipes-containers/kubernetes/kubernetes_git.bb
+++ b/recipes-containers/kubernetes/kubernetes_git.bb
@@ -103,8 +103,8 @@ do_install() {
     install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
 
     if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
-	install -d "${D}${BIN_PREFIX}/bin"
-	install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
+	install -d ${D}${bindir}
+	install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
 
 	install -d ${D}${sysconfdir}/sysctl.d
 	install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
@@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
 FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
 
 ALLOW_EMPTY:${PN}-host = "1"
-FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
+FILE:${PN}-host = "${bindir}/k8s-init"
 RDEPENDS:${PN}-host = "${PN}"
 
 RRECOMMENDS:${PN} = "\
-- 
2.37.1

[meta-virtualization][kirkstone][master][PATCH 3/4] kubernetes: replace tabs with spaces for identation

[Jose Quaresma]

Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
---
 recipes-containers/kubernetes/kubernetes_git.bb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
index 82b75b1..4d65b27 100644
--- a/recipes-containers/kubernetes/kubernetes_git.bb
+++ b/recipes-containers/kubernetes/kubernetes_git.bb
@@ -103,11 +103,11 @@ do_install() {
     install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
 
     if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
-	install -d ${D}${bindir}
-	install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
+        install -d ${D}${bindir}
+        install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
 
-	install -d ${D}${sysconfdir}/sysctl.d
-	install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
+        install -d ${D}${sysconfdir}/sysctl.d
+        install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
     fi
 }
 
-- 
2.37.1

[meta-virtualization][kirkstone][master][PATCH 4/4] k3s: install the binaries in OE standard places

[Jose Quaresma]

Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
---
 recipes-containers/k3s/k3s/k3s-agent.service |  4 ++--
 recipes-containers/k3s/k3s/k3s.service       |  4 ++--
 recipes-containers/k3s/k3s_git.bb            | 22 +++++++++-----------
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/recipes-containers/k3s/k3s/k3s-agent.service b/recipes-containers/k3s/k3s/k3s-agent.service
index 9f9016d..0792970 100644
--- a/recipes-containers/k3s/k3s/k3s-agent.service
+++ b/recipes-containers/k3s/k3s/k3s-agent.service
@@ -21,6 +21,6 @@ Restart=always
 RestartSec=5s
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent
-ExecStopPost=/usr/local/bin/k3s-clean
+ExecStart=/usr/bin/k3s agent
+ExecStopPost=/usr/bin/k3s-clean
 
diff --git a/recipes-containers/k3s/k3s/k3s.service b/recipes-containers/k3s/k3s/k3s.service
index 33d3ee7..647fc66 100644
--- a/recipes-containers/k3s/k3s/k3s.service
+++ b/recipes-containers/k3s/k3s/k3s.service
@@ -29,9 +29,9 @@ RestartSec=5s
 ExecStartPre=/bin/sh -xc '! systemctl is-enabled --quiet nm-cloud-setup.service'
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s server
+ExecStart=/usr/bin/k3s server
 # Avoid any delay due to this service when the system is rebooting or shutting
 # down by using the k3s-killall.sh script to kill all of the running k3s
 # services and containers
 ExecStopPost=/bin/sh -c "if systemctl is-system-running | grep -i \
-                           'stopping'; then /usr/local/bin/k3s-killall.sh; fi"
+                           'stopping'; then /usr/bin/k3s-killall.sh; fi"
diff --git a/recipes-containers/k3s/k3s_git.bb b/recipes-containers/k3s/k3s_git.bb
index 7f9f549..f8595df 100644
--- a/recipes-containers/k3s/k3s_git.bb
+++ b/recipes-containers/k3s/k3s_git.bb
@@ -37,7 +37,6 @@ GO_BUILD_LDFLAGS = "-X github.com/rancher/k3s/pkg/version.Version=${PV} \
                     -X github.com/rancher/k3s/pkg/version.GitCommit=${@d.getVar('SRCREV_k3s', d, 1)[:8]} \
                     -w -s \
                    "
-BIN_PREFIX ?= "${exec_prefix}/local"
 
 inherit features_check
 REQUIRED_DISTRO_FEATURES ?= "seccomp"
@@ -634,20 +633,20 @@ do_compile() {
 }
 
 do_install() {
-        install -d "${D}${BIN_PREFIX}/bin"
-        install -m 755 "${S}/src/import/dist/artifacts/k3s" "${D}${BIN_PREFIX}/bin"
-        ln -sr "${D}/${BIN_PREFIX}/bin/k3s" "${D}${BIN_PREFIX}/bin/crictl"
+        install -d "${D}${bindir}"
+        install -m 755 "${S}/src/import/dist/artifacts/k3s" "${D}${bindir}"
+        ln -sr "${D}${bindir}/k3s" "${D}${bindir}/crictl"
         # We want to use the containerd provided ctr
-        # ln -sr "${D}/${BIN_PREFIX}/bin/k3s" "${D}${BIN_PREFIX}/bin/ctr"
-        ln -sr "${D}/${BIN_PREFIX}/bin/k3s" "${D}${BIN_PREFIX}/bin/kubectl"
-        install -m 755 "${WORKDIR}/k3s-clean" "${D}${BIN_PREFIX}/bin"
-        install -m 755 "${WORKDIR}/k3s-killall.sh" "${D}${BIN_PREFIX}/bin"
+        # ln -sr "${D}${bindir}/k3s" "${D}${bindir}/ctr"
+        ln -sr "${D}${bindir}/k3s" "${D}${bindir}/kubectl"
+        install -m 755 "${WORKDIR}/k3s-clean" "${D}${bindir}"
+        install -m 755 "${WORKDIR}/k3s-killall.sh" "${D}${bindir}"
 
         if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
                 install -D -m 0644 "${WORKDIR}/k3s.service" "${D}${systemd_system_unitdir}/k3s.service"
                 install -D -m 0644 "${WORKDIR}/k3s-agent.service" "${D}${systemd_system_unitdir}/k3s-agent.service"
-                sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${BIN_PREFIX}/bin/\4#g" "${D}${systemd_system_unitdir}/k3s.service" "${D}${systemd_system_unitdir}/k3s-agent.service"
-                install -m 755 "${WORKDIR}/k3s-agent" "${D}${BIN_PREFIX}/bin"
+                sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${bindir}/\4#g" "${D}${systemd_system_unitdir}/k3s.service" "${D}${systemd_system_unitdir}/k3s-agent.service"
+                install -m 755 "${WORKDIR}/k3s-agent" "${D}${bindir}"
         fi
 }
 
@@ -658,8 +657,7 @@ SYSTEMD_SERVICE:${PN}-server = "${@bb.utils.contains('DISTRO_FEATURES','systemd'
 SYSTEMD_SERVICE:${PN}-agent = "${@bb.utils.contains('DISTRO_FEATURES','systemd','k3s-agent.service','',d)}"
 SYSTEMD_AUTO_ENABLE:${PN}-agent = "disable"
 
-FILES:${PN}-agent = "${BIN_PREFIX}/bin/k3s-agent"
-FILES:${PN} += "${BIN_PREFIX}/bin/*"
+FILES:${PN}-agent = "${bindir}/k3s-agent"
 
 RDEPENDS:${PN} = "k3s-cni conntrack-tools coreutils findutils iptables iproute2 ipset virtual-containerd"
 RDEPENDS:${PN}-server = "${PN}"
-- 
2.37.1

Re: [meta-virtualization][kirkstone][master][PATCH 1/4] nerdctl: install the binaries in OE standard places

[Bruce Ashfield]

On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>
> - The env BIN_PREFIX is there from the beginning but there are no references to it,
> also fix a fatal QA errors installed-vs-shipped:
>
> ERROR: nerdctl-v0.18.0-r0 do_package: QA Issue: nerdctl: Files/directories were installed but not shipped in any package:
>   /bin
>   /bin/nerdctl
> Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
> nerdctl: 2 installed and not shipped files. [installed-vs-shipped]
> ERROR: nerdctl-v0.18.0-r0 do_package: Fatal QA errors were found, failing task.
>

I'm building nerdctl constantly, and I'm not seeing this.

So until I can reproduce the issue, I can't merge this change.

Bruce

> Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
> ---
>  recipes-containers/nerdctl/nerdctl_git.bb | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/recipes-containers/nerdctl/nerdctl_git.bb b/recipes-containers/nerdctl/nerdctl_git.bb
> index 5d4d827..ad18f67 100644
> --- a/recipes-containers/nerdctl/nerdctl_git.bb
> +++ b/recipes-containers/nerdctl/nerdctl_git.bb
> @@ -239,8 +239,8 @@ do_compile() {
>  }
>
>  do_install() {
> -        install -d "${D}${BIN_PREFIX}/bin"
> -        install -m 755 "${S}/src/import/_output/nerdctl" "${D}${BIN_PREFIX}/bin"
> +        install -d ${D}${bindir}
> +        install -m 755 ${S}/src/import/_output/nerdctl ${D}${bindir}
>  }
>
>  INHIBIT_PACKAGE_STRIP = "1"
> --
> 2.37.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#7495): https://lists.yoctoproject.org/g/meta-virtualization/message/7495
> Mute This Topic: https://lists.yoctoproject.org/mt/92913425/1050810
> Group Owner: meta-virtualization+owner@lists.yoctoproject.org
> Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub [bruce.ashfield@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>


-- 
- Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end
- "Use the force Harry" - Gandalf, Star Trek II

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Bruce Ashfield]

You must have something different in your environment, I'm not seeing
any of these same issues.

Cheers,

Bruce

On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>
> - The env BIN_PREFIX is there from the beginning but there are no references to it,
> also fix a fatal QA errors installed-vs-shipped.
>
> ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: QA Issue: kubernetes: Files/directories were installed but not shipped in any package:
>   /bin
>   /bin/k8s-init
> Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
> kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
> ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: Fatal QA errors were found, failing task.
>
> Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
> ---
>  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
>
> diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
> index e9460d4..82b75b1 100644
> --- a/recipes-containers/kubernetes/kubernetes_git.bb
> +++ b/recipes-containers/kubernetes/kubernetes_git.bb
> @@ -103,8 +103,8 @@ do_install() {
>      install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
>
>      if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
> -       install -d "${D}${BIN_PREFIX}/bin"
> -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
> +       install -d ${D}${bindir}
> +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
>
>         install -d ${D}${sysconfdir}/sysctl.d
>         install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
> @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
>  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
>
>  ALLOW_EMPTY:${PN}-host = "1"
> -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
> +FILE:${PN}-host = "${bindir}/k8s-init"
>  RDEPENDS:${PN}-host = "${PN}"
>
>  RRECOMMENDS:${PN} = "\
> --
> 2.37.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#7496): https://lists.yoctoproject.org/g/meta-virtualization/message/7496
> Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
> Group Owner: meta-virtualization+owner@lists.yoctoproject.org
> Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub [bruce.ashfield@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>


-- 
- Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end
- "Use the force Harry" - Gandalf, Star Trek II

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Jose Quaresma]

[-- Attachment #1: Type: text/plain, Size: 3423 bytes --]

Hi Bruce,

Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022
à(s) 14:19:

> You must have something different in your environment, I'm not seeing
> any of these same issues.
>

Maybe because you have the BIN_PREFIX defined somewhere in your distro or
local.conf
BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?=
"${exec_prefix}/local"

Jose


>
> Cheers,
>
> Bruce
>
> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com>
> wrote:
> >
> > - The env BIN_PREFIX is there from the beginning but there are no
> references to it,
> > also fix a fatal QA errors installed-vs-shipped.
> >
> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: QA Issue: kubernetes: Files/directories were installed but not
> shipped in any package:
> >   /bin
> >   /bin/k8s-init
> > Please set FILES such that these items are packaged. Alternatively if
> they are unneeded, avoid installing them or delete them within do_install.
> > kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: Fatal QA errors were found, failing task.
> >
> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
> > ---
> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
> >  1 file changed, 3 insertions(+), 3 deletions(-)
> >
> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb
> b/recipes-containers/kubernetes/kubernetes_git.bb
> > index e9460d4..82b75b1 100644
> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
> > @@ -103,8 +103,8 @@ do_install() {
> >      install -m 0644
> ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf
> ${D}${systemd_unitdir}/system/kubelet.service.d/
> >
> >      if
> ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
> > -       install -d "${D}${BIN_PREFIX}/bin"
> > -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
> > +       install -d ${D}${bindir}
> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
> >
> >         install -d ${D}${sysconfdir}/sysctl.d
> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf"
> "${D}${sysconfdir}/sysctl.d"
> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
> >
> >  ALLOW_EMPTY:${PN}-host = "1"
> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
> > +FILE:${PN}-host = "${bindir}/k8s-init"
> >  RDEPENDS:${PN}-host = "${PN}"
> >
> >  RRECOMMENDS:${PN} = "\
> > --
> > 2.37.1
> >
> >
> > -=-=-=-=-=-=-=-=-=-=-=-
> > Links: You receive all messages sent to this group.
> > View/Reply Online (#7496):
> https://lists.yoctoproject.org/g/meta-virtualization/message/7496
> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
> > Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub
> [bruce.ashfield@gmail.com]
> > -=-=-=-=-=-=-=-=-=-=-=-
> >
>
>
> --
> - Thou shalt not follow the NULL pointer, for chaos and madness await
> thee at its end
> - "Use the force Harry" - Gandalf, Star Trek II
>


-- 
Best regards,

José Quaresma

[-- Attachment #2: Type: text/html, Size: 5555 bytes --]

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Bruce Ashfield]

On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>
> Hi Bruce,
>
> Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:19:
>>
>> You must have something different in your environment, I'm not seeing
>> any of these same issues.
>
>
> Maybe because you have the BIN_PREFIX defined somewhere in your distro or local.conf
> BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?= "${exec_prefix}/local"
>

I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
environment anywhere (which is admittedly strange .. but that's
consistent with how it has been).

Also, there's no other reports of this ever on the mailing list,
including demos for conferences, etc. ... that fails the "how can this
be working for everyone else ?" test.

So there's definitely something different that I'm not seeing. I use
OE nodistro or poky, others are using some other distros .. so I need
to figure out what is the difference.

That being said, even if we tweaked the binaries install, we don't
want them changing where they've been installed previously, there
could be any number of scripts expecting those locations in layers
that I don't maintain.

i.e. there's no way we should be patching the .service file, since
that indicates the binaries have moved from where they were before.

Bruce



> Jose
>
>>
>>
>> Cheers,
>>
>> Bruce
>>
>> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> >
>> > - The env BIN_PREFIX is there from the beginning but there are no references to it,
>> > also fix a fatal QA errors installed-vs-shipped.
>> >
>> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: QA Issue: kubernetes: Files/directories were installed but not shipped in any package:
>> >   /bin
>> >   /bin/k8s-init
>> > Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
>> > kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
>> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: Fatal QA errors were found, failing task.
>> >
>> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
>> > ---
>> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
>> >  1 file changed, 3 insertions(+), 3 deletions(-)
>> >
>> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
>> > index e9460d4..82b75b1 100644
>> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
>> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
>> > @@ -103,8 +103,8 @@ do_install() {
>> >      install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
>> >
>> >      if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
>> > -       install -d "${D}${BIN_PREFIX}/bin"
>> > -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
>> > +       install -d ${D}${bindir}
>> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
>> >
>> >         install -d ${D}${sysconfdir}/sysctl.d
>> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
>> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
>> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
>> >
>> >  ALLOW_EMPTY:${PN}-host = "1"
>> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
>> > +FILE:${PN}-host = "${bindir}/k8s-init"
>> >  RDEPENDS:${PN}-host = "${PN}"
>> >
>> >  RRECOMMENDS:${PN} = "\
>> > --
>> > 2.37.1
>> >
>> >
>> > -=-=-=-=-=-=-=-=-=-=-=-
>> > Links: You receive all messages sent to this group.
>> > View/Reply Online (#7496): https://lists.yoctoproject.org/g/meta-virtualization/message/7496
>> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
>> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
>> > Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub [bruce.ashfield@gmail.com]
>> > -=-=-=-=-=-=-=-=-=-=-=-
>> >
>>
>>
>> --
>> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> thee at its end
>> - "Use the force Harry" - Gandalf, Star Trek II
>
>
>
> --
> Best regards,
>
> José Quaresma



--
- Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end
- "Use the force Harry" - Gandalf, Star Trek II

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Jose Quaresma]

[-- Attachment #1: Type: text/plain, Size: 5603 bytes --]

Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022
à(s) 14:43:

> On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <quaresma.jose@gmail.com>
> wrote:
> >
> > Hi Bruce,
> >
> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça,
> 9/08/2022 à(s) 14:19:
> >>
> >> You must have something different in your environment, I'm not seeing
> >> any of these same issues.
> >
> >
> > Maybe because you have the BIN_PREFIX defined somewhere in your distro
> or local.conf
> > BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?=
> "${exec_prefix}/local"
> >
>
> I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
> environment anywhere (which is admittedly strange .. but that's
> consistent with how it has been).
>
> Also, there's no other reports of this ever on the mailing list,
> including demos for conferences, etc. ... that fails the "how can this
> be working for everyone else ?" test.
>
> So there's definitely something different that I'm not seeing. I use
> OE nodistro or poky, others are using some other distros .. so I need
> to figure out what is the difference.
>
> That being said, even if we tweaked the binaries install, we don't
> want them changing where they've been installed previously, there
> could be any number of scripts expecting those locations in layers
> that I don't maintain.
>
> i.e. there's no way we should be patching the .service file, since
> that indicates the binaries have moved from where they were before.
>
> Bruce
>
>
By default with BIN_PREFIX="" the binaries seem to be installed on
/bin/k8s-init
so they will work as this is in the PATH.
For OE nodistro maybe this QA is disabled (need to confirm that) but for
distros that have this QA enabled it will fire up.

About moving the binaries for another place is mainly because /bin is not
the right place for them IMO.
https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/bin.html

But I forgot to update all the services involved in this patch set so
please drop it all and I will update all services as well in V2.

Jose


>
> > Jose
> >
> >>
> >>
> >> Cheers,
> >>
> >> Bruce
> >>
> >> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com>
> wrote:
> >> >
> >> > - The env BIN_PREFIX is there from the beginning but there are no
> references to it,
> >> > also fix a fatal QA errors installed-vs-shipped.
> >> >
> >> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: QA Issue: kubernetes: Files/directories were installed but not
> shipped in any package:
> >> >   /bin
> >> >   /bin/k8s-init
> >> > Please set FILES such that these items are packaged. Alternatively if
> they are unneeded, avoid installing them or delete them within do_install.
> >> > kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
> >> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: Fatal QA errors were found, failing task.
> >> >
> >> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
> >> > ---
> >> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
> >> >  1 file changed, 3 insertions(+), 3 deletions(-)
> >> >
> >> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb
> b/recipes-containers/kubernetes/kubernetes_git.bb
> >> > index e9460d4..82b75b1 100644
> >> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
> >> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
> >> > @@ -103,8 +103,8 @@ do_install() {
> >> >      install -m 0644
> ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf
> ${D}${systemd_unitdir}/system/kubelet.service.d/
> >> >
> >> >      if
> ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
> >> > -       install -d "${D}${BIN_PREFIX}/bin"
> >> > -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
> >> > +       install -d ${D}${bindir}
> >> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
> >> >
> >> >         install -d ${D}${sysconfdir}/sysctl.d
> >> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf"
> "${D}${sysconfdir}/sysctl.d"
> >> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
> >> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
> >> >
> >> >  ALLOW_EMPTY:${PN}-host = "1"
> >> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
> >> > +FILE:${PN}-host = "${bindir}/k8s-init"
> >> >  RDEPENDS:${PN}-host = "${PN}"
> >> >
> >> >  RRECOMMENDS:${PN} = "\
> >> > --
> >> > 2.37.1
> >> >
> >> >
> >> > -=-=-=-=-=-=-=-=-=-=-=-
> >> > Links: You receive all messages sent to this group.
> >> > View/Reply Online (#7496):
> https://lists.yoctoproject.org/g/meta-virtualization/message/7496
> >> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
> >> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
> >> > Unsubscribe:
> https://lists.yoctoproject.org/g/meta-virtualization/unsub [
> bruce.ashfield@gmail.com]
> >> > -=-=-=-=-=-=-=-=-=-=-=-
> >> >
> >>
> >>
> >> --
> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
> >> thee at its end
> >> - "Use the force Harry" - Gandalf, Star Trek II
> >
> >
> >
> > --
> > Best regards,
> >
> > José Quaresma
>
>
>
> --
> - Thou shalt not follow the NULL pointer, for chaos and madness await
> thee at its end
> - "Use the force Harry" - Gandalf, Star Trek II
>


-- 
Best regards,

José Quaresma

[-- Attachment #2: Type: text/html, Size: 8647 bytes --]

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Bruce Ashfield]

On Tue, Aug 9, 2022 at 10:02 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>
>
>
> Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:43:
>>
>> On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> >
>> > Hi Bruce,
>> >
>> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:19:
>> >>
>> >> You must have something different in your environment, I'm not seeing
>> >> any of these same issues.
>> >
>> >
>> > Maybe because you have the BIN_PREFIX defined somewhere in your distro or local.conf
>> > BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?= "${exec_prefix}/local"
>> >
>>
>> I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
>> environment anywhere (which is admittedly strange .. but that's
>> consistent with how it has been).
>>
>> Also, there's no other reports of this ever on the mailing list,
>> including demos for conferences, etc. ... that fails the "how can this
>> be working for everyone else ?" test.
>>
>> So there's definitely something different that I'm not seeing. I use
>> OE nodistro or poky, others are using some other distros .. so I need
>> to figure out what is the difference.
>>
>> That being said, even if we tweaked the binaries install, we don't
>> want them changing where they've been installed previously, there
>> could be any number of scripts expecting those locations in layers
>> that I don't maintain.
>>
>> i.e. there's no way we should be patching the .service file, since
>> that indicates the binaries have moved from where they were before.
>>
>> Bruce
>>
>
> By default with BIN_PREFIX="" the binaries seem to be installed on /bin/k8s-init so they will work as this is in the PATH.
> For OE nodistro maybe this QA is disabled (need to confirm that) but for distros that have this QA enabled it will fire up.
>

The installed but not shipped, isn't inhibited anywhere that I know
of. Certainly not on poky, which is where most of the nightly builds
happen for this.

> About moving the binaries for another place is mainly because /bin is not the right place for them IMO.
> https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/bin.html

But that's not where they are going for all the rest of the builds :)
I'm aware of the various filesystem standards.

>
> But I forgot to update all the services involved in this patch set so please drop it all and I will update all services as well in V2.
>

I wouldn't bother with that for v2, since again, they are using
/usr/bin and /usr/local/bin from all the .service files I recall, and
that's where we want the binaries to continue to be.

Bruce

> Jose
>
>>
>>
>> > Jose
>> >
>> >>
>> >>
>> >> Cheers,
>> >>
>> >> Bruce
>> >>
>> >> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> >> >
>> >> > - The env BIN_PREFIX is there from the beginning but there are no references to it,
>> >> > also fix a fatal QA errors installed-vs-shipped.
>> >> >
>> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: QA Issue: kubernetes: Files/directories were installed but not shipped in any package:
>> >> >   /bin
>> >> >   /bin/k8s-init
>> >> > Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
>> >> > kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
>> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: Fatal QA errors were found, failing task.
>> >> >
>> >> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
>> >> > ---
>> >> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
>> >> >  1 file changed, 3 insertions(+), 3 deletions(-)
>> >> >
>> >> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
>> >> > index e9460d4..82b75b1 100644
>> >> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
>> >> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
>> >> > @@ -103,8 +103,8 @@ do_install() {
>> >> >      install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
>> >> >
>> >> >      if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
>> >> > -       install -d "${D}${BIN_PREFIX}/bin"
>> >> > -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
>> >> > +       install -d ${D}${bindir}
>> >> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
>> >> >
>> >> >         install -d ${D}${sysconfdir}/sysctl.d
>> >> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
>> >> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
>> >> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
>> >> >
>> >> >  ALLOW_EMPTY:${PN}-host = "1"
>> >> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
>> >> > +FILE:${PN}-host = "${bindir}/k8s-init"
>> >> >  RDEPENDS:${PN}-host = "${PN}"
>> >> >
>> >> >  RRECOMMENDS:${PN} = "\
>> >> > --
>> >> > 2.37.1
>> >> >
>> >> >
>> >> > -=-=-=-=-=-=-=-=-=-=-=-
>> >> > Links: You receive all messages sent to this group.
>> >> > View/Reply Online (#7496): https://lists.yoctoproject.org/g/meta-virtualization/message/7496
>> >> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
>> >> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
>> >> > Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub [bruce.ashfield@gmail.com]
>> >> > -=-=-=-=-=-=-=-=-=-=-=-
>> >> >
>> >>
>> >>
>> >> --
>> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> >> thee at its end
>> >> - "Use the force Harry" - Gandalf, Star Trek II
>> >
>> >
>> >
>> > --
>> > Best regards,
>> >
>> > José Quaresma
>>
>>
>>
>> --
>> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> thee at its end
>> - "Use the force Harry" - Gandalf, Star Trek II
>
>
>
> --
> Best regards,
>
> José Quaresma



-- 
- Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end
- "Use the force Harry" - Gandalf, Star Trek II

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Bruce Ashfield]

On Tue, Aug 9, 2022 at 10:21 AM Bruce Ashfield via
lists.yoctoproject.org
<bruce.ashfield=gmail.com@lists.yoctoproject.org> wrote:
>
> On Tue, Aug 9, 2022 at 10:02 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
> >
> >
> >
> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:43:
> >>
> >> On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
> >> >
> >> > Hi Bruce,
> >> >
> >> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:19:
> >> >>
> >> >> You must have something different in your environment, I'm not seeing
> >> >> any of these same issues.
> >> >
> >> >
> >> > Maybe because you have the BIN_PREFIX defined somewhere in your distro or local.conf
> >> > BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?= "${exec_prefix}/local"
> >> >
> >>
> >> I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
> >> environment anywhere (which is admittedly strange .. but that's
> >> consistent with how it has been).
> >>
> >> Also, there's no other reports of this ever on the mailing list,
> >> including demos for conferences, etc. ... that fails the "how can this
> >> be working for everyone else ?" test.
> >>
> >> So there's definitely something different that I'm not seeing. I use
> >> OE nodistro or poky, others are using some other distros .. so I need
> >> to figure out what is the difference.
> >>
> >> That being said, even if we tweaked the binaries install, we don't
> >> want them changing where they've been installed previously, there
> >> could be any number of scripts expecting those locations in layers
> >> that I don't maintain.
> >>
> >> i.e. there's no way we should be patching the .service file, since
> >> that indicates the binaries have moved from where they were before.
> >>
> >> Bruce
> >>
> >
> > By default with BIN_PREFIX="" the binaries seem to be installed on /bin/k8s-init so they will work as this is in the PATH.
> > For OE nodistro maybe this QA is disabled (need to confirm that) but for distros that have this QA enabled it will fire up.
> >
>
> The installed but not shipped, isn't inhibited anywhere that I know
> of. Certainly not on poky, which is where most of the nightly builds
> happen for this.
>
> > About moving the binaries for another place is mainly because /bin is not the right place for them IMO.
> > https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/bin.html
>
> But that's not where they are going for all the rest of the builds :)
> I'm aware of the various filesystem standards.
>
> >
> > But I forgot to update all the services involved in this patch set so please drop it all and I will update all services as well in V2.
> >
>
> I wouldn't bother with that for v2, since again, they are using
> /usr/bin and /usr/local/bin from all the .service files I recall, and
> that's where we want the binaries to continue to be.
>

aha. I see that k3s does set BIN_PREFIX, but it has been lost for
kubernetes and nerdctl.

The easiest / lowest footprint route is to just add the variable back
in for kubernetes and nerdctl, to the current default location. That
allows folks to override it and/or keep their scripts/service files
unmodified.

Bruce

> Bruce
>
> > Jose
> >
> >>
> >>
> >> > Jose
> >> >
> >> >>
> >> >>
> >> >> Cheers,
> >> >>
> >> >> Bruce
> >> >>
> >> >> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
> >> >> >
> >> >> > - The env BIN_PREFIX is there from the beginning but there are no references to it,
> >> >> > also fix a fatal QA errors installed-vs-shipped.
> >> >> >
> >> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: QA Issue: kubernetes: Files/directories were installed but not shipped in any package:
> >> >> >   /bin
> >> >> >   /bin/k8s-init
> >> >> > Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
> >> >> > kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
> >> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: Fatal QA errors were found, failing task.
> >> >> >
> >> >> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
> >> >> > ---
> >> >> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
> >> >> >  1 file changed, 3 insertions(+), 3 deletions(-)
> >> >> >
> >> >> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
> >> >> > index e9460d4..82b75b1 100644
> >> >> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
> >> >> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
> >> >> > @@ -103,8 +103,8 @@ do_install() {
> >> >> >      install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
> >> >> >
> >> >> >      if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
> >> >> > -       install -d "${D}${BIN_PREFIX}/bin"
> >> >> > -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
> >> >> > +       install -d ${D}${bindir}
> >> >> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
> >> >> >
> >> >> >         install -d ${D}${sysconfdir}/sysctl.d
> >> >> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
> >> >> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
> >> >> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
> >> >> >
> >> >> >  ALLOW_EMPTY:${PN}-host = "1"
> >> >> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
> >> >> > +FILE:${PN}-host = "${bindir}/k8s-init"
> >> >> >  RDEPENDS:${PN}-host = "${PN}"
> >> >> >
> >> >> >  RRECOMMENDS:${PN} = "\
> >> >> > --
> >> >> > 2.37.1
> >> >> >
> >> >> >
> >> >> >
> >> >> >
> >> >>
> >> >>
> >> >> --
> >> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
> >> >> thee at its end
> >> >> - "Use the force Harry" - Gandalf, Star Trek II
> >> >
> >> >
> >> >
> >> > --
> >> > Best regards,
> >> >
> >> > José Quaresma
> >>
> >>
> >>
> >> --
> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
> >> thee at its end
> >> - "Use the force Harry" - Gandalf, Star Trek II
> >
> >
> >
> > --
> > Best regards,
> >
> > José Quaresma
>
>
>
> --
> - Thou shalt not follow the NULL pointer, for chaos and madness await
> thee at its end
> - "Use the force Harry" - Gandalf, Star Trek II
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#7504): https://lists.yoctoproject.org/g/meta-virtualization/message/7504
> Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
> Group Owner: meta-virtualization+owner@lists.yoctoproject.org
> Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub [bruce.ashfield@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>


-- 
- Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end
- "Use the force Harry" - Gandalf, Star Trek II

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Jose Quaresma]

[-- Attachment #1: Type: text/plain, Size: 8910 bytes --]

Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022
à(s) 16:05:

> On Tue, Aug 9, 2022 at 10:21 AM Bruce Ashfield via
> lists.yoctoproject.org
> <bruce.ashfield=gmail.com@lists.yoctoproject.org> wrote:
> >
> > On Tue, Aug 9, 2022 at 10:02 AM Jose Quaresma <quaresma.jose@gmail.com>
> wrote:
> > >
> > >
> > >
> > > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça,
> 9/08/2022 à(s) 14:43:
> > >>
> > >> On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <quaresma.jose@gmail.com>
> wrote:
> > >> >
> > >> > Hi Bruce,
> > >> >
> > >> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça,
> 9/08/2022 à(s) 14:19:
> > >> >>
> > >> >> You must have something different in your environment, I'm not
> seeing
> > >> >> any of these same issues.
> > >> >
> > >> >
> > >> > Maybe because you have the BIN_PREFIX defined somewhere in your
> distro or local.conf
> > >> > BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?=
> "${exec_prefix}/local"
> > >> >
> > >>
> > >> I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
> > >> environment anywhere (which is admittedly strange .. but that's
> > >> consistent with how it has been).
> > >>
> > >> Also, there's no other reports of this ever on the mailing list,
> > >> including demos for conferences, etc. ... that fails the "how can this
> > >> be working for everyone else ?" test.
> > >>
> > >> So there's definitely something different that I'm not seeing. I use
> > >> OE nodistro or poky, others are using some other distros .. so I need
> > >> to figure out what is the difference.
> > >>
> > >> That being said, even if we tweaked the binaries install, we don't
> > >> want them changing where they've been installed previously, there
> > >> could be any number of scripts expecting those locations in layers
> > >> that I don't maintain.
> > >>
> > >> i.e. there's no way we should be patching the .service file, since
> > >> that indicates the binaries have moved from where they were before.
> > >>
> > >> Bruce
> > >>
> > >
> > > By default with BIN_PREFIX="" the binaries seem to be installed on
> /bin/k8s-init so they will work as this is in the PATH.
> > > For OE nodistro maybe this QA is disabled (need to confirm that) but
> for distros that have this QA enabled it will fire up.
> > >
> >
> > The installed but not shipped, isn't inhibited anywhere that I know
> > of. Certainly not on poky, which is where most of the nightly builds
> > happen for this.
> >
> > > About moving the binaries for another place is mainly because /bin is
> not the right place for them IMO.
> > > https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/bin.html
> >
> > But that's not where they are going for all the rest of the builds :)
> > I'm aware of the various filesystem standards.
> >
> > >
> > > But I forgot to update all the services involved in this patch set so
> please drop it all and I will update all services as well in V2.
> > >
> >
> > I wouldn't bother with that for v2, since again, they are using
> > /usr/bin and /usr/local/bin from all the .service files I recall, and
> > that's where we want the binaries to continue to be.
> >
>
> aha. I see that k3s does set BIN_PREFIX, but it has been lost for
> kubernetes and nerdctl.
>

right, maybe you don't see the QA because it will only shon for files
installed in /bin when usrmerge is enabled.

for nerdctl, the BIN_PREFIX is there from the beginning so the recipe
allway install it in /bin.
for kubernetes, BIN_PREFIX is added in [1] and this only install k8s-init
in /bin and all the other binaries
continue where they were installed (in /usr/bin in this case).

[1] -
https://git.yoctoproject.org/meta-virtualization/commit/?id=4d0f0a5ca2338e5f6ed3fe3a18c602447cf60eb4


> The easiest / lowest footprint route is to just add the variable back
> in for kubernetes and nerdctl, to the current default location. That
> allows folks to override it and/or keep their scripts/service files
> unmodified.
>

This is what I did but using ${bindir} instead of ${BIN_PREFIX}.
Adding the BIN_PREFIX ?= "${exec_prefix}/local" will change install
location so
since we have to change, we can use the ${bindir} and put the files in the
right place.

The last patch for k3s is more invasive as it touches many places and the
systemd services as well.
I will send it separate because this one don't fix anything and only change
the installation of binaries
from /usr/local/bin to /usr/bin

Jose


> Bruce
>
> > Bruce
> >
> > > Jose
> > >
> > >>
> > >>
> > >> > Jose
> > >> >
> > >> >>
> > >> >>
> > >> >> Cheers,
> > >> >>
> > >> >> Bruce
> > >> >>
> > >> >> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <
> quaresma.jose@gmail.com> wrote:
> > >> >> >
> > >> >> > - The env BIN_PREFIX is there from the beginning but there are
> no references to it,
> > >> >> > also fix a fatal QA errors installed-vs-shipped.
> > >> >> >
> > >> >> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: QA Issue: kubernetes: Files/directories were installed but not
> shipped in any package:
> > >> >> >   /bin
> > >> >> >   /bin/k8s-init
> > >> >> > Please set FILES such that these items are packaged.
> Alternatively if they are unneeded, avoid installing them or delete them
> within do_install.
> > >> >> > kubernetes: 2 installed and not shipped files.
> [installed-vs-shipped]
> > >> >> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: Fatal QA errors were found, failing task.
> > >> >> >
> > >> >> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
> > >> >> > ---
> > >> >> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
> > >> >> >  1 file changed, 3 insertions(+), 3 deletions(-)
> > >> >> >
> > >> >> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb
> b/recipes-containers/kubernetes/kubernetes_git.bb
> > >> >> > index e9460d4..82b75b1 100644
> > >> >> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
> > >> >> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
> > >> >> > @@ -103,8 +103,8 @@ do_install() {
> > >> >> >      install -m 0644
> ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf
> ${D}${systemd_unitdir}/system/kubelet.service.d/
> > >> >> >
> > >> >> >      if
> ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
> > >> >> > -       install -d "${D}${BIN_PREFIX}/bin"
> > >> >> > -       install -m 755 "${WORKDIR}/k8s-init"
> "${D}${BIN_PREFIX}/bin"
> > >> >> > +       install -d ${D}${bindir}
> > >> >> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
> > >> >> >
> > >> >> >         install -d ${D}${sysconfdir}/sysctl.d
> > >> >> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf"
> "${D}${sysconfdir}/sysctl.d"
> > >> >> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
> > >> >> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
> > >> >> >
> > >> >> >  ALLOW_EMPTY:${PN}-host = "1"
> > >> >> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
> > >> >> > +FILE:${PN}-host = "${bindir}/k8s-init"
> > >> >> >  RDEPENDS:${PN}-host = "${PN}"
> > >> >> >
> > >> >> >  RRECOMMENDS:${PN} = "\
> > >> >> > --
> > >> >> > 2.37.1
> > >> >> >
> > >> >> >
> > >> >> >
> > >> >> >
> > >> >>
> > >> >>
> > >> >> --
> > >> >> - Thou shalt not follow the NULL pointer, for chaos and madness
> await
> > >> >> thee at its end
> > >> >> - "Use the force Harry" - Gandalf, Star Trek II
> > >> >
> > >> >
> > >> >
> > >> > --
> > >> > Best regards,
> > >> >
> > >> > José Quaresma
> > >>
> > >>
> > >>
> > >> --
> > >> - Thou shalt not follow the NULL pointer, for chaos and madness await
> > >> thee at its end
> > >> - "Use the force Harry" - Gandalf, Star Trek II
> > >
> > >
> > >
> > > --
> > > Best regards,
> > >
> > > José Quaresma
> >
> >
> >
> > --
> > - Thou shalt not follow the NULL pointer, for chaos and madness await
> > thee at its end
> > - "Use the force Harry" - Gandalf, Star Trek II
> >
> > -=-=-=-=-=-=-=-=-=-=-=-
> > Links: You receive all messages sent to this group.
> > View/Reply Online (#7504):
> https://lists.yoctoproject.org/g/meta-virtualization/message/7504
> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
> > Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub
> [bruce.ashfield@gmail.com]
> > -=-=-=-=-=-=-=-=-=-=-=-
> >
>
>
> --
> - Thou shalt not follow the NULL pointer, for chaos and madness await
> thee at its end
> - "Use the force Harry" - Gandalf, Star Trek II
>


-- 
Best regards,

José Quaresma

[-- Attachment #2: Type: text/html, Size: 13873 bytes --]

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Bruce Ashfield]

On Tue, Aug 9, 2022 at 12:36 PM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>
>
>
> Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 16:05:
>>
>> On Tue, Aug 9, 2022 at 10:21 AM Bruce Ashfield via
>> lists.yoctoproject.org
>> <bruce.ashfield=gmail.com@lists.yoctoproject.org> wrote:
>> >
>> > On Tue, Aug 9, 2022 at 10:02 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> > >
>> > >
>> > >
>> > > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:43:
>> > >>
>> > >> On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> > >> >
>> > >> > Hi Bruce,
>> > >> >
>> > >> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:19:
>> > >> >>
>> > >> >> You must have something different in your environment, I'm not seeing
>> > >> >> any of these same issues.
>> > >> >
>> > >> >
>> > >> > Maybe because you have the BIN_PREFIX defined somewhere in your distro or local.conf
>> > >> > BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?= "${exec_prefix}/local"
>> > >> >
>> > >>
>> > >> I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
>> > >> environment anywhere (which is admittedly strange .. but that's
>> > >> consistent with how it has been).
>> > >>
>> > >> Also, there's no other reports of this ever on the mailing list,
>> > >> including demos for conferences, etc. ... that fails the "how can this
>> > >> be working for everyone else ?" test.
>> > >>
>> > >> So there's definitely something different that I'm not seeing. I use
>> > >> OE nodistro or poky, others are using some other distros .. so I need
>> > >> to figure out what is the difference.
>> > >>
>> > >> That being said, even if we tweaked the binaries install, we don't
>> > >> want them changing where they've been installed previously, there
>> > >> could be any number of scripts expecting those locations in layers
>> > >> that I don't maintain.
>> > >>
>> > >> i.e. there's no way we should be patching the .service file, since
>> > >> that indicates the binaries have moved from where they were before.
>> > >>
>> > >> Bruce
>> > >>
>> > >
>> > > By default with BIN_PREFIX="" the binaries seem to be installed on /bin/k8s-init so they will work as this is in the PATH.
>> > > For OE nodistro maybe this QA is disabled (need to confirm that) but for distros that have this QA enabled it will fire up.
>> > >
>> >
>> > The installed but not shipped, isn't inhibited anywhere that I know
>> > of. Certainly not on poky, which is where most of the nightly builds
>> > happen for this.
>> >
>> > > About moving the binaries for another place is mainly because /bin is not the right place for them IMO.
>> > > https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/bin.html
>> >
>> > But that's not where they are going for all the rest of the builds :)
>> > I'm aware of the various filesystem standards.
>> >
>> > >
>> > > But I forgot to update all the services involved in this patch set so please drop it all and I will update all services as well in V2.
>> > >
>> >
>> > I wouldn't bother with that for v2, since again, they are using
>> > /usr/bin and /usr/local/bin from all the .service files I recall, and
>> > that's where we want the binaries to continue to be.
>> >
>>
>> aha. I see that k3s does set BIN_PREFIX, but it has been lost for
>> kubernetes and nerdctl.
>
>
> right, maybe you don't see the QA because it will only shon for files installed in /bin when usrmerge is enabled.

That could very well be it.

>
> for nerdctl, the BIN_PREFIX is there from the beginning so the recipe allway install it in /bin.
> for kubernetes, BIN_PREFIX is added in [1] and this only install k8s-init in /bin and all the other binaries
> continue where they were installed (in /usr/bin in this case).
>
> [1] - https://git.yoctoproject.org/meta-virtualization/commit/?id=4d0f0a5ca2338e5f6ed3fe3a18c602447cf60eb4
>
>>
>> The easiest / lowest footprint route is to just add the variable back
>> in for kubernetes and nerdctl, to the current default location. That
>> allows folks to override it and/or keep their scripts/service files
>> unmodified.
>
>
> This is what I did but using ${bindir} instead of ${BIN_PREFIX}.
> Adding the BIN_PREFIX ?= "${exec_prefix}/local" will change install location so
> since we have to change, we can use the ${bindir} and put the files in the right place.

I'm really only interested in the BIN_PREFIX being set in those
recipes. The smallest
footprint change.

So as the series currently stands, it isn't something I'll merge.

Cheers,

Bruce


>
> The last patch for k3s is more invasive as it touches many places and the systemd services as well.
> I will send it separate because this one don't fix anything and only change the installation of binaries
> from /usr/local/bin to /usr/bin
>
> Jose
>
>>
>> Bruce
>>
>> > Bruce
>> >
>> > > Jose
>> > >
>> > >>
>> > >>
>> > >> > Jose
>> > >> >
>> > >> >>
>> > >> >>
>> > >> >> Cheers,
>> > >> >>
>> > >> >> Bruce
>> > >> >>
>> > >> >> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> > >> >> >
>> > >> >> > - The env BIN_PREFIX is there from the beginning but there are no references to it,
>> > >> >> > also fix a fatal QA errors installed-vs-shipped.
>> > >> >> >
>> > >> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: QA Issue: kubernetes: Files/directories were installed but not shipped in any package:
>> > >> >> >   /bin
>> > >> >> >   /bin/k8s-init
>> > >> >> > Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
>> > >> >> > kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
>> > >> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: Fatal QA errors were found, failing task.
>> > >> >> >
>> > >> >> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
>> > >> >> > ---
>> > >> >> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
>> > >> >> >  1 file changed, 3 insertions(+), 3 deletions(-)
>> > >> >> >
>> > >> >> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
>> > >> >> > index e9460d4..82b75b1 100644
>> > >> >> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
>> > >> >> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
>> > >> >> > @@ -103,8 +103,8 @@ do_install() {
>> > >> >> >      install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
>> > >> >> >
>> > >> >> >      if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
>> > >> >> > -       install -d "${D}${BIN_PREFIX}/bin"
>> > >> >> > -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
>> > >> >> > +       install -d ${D}${bindir}
>> > >> >> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
>> > >> >> >
>> > >> >> >         install -d ${D}${sysconfdir}/sysctl.d
>> > >> >> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
>> > >> >> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
>> > >> >> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
>> > >> >> >
>> > >> >> >  ALLOW_EMPTY:${PN}-host = "1"
>> > >> >> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
>> > >> >> > +FILE:${PN}-host = "${bindir}/k8s-init"
>> > >> >> >  RDEPENDS:${PN}-host = "${PN}"
>> > >> >> >
>> > >> >> >  RRECOMMENDS:${PN} = "\
>> > >> >> > --
>> > >> >> > 2.37.1
>> > >> >> >
>> > >> >> >
>> > >> >> >
>> > >> >> >
>> > >> >>
>> > >> >>
>> > >> >> --
>> > >> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> > >> >> thee at its end
>> > >> >> - "Use the force Harry" - Gandalf, Star Trek II
>> > >> >
>> > >> >
>> > >> >
>> > >> > --
>> > >> > Best regards,
>> > >> >
>> > >> > José Quaresma
>> > >>
>> > >>
>> > >>
>> > >> --
>> > >> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> > >> thee at its end
>> > >> - "Use the force Harry" - Gandalf, Star Trek II
>> > >
>> > >
>> > >
>> > > --
>> > > Best regards,
>> > >
>> > > José Quaresma
>> >
>> >
>> >
>> > --
>> > - Thou shalt not follow the NULL pointer, for chaos and madness await
>> > thee at its end
>> > - "Use the force Harry" - Gandalf, Star Trek II
>> >
>> > -=-=-=-=-=-=-=-=-=-=-=-
>> > Links: You receive all messages sent to this group.
>> > View/Reply Online (#7504): https://lists.yoctoproject.org/g/meta-virtualization/message/7504
>> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
>> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
>> > Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub [bruce.ashfield@gmail.com]
>> > -=-=-=-=-=-=-=-=-=-=-=-
>> >
>>
>>
>> --
>> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> thee at its end
>> - "Use the force Harry" - Gandalf, Star Trek II
>
>
>
> --
> Best regards,
>
> José Quaresma



--
- Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end
- "Use the force Harry" - Gandalf, Star Trek II

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Jose Quaresma]

[-- Attachment #1: Type: text/plain, Size: 11152 bytes --]

Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022
à(s) 18:07:

> On Tue, Aug 9, 2022 at 12:36 PM Jose Quaresma <quaresma.jose@gmail.com>
> wrote:
> >
> >
> >
> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça,
> 9/08/2022 à(s) 16:05:
> >>
> >> On Tue, Aug 9, 2022 at 10:21 AM Bruce Ashfield via
> >> lists.yoctoproject.org
> >> <bruce.ashfield=gmail.com@lists.yoctoproject.org> wrote:
> >> >
> >> > On Tue, Aug 9, 2022 at 10:02 AM Jose Quaresma <
> quaresma.jose@gmail.com> wrote:
> >> > >
> >> > >
> >> > >
> >> > > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça,
> 9/08/2022 à(s) 14:43:
> >> > >>
> >> > >> On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <
> quaresma.jose@gmail.com> wrote:
> >> > >> >
> >> > >> > Hi Bruce,
> >> > >> >
> >> > >> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia
> terça, 9/08/2022 à(s) 14:19:
> >> > >> >>
> >> > >> >> You must have something different in your environment, I'm not
> seeing
> >> > >> >> any of these same issues.
> >> > >> >
> >> > >> >
> >> > >> > Maybe because you have the BIN_PREFIX defined somewhere in your
> distro or local.conf
> >> > >> > BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?=
> "${exec_prefix}/local"
> >> > >> >
> >> > >>
> >> > >> I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
> >> > >> environment anywhere (which is admittedly strange .. but that's
> >> > >> consistent with how it has been).
> >> > >>
> >> > >> Also, there's no other reports of this ever on the mailing list,
> >> > >> including demos for conferences, etc. ... that fails the "how can
> this
> >> > >> be working for everyone else ?" test.
> >> > >>
> >> > >> So there's definitely something different that I'm not seeing. I
> use
> >> > >> OE nodistro or poky, others are using some other distros .. so I
> need
> >> > >> to figure out what is the difference.
> >> > >>
> >> > >> That being said, even if we tweaked the binaries install, we don't
> >> > >> want them changing where they've been installed previously, there
> >> > >> could be any number of scripts expecting those locations in layers
> >> > >> that I don't maintain.
> >> > >>
> >> > >> i.e. there's no way we should be patching the .service file, since
> >> > >> that indicates the binaries have moved from where they were before.
> >> > >>
> >> > >> Bruce
> >> > >>
> >> > >
> >> > > By default with BIN_PREFIX="" the binaries seem to be installed on
> /bin/k8s-init so they will work as this is in the PATH.
> >> > > For OE nodistro maybe this QA is disabled (need to confirm that)
> but for distros that have this QA enabled it will fire up.
> >> > >
> >> >
> >> > The installed but not shipped, isn't inhibited anywhere that I know
> >> > of. Certainly not on poky, which is where most of the nightly builds
> >> > happen for this.
> >> >
> >> > > About moving the binaries for another place is mainly because /bin
> is not the right place for them IMO.
> >> > > https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/bin.html
> >> >
> >> > But that's not where they are going for all the rest of the builds :)
> >> > I'm aware of the various filesystem standards.
> >> >
> >> > >
> >> > > But I forgot to update all the services involved in this patch set
> so please drop it all and I will update all services as well in V2.
> >> > >
> >> >
> >> > I wouldn't bother with that for v2, since again, they are using
> >> > /usr/bin and /usr/local/bin from all the .service files I recall, and
> >> > that's where we want the binaries to continue to be.
> >> >
> >>
> >> aha. I see that k3s does set BIN_PREFIX, but it has been lost for
> >> kubernetes and nerdctl.
> >
> >
> > right, maybe you don't see the QA because it will only shon for files
> installed in /bin when usrmerge is enabled.
>
> That could very well be it.
>
> >
> > for nerdctl, the BIN_PREFIX is there from the beginning so the recipe
> allway install it in /bin.
> > for kubernetes, BIN_PREFIX is added in [1] and this only install
> k8s-init in /bin and all the other binaries
> > continue where they were installed (in /usr/bin in this case).
> >
> > [1] -
> https://git.yoctoproject.org/meta-virtualization/commit/?id=4d0f0a5ca2338e5f6ed3fe3a18c602447cf60eb4
> >
> >>
> >> The easiest / lowest footprint route is to just add the variable back
> >> in for kubernetes and nerdctl, to the current default location. That
> >> allows folks to override it and/or keep their scripts/service files
> >> unmodified.
> >
> >
> > This is what I did but using ${bindir} instead of ${BIN_PREFIX}.
> > Adding the BIN_PREFIX ?= "${exec_prefix}/local" will change install
> location so
> > since we have to change, we can use the ${bindir} and put the files in
> the right place.
>
> I'm really only interested in the BIN_PREFIX being set in those
> recipes. The smallest
> footprint change.
>

kubernetes is installing all binaries files to ${D}/${bindir}:
install -m 755 -D
${S}/src/import/_output/local/bin/${TARGET_GOOS}/${TARGET_GOARCH}/*
${D}/${bindir}

and you are suggesting to install only k8s-init with BIN_PREFIX:
install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"

this will end up with BIN_PREFIX = "/usr" to be consistent with the other
binaries in the recipe
or BIN_PREFIX = "${exec_prefix}/local" to use something not defined in
bitbake.conf.
I don't really know what you prefer BIN_PREFIX = "/usr" or BIN_PREFIX =
"${exec_prefix}/local"
as the two will change the installation path and you said that:
"even if we tweaked the binaries install, we don't want them changing where
they've been installed previously"

sorry but changing this patch and using BIN_PREFIX doesn't make much sense
in my opinion.

Jose


>
> So as the series currently stands, it isn't something I'll merge.
>
> Cheers,
>
> Bruce
>
>
> >
> > The last patch for k3s is more invasive as it touches many places and
> the systemd services as well.
> > I will send it separate because this one don't fix anything and only
> change the installation of binaries
> > from /usr/local/bin to /usr/bin
> >
> > Jose
> >
> >>
> >> Bruce
> >>
> >> > Bruce
> >> >
> >> > > Jose
> >> > >
> >> > >>
> >> > >>
> >> > >> > Jose
> >> > >> >
> >> > >> >>
> >> > >> >>
> >> > >> >> Cheers,
> >> > >> >>
> >> > >> >> Bruce
> >> > >> >>
> >> > >> >> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <
> quaresma.jose@gmail.com> wrote:
> >> > >> >> >
> >> > >> >> > - The env BIN_PREFIX is there from the beginning but there
> are no references to it,
> >> > >> >> > also fix a fatal QA errors installed-vs-shipped.
> >> > >> >> >
> >> > >> >> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: QA Issue: kubernetes: Files/directories were installed but not
> shipped in any package:
> >> > >> >> >   /bin
> >> > >> >> >   /bin/k8s-init
> >> > >> >> > Please set FILES such that these items are packaged.
> Alternatively if they are unneeded, avoid installing them or delete them
> within do_install.
> >> > >> >> > kubernetes: 2 installed and not shipped files.
> [installed-vs-shipped]
> >> > >> >> > ERROR:
> kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0
> do_package: Fatal QA errors were found, failing task.
> >> > >> >> >
> >> > >> >> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
> >> > >> >> > ---
> >> > >> >> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
> >> > >> >> >  1 file changed, 3 insertions(+), 3 deletions(-)
> >> > >> >> >
> >> > >> >> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb
> b/recipes-containers/kubernetes/kubernetes_git.bb
> >> > >> >> > index e9460d4..82b75b1 100644
> >> > >> >> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
> >> > >> >> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
> >> > >> >> > @@ -103,8 +103,8 @@ do_install() {
> >> > >> >> >      install -m 0644
> ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf
> ${D}${systemd_unitdir}/system/kubelet.service.d/
> >> > >> >> >
> >> > >> >> >      if
> ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
> >> > >> >> > -       install -d "${D}${BIN_PREFIX}/bin"
> >> > >> >> > -       install -m 755 "${WORKDIR}/k8s-init"
> "${D}${BIN_PREFIX}/bin"
> >> > >> >> > +       install -d ${D}${bindir}
> >> > >> >> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
> >> > >> >> >
> >> > >> >> >         install -d ${D}${sysconfdir}/sysctl.d
> >> > >> >> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf"
> "${D}${sysconfdir}/sysctl.d"
> >> > >> >> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
> >> > >> >> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
> >> > >> >> >
> >> > >> >> >  ALLOW_EMPTY:${PN}-host = "1"
> >> > >> >> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
> >> > >> >> > +FILE:${PN}-host = "${bindir}/k8s-init"
> >> > >> >> >  RDEPENDS:${PN}-host = "${PN}"
> >> > >> >> >
> >> > >> >> >  RRECOMMENDS:${PN} = "\
> >> > >> >> > --
> >> > >> >> > 2.37.1
> >> > >> >> >
> >> > >> >> >
> >> > >> >> >
> >> > >> >> >
> >> > >> >>
> >> > >> >>
> >> > >> >> --
> >> > >> >> - Thou shalt not follow the NULL pointer, for chaos and madness
> await
> >> > >> >> thee at its end
> >> > >> >> - "Use the force Harry" - Gandalf, Star Trek II
> >> > >> >
> >> > >> >
> >> > >> >
> >> > >> > --
> >> > >> > Best regards,
> >> > >> >
> >> > >> > José Quaresma
> >> > >>
> >> > >>
> >> > >>
> >> > >> --
> >> > >> - Thou shalt not follow the NULL pointer, for chaos and madness
> await
> >> > >> thee at its end
> >> > >> - "Use the force Harry" - Gandalf, Star Trek II
> >> > >
> >> > >
> >> > >
> >> > > --
> >> > > Best regards,
> >> > >
> >> > > José Quaresma
> >> >
> >> >
> >> >
> >> > --
> >> > - Thou shalt not follow the NULL pointer, for chaos and madness await
> >> > thee at its end
> >> > - "Use the force Harry" - Gandalf, Star Trek II
> >> >
> >> > -=-=-=-=-=-=-=-=-=-=-=-
> >> > Links: You receive all messages sent to this group.
> >> > View/Reply Online (#7504):
> https://lists.yoctoproject.org/g/meta-virtualization/message/7504
> >> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
> >> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
> >> > Unsubscribe:
> https://lists.yoctoproject.org/g/meta-virtualization/unsub [
> bruce.ashfield@gmail.com]
> >> > -=-=-=-=-=-=-=-=-=-=-=-
> >> >
> >>
> >>
> >> --
> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
> >> thee at its end
> >> - "Use the force Harry" - Gandalf, Star Trek II
> >
> >
> >
> > --
> > Best regards,
> >
> > José Quaresma
>
>
>
> --
> - Thou shalt not follow the NULL pointer, for chaos and madness await
> thee at its end
> - "Use the force Harry" - Gandalf, Star Trek II
>

>

-- 
Best regards,

José Quaresma

[-- Attachment #2: Type: text/html, Size: 17598 bytes --]

Re: [meta-virtualization][kirkstone][master][PATCH 2/4] kubernetes: install the binaries in OE standard places

[Bruce Ashfield]

On Tue, Aug 9, 2022 at 1:36 PM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>
>
>
> Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 18:07:
>>
>> On Tue, Aug 9, 2022 at 12:36 PM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> >
>> >
>> >
>> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 16:05:
>> >>
>> >> On Tue, Aug 9, 2022 at 10:21 AM Bruce Ashfield via
>> >> lists.yoctoproject.org
>> >> <bruce.ashfield=gmail.com@lists.yoctoproject.org> wrote:
>> >> >
>> >> > On Tue, Aug 9, 2022 at 10:02 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> >> > >
>> >> > >
>> >> > >
>> >> > > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:43:
>> >> > >>
>> >> > >> On Tue, Aug 9, 2022 at 9:34 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> >> > >> >
>> >> > >> > Hi Bruce,
>> >> > >> >
>> >> > >> > Bruce Ashfield <bruce.ashfield@gmail.com> escreveu no dia terça, 9/08/2022 à(s) 14:19:
>> >> > >> >>
>> >> > >> >> You must have something different in your environment, I'm not seeing
>> >> > >> >> any of these same issues.
>> >> > >> >
>> >> > >> >
>> >> > >> > Maybe because you have the BIN_PREFIX defined somewhere in your distro or local.conf
>> >> > >> > BIN_PREFIX is defined only in k3s recipe BIN_PREFIX ?= "${exec_prefix}/local"
>> >> > >> >
>> >> > >>
>> >> > >> I'm building a plain OE + meta-virt. BIN_PREFIX isn't in the
>> >> > >> environment anywhere (which is admittedly strange .. but that's
>> >> > >> consistent with how it has been).
>> >> > >>
>> >> > >> Also, there's no other reports of this ever on the mailing list,
>> >> > >> including demos for conferences, etc. ... that fails the "how can this
>> >> > >> be working for everyone else ?" test.
>> >> > >>
>> >> > >> So there's definitely something different that I'm not seeing. I use
>> >> > >> OE nodistro or poky, others are using some other distros .. so I need
>> >> > >> to figure out what is the difference.
>> >> > >>
>> >> > >> That being said, even if we tweaked the binaries install, we don't
>> >> > >> want them changing where they've been installed previously, there
>> >> > >> could be any number of scripts expecting those locations in layers
>> >> > >> that I don't maintain.
>> >> > >>
>> >> > >> i.e. there's no way we should be patching the .service file, since
>> >> > >> that indicates the binaries have moved from where they were before.
>> >> > >>
>> >> > >> Bruce
>> >> > >>
>> >> > >
>> >> > > By default with BIN_PREFIX="" the binaries seem to be installed on /bin/k8s-init so they will work as this is in the PATH.
>> >> > > For OE nodistro maybe this QA is disabled (need to confirm that) but for distros that have this QA enabled it will fire up.
>> >> > >
>> >> >
>> >> > The installed but not shipped, isn't inhibited anywhere that I know
>> >> > of. Certainly not on poky, which is where most of the nightly builds
>> >> > happen for this.
>> >> >
>> >> > > About moving the binaries for another place is mainly because /bin is not the right place for them IMO.
>> >> > > https://tldp.org/LDP/Linux-Filesystem-Hierarchy/html/bin.html
>> >> >
>> >> > But that's not where they are going for all the rest of the builds :)
>> >> > I'm aware of the various filesystem standards.
>> >> >
>> >> > >
>> >> > > But I forgot to update all the services involved in this patch set so please drop it all and I will update all services as well in V2.
>> >> > >
>> >> >
>> >> > I wouldn't bother with that for v2, since again, they are using
>> >> > /usr/bin and /usr/local/bin from all the .service files I recall, and
>> >> > that's where we want the binaries to continue to be.
>> >> >
>> >>
>> >> aha. I see that k3s does set BIN_PREFIX, but it has been lost for
>> >> kubernetes and nerdctl.
>> >
>> >
>> > right, maybe you don't see the QA because it will only shon for files installed in /bin when usrmerge is enabled.
>>
>> That could very well be it.
>>
>> >
>> > for nerdctl, the BIN_PREFIX is there from the beginning so the recipe allway install it in /bin.
>> > for kubernetes, BIN_PREFIX is added in [1] and this only install k8s-init in /bin and all the other binaries
>> > continue where they were installed (in /usr/bin in this case).
>> >
>> > [1] - https://git.yoctoproject.org/meta-virtualization/commit/?id=4d0f0a5ca2338e5f6ed3fe3a18c602447cf60eb4
>> >
>> >>
>> >> The easiest / lowest footprint route is to just add the variable back
>> >> in for kubernetes and nerdctl, to the current default location. That
>> >> allows folks to override it and/or keep their scripts/service files
>> >> unmodified.
>> >
>> >
>> > This is what I did but using ${bindir} instead of ${BIN_PREFIX}.
>> > Adding the BIN_PREFIX ?= "${exec_prefix}/local" will change install location so
>> > since we have to change, we can use the ${bindir} and put the files in the right place.
>>
>> I'm really only interested in the BIN_PREFIX being set in those
>> recipes. The smallest
>> footprint change.
>
>
> kubernetes is installing all binaries files to ${D}/${bindir}:
> install -m 755 -D ${S}/src/import/_output/local/bin/${TARGET_GOOS}/${TARGET_GOARCH}/* ${D}/${bindir}

I realize that.

>
> and you are suggesting to install only k8s-init with BIN_PREFIX:
> install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"

No, I'm suggesting the smallest footprint change, but that is more for
the other recipes, versus the single use of it in this kubernetes
recipe.

>
> this will end up with BIN_PREFIX = "/usr" to be consistent with the other binaries in the recipe
> or BIN_PREFIX = "${exec_prefix}/local" to use something not defined in bitbake.conf.
> I don't really know what you prefer BIN_PREFIX = "/usr" or BIN_PREFIX = "${exec_prefix}/local"
> as the two will change the installation path and you said that:
> "even if we tweaked the binaries install, we don't want them changing where they've been installed previously"

That comment was more for the k3s recipe, this helper is being
installed to the wrong place, and likely I'm the only one that uses
it.

For this specific recipe, I'd make it consistent with the other
binaries and set it to ${prefix} which is "/usr", which of course is
the same as ${exec_prefix}

>
> sorry but changing this patch and using BIN_PREFIX doesn't make much sense in my opinion.
>

That's the style of the recipes as they stand, keeping them consistent
to that has value .. in particular, since I end up spending the
majority of time debugging system level runtime issues. There were
many issues found during the creation of the recipes and stack with
binaries being assumed in one location and then someone changing their
bitbake configuration and them not being present. So the control was
pulled into the recipes under BIN_PREFIX to make it explicit and to
save many hours debugging. That's the same reason why many of the go
build settings are in the recipes when for the most part they could be
used from the bbclasses.

That being said, we could change k8s and nerdctl to just use
${bindir}, but k3s should be left as-is.

Bruce

> Jose
>
>>
>>
>> So as the series currently stands, it isn't something I'll merge.
>>
>> Cheers,
>>
>> Bruce
>>
>>
>> >
>> > The last patch for k3s is more invasive as it touches many places and the systemd services as well.
>> > I will send it separate because this one don't fix anything and only change the installation of binaries
>> > from /usr/local/bin to /usr/bin
>> >
>> > Jose
>> >
>> >>
>> >> Bruce
>> >>
>> >> > Bruce
>> >> >
>> >> > > Jose
>> >> > >
>> >> > >>
>> >> > >>
>> >> > >> > Jose
>> >> > >> >
>> >> > >> >>
>> >> > >> >>
>> >> > >> >> Cheers,
>> >> > >> >>
>> >> > >> >> Bruce
>> >> > >> >>
>> >> > >> >> On Tue, Aug 9, 2022 at 8:16 AM Jose Quaresma <quaresma.jose@gmail.com> wrote:
>> >> > >> >> >
>> >> > >> >> > - The env BIN_PREFIX is there from the beginning but there are no references to it,
>> >> > >> >> > also fix a fatal QA errors installed-vs-shipped.
>> >> > >> >> >
>> >> > >> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: QA Issue: kubernetes: Files/directories were installed but not shipped in any package:
>> >> > >> >> >   /bin
>> >> > >> >> >   /bin/k8s-init
>> >> > >> >> > Please set FILES such that these items are packaged. Alternatively if they are unneeded, avoid installing them or delete them within do_install.
>> >> > >> >> > kubernetes: 2 installed and not shipped files. [installed-vs-shipped]
>> >> > >> >> > ERROR: kubernetes-1_v1.23.6+gitfbcfa33018159c033aee77b0d5456df6771aa9b5-r0 do_package: Fatal QA errors were found, failing task.
>> >> > >> >> >
>> >> > >> >> > Signed-off-by: Jose Quaresma <jose.quaresma@foundries.io>
>> >> > >> >> > ---
>> >> > >> >> >  recipes-containers/kubernetes/kubernetes_git.bb | 6 +++---
>> >> > >> >> >  1 file changed, 3 insertions(+), 3 deletions(-)
>> >> > >> >> >
>> >> > >> >> > diff --git a/recipes-containers/kubernetes/kubernetes_git.bb b/recipes-containers/kubernetes/kubernetes_git.bb
>> >> > >> >> > index e9460d4..82b75b1 100644
>> >> > >> >> > --- a/recipes-containers/kubernetes/kubernetes_git.bb
>> >> > >> >> > +++ b/recipes-containers/kubernetes/kubernetes_git.bb
>> >> > >> >> > @@ -103,8 +103,8 @@ do_install() {
>> >> > >> >> >      install -m 0644 ${WORKDIR}/git/release/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf  ${D}${systemd_unitdir}/system/kubelet.service.d/
>> >> > >> >> >
>> >> > >> >> >      if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
>> >> > >> >> > -       install -d "${D}${BIN_PREFIX}/bin"
>> >> > >> >> > -       install -m 755 "${WORKDIR}/k8s-init" "${D}${BIN_PREFIX}/bin"
>> >> > >> >> > +       install -d ${D}${bindir}
>> >> > >> >> > +       install -m 755 ${WORKDIR}/k8s-init ${D}${bindir}
>> >> > >> >> >
>> >> > >> >> >         install -d ${D}${sysconfdir}/sysctl.d
>> >> > >> >> >         install -m 0644 "${WORKDIR}/99-kubernetes.conf" "${D}${sysconfdir}/sysctl.d"
>> >> > >> >> > @@ -141,7 +141,7 @@ FILES:kube-proxy = "${bindir}/kube-proxy"
>> >> > >> >> >  FILES:${PN}-misc = "${bindir} ${sysconfdir}/sysctl.d"
>> >> > >> >> >
>> >> > >> >> >  ALLOW_EMPTY:${PN}-host = "1"
>> >> > >> >> > -FILE:${PN}-host = "${BIN_PREFIX}/bin/k8s-init"
>> >> > >> >> > +FILE:${PN}-host = "${bindir}/k8s-init"
>> >> > >> >> >  RDEPENDS:${PN}-host = "${PN}"
>> >> > >> >> >
>> >> > >> >> >  RRECOMMENDS:${PN} = "\
>> >> > >> >> > --
>> >> > >> >> > 2.37.1
>> >> > >> >> >
>> >> > >> >> >
>> >> > >> >> >
>> >> > >> >> >
>> >> > >> >>
>> >> > >> >>
>> >> > >> >> --
>> >> > >> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> >> > >> >> thee at its end
>> >> > >> >> - "Use the force Harry" - Gandalf, Star Trek II
>> >> > >> >
>> >> > >> >
>> >> > >> >
>> >> > >> > --
>> >> > >> > Best regards,
>> >> > >> >
>> >> > >> > José Quaresma
>> >> > >>
>> >> > >>
>> >> > >>
>> >> > >> --
>> >> > >> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> >> > >> thee at its end
>> >> > >> - "Use the force Harry" - Gandalf, Star Trek II
>> >> > >
>> >> > >
>> >> > >
>> >> > > --
>> >> > > Best regards,
>> >> > >
>> >> > > José Quaresma
>> >> >
>> >> >
>> >> >
>> >> > --
>> >> > - Thou shalt not follow the NULL pointer, for chaos and madness await
>> >> > thee at its end
>> >> > - "Use the force Harry" - Gandalf, Star Trek II
>> >> >
>> >> > -=-=-=-=-=-=-=-=-=-=-=-
>> >> > Links: You receive all messages sent to this group.
>> >> > View/Reply Online (#7504): https://lists.yoctoproject.org/g/meta-virtualization/message/7504
>> >> > Mute This Topic: https://lists.yoctoproject.org/mt/92913426/1050810
>> >> > Group Owner: meta-virtualization+owner@lists.yoctoproject.org
>> >> > Unsubscribe: https://lists.yoctoproject.org/g/meta-virtualization/unsub [bruce.ashfield@gmail.com]
>> >> > -=-=-=-=-=-=-=-=-=-=-=-
>> >> >
>> >>
>> >>
>> >> --
>> >> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> >> thee at its end
>> >> - "Use the force Harry" - Gandalf, Star Trek II
>> >
>> >
>> >
>> > --
>> > Best regards,
>> >
>> > José Quaresma
>>
>>
>>
>> --
>> - Thou shalt not follow the NULL pointer, for chaos and madness await
>> thee at its end
>> - "Use the force Harry" - Gandalf, Star Trek II
>>
>>
>
>
> --
> Best regards,
>
> José Quaresma



--
- Thou shalt not follow the NULL pointer, for chaos and madness await
thee at its end
- "Use the force Harry" - Gandalf, Star Trek II