[Buildroot] [PATCH v2 1/1] package/pkg-python: migrate flit to new bootstrapping sequence

[Buildroot] [PATCH v2 1/1] package/pkg-python: migrate flit to new bootstrapping sequence

[James Hilliard]

There are a number of flit toolchain dependencies currently in the
process of deprecating distutils based fallbacks.

This will be needed in order to update tomli.

We need to migrate these to use a new bootstrap based build+install
sequence which relies on flit's bootstrap wheel build+install
features to build and install host-python-pypa-build and
host-python-installer which gives us a full pep517 toolchain.

Note that one can run host-python-flit-core commands for building
and installing itself since the package build directory is the cwd.

For host-python-installer to do this we need to add the package src
directory to PYTHONPATH as we need to run it from the source directory
for installing itself. This is due to a directory structure that does
not let us run directly from the top level.

Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
Changes v1 -> v2:
  - formatting/cleanup
  - add comments
---
 package/pkg-python.mk                        | 45 ++++++++++++++++++--
 package/python-flit-core/python-flit-core.mk |  2 +-
 package/python-installer/python-installer.mk |  3 +-
 package/python-pep517/python-pep517.mk       |  2 +-
 package/python-tomli/python-tomli.mk         |  2 +-
 5 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/package/pkg-python.mk b/package/pkg-python.mk
index 52ce402281..f08d8ce04c 100644
--- a/package/pkg-python.mk
+++ b/package/pkg-python.mk
@@ -154,6 +154,9 @@ HOST_PKG_PYTHON_PEP517_INSTALL_OPTS = \
 	--scripts=$(HOST_DIR)/usr/bin \
 	--data=$(HOST_DIR)/usr
 
+HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \
+	--installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages
+
 ################################################################################
 # inner-python-package -- defines how the configuration, compilation
 # and installation of a Python package should be done, implements a
@@ -211,8 +214,25 @@ $(2)_BASE_INSTALL_TARGET_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $
 $(2)_BASE_INSTALL_STAGING_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(PKG_PYTHON_PEP517_INSTALL_STAGING_OPTS)
 else
 $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV)
-$(2)_BASE_BUILD_CMD = -m build -n -w
-$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS)
+# Use flit built in wheel builder for packages that are host-python-pypa-build dependencies.
+# This is needed to avoid a circular with host-python-pypa-build and those dependencies.
+#
+$(2)_BASE_BUILD_CMD = $$(if \
+		$$(filter \
+			host-python-flit-core host-python-installer host-python-pep517 host-python-tomli, \
+			$(1)) \
+		, \
+		-m flit_core.wheel, \
+		-m build -n -w \
+	)
+# Use flit built in bootstrap_install for installing host-python-flit-core.
+# This is due to host-python-installer depending on host-python-flit-core.
+#
+$(2)_BASE_INSTALL_CMD = $$(if \
+		$$(filter host-python-flit-core,$(1)), \
+		-m bootstrap_install dist/* $$(HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS), \
+		$(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS) \
+	)
 endif
 else
 $$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'setuptools', 'pep517' or 'flit'.")
@@ -235,9 +255,26 @@ endif # ($(4),target)
 ifeq ($$($(2)_SETUP_TYPE),setuptools)
 $(2)_DEPENDENCIES += $$(if $$(filter host-python-setuptools,$(1)),,host-python-setuptools)
 else ifneq ($$(filter flit pep517,$$($(2)_SETUP_TYPE)),)
-$(2)_DEPENDENCIES += host-python-pypa-build host-python-installer
+# Filter out host-python-pypa-build and any of its dependencies that need a flit based build.
+#
+$(2)_DEPENDENCIES += $$(if \
+		$$(filter \
+			host-python-flit-core host-python-installer host-python-pep517 host-python-pypa-build host-python-tomli, \
+			$(1)) \
+		,, \
+		host-python-pypa-build \
+	)
+# Filter out host-python-installer and any of its dependencies that need a flit based build.
+#
+$(2)_DEPENDENCIES += $$(if \
+		$$(filter \
+			host-python-flit-core host-python-installer, \
+			$(1)) \
+		,, \
+		host-python-installer \
+	)
 ifeq ($$($(2)_SETUP_TYPE),flit)
-$(2)_DEPENDENCIES += host-python-flit-core
+$(2)_DEPENDENCIES += $$(if $$(filter host-python-flit-core,$(1)),,host-python-flit-core)
 endif
 endif # SETUP_TYPE
 
diff --git a/package/python-flit-core/python-flit-core.mk b/package/python-flit-core/python-flit-core.mk
index 0e058a1f17..d206a72f82 100644
--- a/package/python-flit-core/python-flit-core.mk
+++ b/package/python-flit-core/python-flit-core.mk
@@ -8,6 +8,6 @@ PYTHON_FLIT_CORE_VERSION = 3.7.1
 PYTHON_FLIT_CORE_SOURCE = flit_core-$(PYTHON_FLIT_CORE_VERSION).tar.gz
 PYTHON_FLIT_CORE_SITE = https://files.pythonhosted.org/packages/15/d1/d8798b83e953fd6f86ca9b50f93eec464a9305b0661469c8234e61095481
 PYTHON_FLIT_CORE_LICENSE = BSD-3-Clause
-PYTHON_FLIT_CORE_SETUP_TYPE = pep517
+PYTHON_FLIT_CORE_SETUP_TYPE = flit
 
 $(eval $(host-python-package))
diff --git a/package/python-installer/python-installer.mk b/package/python-installer/python-installer.mk
index 862a251415..97a158b738 100644
--- a/package/python-installer/python-installer.mk
+++ b/package/python-installer/python-installer.mk
@@ -9,6 +9,7 @@ PYTHON_INSTALLER_SOURCE = installer-$(PYTHON_INSTALLER_VERSION).tar.gz
 PYTHON_INSTALLER_SITE = https://files.pythonhosted.org/packages/74/b7/9187323cd732840f1cddd6a9f05961406636b50c799eef37c920b63110c0
 PYTHON_INSTALLER_LICENSE = MIT
 PYTHON_INSTALLER_LICENSE_FILES = LICENSE
-PYTHON_INSTALLER_SETUP_TYPE = distutils
+PYTHON_INSTALLER_SETUP_TYPE = flit
+HOST_PYTHON_INSTALLER_ENV = PYTHONPATH=$(@D)/src
 
 $(eval $(host-python-package))
diff --git a/package/python-pep517/python-pep517.mk b/package/python-pep517/python-pep517.mk
index 99aa62d51d..1ca1bc4e35 100644
--- a/package/python-pep517/python-pep517.mk
+++ b/package/python-pep517/python-pep517.mk
@@ -9,7 +9,7 @@ PYTHON_PEP517_SOURCE = pep517-$(PYTHON_PEP517_VERSION).tar.gz
 PYTHON_PEP517_SITE = https://files.pythonhosted.org/packages/0a/65/6e656d49c679136edfba25f25791f45ffe1ea4ae2ec1c59fe9c35e061cd1
 PYTHON_PEP517_LICENSE = MIT
 PYTHON_PEP517_LICENSE_FILES = LICENSE
-PYTHON_PEP517_SETUP_TYPE = distutils
+PYTHON_PEP517_SETUP_TYPE = flit
 HOST_PYTHON_PEP517_DEPENDENCIES = host-python-tomli
 
 $(eval $(host-python-package))
diff --git a/package/python-tomli/python-tomli.mk b/package/python-tomli/python-tomli.mk
index b8c20ca736..3539a505be 100644
--- a/package/python-tomli/python-tomli.mk
+++ b/package/python-tomli/python-tomli.mk
@@ -7,7 +7,7 @@
 PYTHON_TOMLI_VERSION = 1.2.0
 PYTHON_TOMLI_SOURCE = tomli-$(PYTHON_TOMLI_VERSION).tar.gz
 PYTHON_TOMLI_SITE = https://files.pythonhosted.org/packages/ec/38/8eccdc662c61aed187d5f5b168c18b1d2de3827976c3691e4da8be7375aa
-PYTHON_TOMLI_SETUP_TYPE = distutils
+PYTHON_TOMLI_SETUP_TYPE = flit
 PYTHON_TOMLI_LICENSE = MIT
 PYTHON_TOMLI_LICENSE_FILES = LICENSE
 
-- 
2.25.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH v2 1/1] package/pkg-python: migrate flit to new bootstrapping sequence

[Arnout Vandecappelle]

On 13/03/2022 18:56, James Hilliard wrote:
> There are a number of flit toolchain dependencies currently in the
> process of deprecating distutils based fallbacks.
> 
> This will be needed in order to update tomli.
> 
> We need to migrate these to use a new bootstrap based build+install
> sequence which relies on flit's bootstrap wheel build+install
> features to build and install host-python-pypa-build and
> host-python-installer which gives us a full pep517 toolchain.
> 
> Note that one can run host-python-flit-core commands for building
> and installing itself since the package build directory is the cwd.
> 
> For host-python-installer to do this we need to add the package src
> directory to PYTHONPATH as we need to run it from the source directory
> for installing itself. This is due to a directory structure that does
> not let us run directly from the top level.
> 
> Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
> ---
> Changes v1 -> v2:
>    - formatting/cleanup
>    - add comments
> ---
>   package/pkg-python.mk                        | 45 ++++++++++++++++++--
>   package/python-flit-core/python-flit-core.mk |  2 +-
>   package/python-installer/python-installer.mk |  3 +-
>   package/python-pep517/python-pep517.mk       |  2 +-
>   package/python-tomli/python-tomli.mk         |  2 +-
>   5 files changed, 46 insertions(+), 8 deletions(-)
> 
> diff --git a/package/pkg-python.mk b/package/pkg-python.mk
> index 52ce402281..f08d8ce04c 100644
> --- a/package/pkg-python.mk
> +++ b/package/pkg-python.mk
> @@ -154,6 +154,9 @@ HOST_PKG_PYTHON_PEP517_INSTALL_OPTS = \
>   	--scripts=$(HOST_DIR)/usr/bin \
>   	--data=$(HOST_DIR)/usr
>   
> +HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS = \
> +	--installdir=$(HOST_DIR)/lib/python$(PYTHON3_VERSION_MAJOR)/site-packages
> +
>   ################################################################################
>   # inner-python-package -- defines how the configuration, compilation
>   # and installation of a Python package should be done, implements a
> @@ -211,8 +214,25 @@ $(2)_BASE_INSTALL_TARGET_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $
>   $(2)_BASE_INSTALL_STAGING_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(PKG_PYTHON_PEP517_INSTALL_STAGING_OPTS)
>   else
>   $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV)
> -$(2)_BASE_BUILD_CMD = -m build -n -w
> -$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS)
> +# Use flit built in wheel builder for packages that are host-python-pypa-build dependencies.
> +# This is needed to avoid a circular with host-python-pypa-build and those dependencies.
> +#
> +$(2)_BASE_BUILD_CMD = $$(if \
> +		$$(filter \
> +			host-python-flit-core host-python-installer host-python-pep517 host-python-tomli, \
> +			$(1)) \
> +		, \
> +		-m flit_core.wheel, \
> +		-m build -n -w \

  This is adding way too much complexity to the infra just to support these four 
packages. I think there a a couple of better options:

- don't use python infra at all for these packages; or
- set HOST_PYTHON_FLIT_CORE_BUILD_CMDS etc. in python-flit-core.mk; or
- add a fifth SETUP_TYPE "generic" or "custom" or something, which requires 
$(2)_BASE_* to be set by the python-flit-core.mk;
- in the infra change it to $(2)_BASE_BUILD_CMD ?=, set 
HOST_PYTHON_FLIT_CORE_BASE_BUILD_CMD in python-flit-core.mk; or
- add a fifth SETUP_TYPE "flit-bootstrap" or something like that which only gets 
used by the problematic packages.


  The latter two options have my preference because I think they are the 
cleanest, but any of them is OK for me. The "generic" setup type and the 
overridable _BASE_BUILD_CMD would also require an update in the manual.


> +	)
> +# Use flit built in bootstrap_install for installing host-python-flit-core.
> +# This is due to host-python-installer depending on host-python-flit-core.
> +#
> +$(2)_BASE_INSTALL_CMD = $$(if \
> +		$$(filter host-python-flit-core,$(1)), \
> +		-m bootstrap_install dist/* $$(HOST_PKG_PYTHON_PEP517_BOOTSTRAP_INSTALL_OPTS), \
> +		$(TOPDIR)/support/scripts/pyinstaller.py dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS) \
> +	)
>   endif
>   else
>   $$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'setuptools', 'pep517' or 'flit'.")
> @@ -235,9 +255,26 @@ endif # ($(4),target)
>   ifeq ($$($(2)_SETUP_TYPE),setuptools)
>   $(2)_DEPENDENCIES += $$(if $$(filter host-python-setuptools,$(1)),,host-python-setuptools)
>   else ifneq ($$(filter flit pep517,$$($(2)_SETUP_TYPE)),)
> -$(2)_DEPENDENCIES += host-python-pypa-build host-python-installer
> +# Filter out host-python-pypa-build and any of its dependencies that need a flit based build.
> +#
> +$(2)_DEPENDENCIES += $$(if \
> +		$$(filter \
> +			host-python-flit-core host-python-installer host-python-pep517 host-python-pypa-build host-python-tomli, \
> +			$(1)) \
> +		,, \
> +		host-python-pypa-build \

  With the "generic" or "flit-bootstrap" setup types, this would no longer be 
needed. For the other solutions, we will indeed still need this ugly thing.

> +	)
> +# Filter out host-python-installer and any of its dependencies that need a flit based build.
> +#
> +$(2)_DEPENDENCIES += $$(if \
> +		$$(filter \
> +			host-python-flit-core host-python-installer, \
> +			$(1)) \
> +		,, \
> +		host-python-installer \
> +	)
>   ifeq ($$($(2)_SETUP_TYPE),flit)
> -$(2)_DEPENDENCIES += host-python-flit-core
> +$(2)_DEPENDENCIES += $$(if $$(filter host-python-flit-core,$(1)),,host-python-flit-core)
>   endif
>   endif # SETUP_TYPE
>   
> diff --git a/package/python-flit-core/python-flit-core.mk b/package/python-flit-core/python-flit-core.mk
> index 0e058a1f17..d206a72f82 100644
> --- a/package/python-flit-core/python-flit-core.mk
> +++ b/package/python-flit-core/python-flit-core.mk
> @@ -8,6 +8,6 @@ PYTHON_FLIT_CORE_VERSION = 3.7.1
>   PYTHON_FLIT_CORE_SOURCE = flit_core-$(PYTHON_FLIT_CORE_VERSION).tar.gz
>   PYTHON_FLIT_CORE_SITE = https://files.pythonhosted.org/packages/15/d1/d8798b83e953fd6f86ca9b50f93eec464a9305b0661469c8234e61095481
>   PYTHON_FLIT_CORE_LICENSE = BSD-3-Clause
> -PYTHON_FLIT_CORE_SETUP_TYPE = pep517
> +PYTHON_FLIT_CORE_SETUP_TYPE = flit

  This means there's no package any more that uses pep517 setup type. Do you 
think it's ever going to be used again? Or can we remove it altogether?

  (Removing it is not urgent at all even if it's unused. But if it's really not 
likely to be used again, I'd prefer to get rid of it by 2022.05.)

>   
>   $(eval $(host-python-package))
> diff --git a/package/python-installer/python-installer.mk b/package/python-installer/python-installer.mk
> index 862a251415..97a158b738 100644
> --- a/package/python-installer/python-installer.mk
> +++ b/package/python-installer/python-installer.mk
> @@ -9,6 +9,7 @@ PYTHON_INSTALLER_SOURCE = installer-$(PYTHON_INSTALLER_VERSION).tar.gz
>   PYTHON_INSTALLER_SITE = https://files.pythonhosted.org/packages/74/b7/9187323cd732840f1cddd6a9f05961406636b50c799eef37c920b63110c0
>   PYTHON_INSTALLER_LICENSE = MIT
>   PYTHON_INSTALLER_LICENSE_FILES = LICENSE
> -PYTHON_INSTALLER_SETUP_TYPE = distutils
> +PYTHON_INSTALLER_SETUP_TYPE = flit
> +HOST_PYTHON_INSTALLER_ENV = PYTHONPATH=$(@D)/src

  You said that all these packages have to be converted in a single patch, but 
that's not true: it's perfectly possible to change the infra and keep using 
distutils for this package. The infra changes don't affect distutils setup type 
at all. Only the change in flit-core really needs to be in the same patch.


  BTW, when someone reviews a patch (i.e. Yann), it's nice to put them in Cc of 
later iterations.

  Regards,
  Arnout


>   
>   $(eval $(host-python-package))
> diff --git a/package/python-pep517/python-pep517.mk b/package/python-pep517/python-pep517.mk
> index 99aa62d51d..1ca1bc4e35 100644
> --- a/package/python-pep517/python-pep517.mk
> +++ b/package/python-pep517/python-pep517.mk
> @@ -9,7 +9,7 @@ PYTHON_PEP517_SOURCE = pep517-$(PYTHON_PEP517_VERSION).tar.gz
>   PYTHON_PEP517_SITE = https://files.pythonhosted.org/packages/0a/65/6e656d49c679136edfba25f25791f45ffe1ea4ae2ec1c59fe9c35e061cd1
>   PYTHON_PEP517_LICENSE = MIT
>   PYTHON_PEP517_LICENSE_FILES = LICENSE
> -PYTHON_PEP517_SETUP_TYPE = distutils
> +PYTHON_PEP517_SETUP_TYPE = flit
>   HOST_PYTHON_PEP517_DEPENDENCIES = host-python-tomli
>   
>   $(eval $(host-python-package))
> diff --git a/package/python-tomli/python-tomli.mk b/package/python-tomli/python-tomli.mk
> index b8c20ca736..3539a505be 100644
> --- a/package/python-tomli/python-tomli.mk
> +++ b/package/python-tomli/python-tomli.mk
> @@ -7,7 +7,7 @@
>   PYTHON_TOMLI_VERSION = 1.2.0
>   PYTHON_TOMLI_SOURCE = tomli-$(PYTHON_TOMLI_VERSION).tar.gz
>   PYTHON_TOMLI_SITE = https://files.pythonhosted.org/packages/ec/38/8eccdc662c61aed187d5f5b168c18b1d2de3827976c3691e4da8be7375aa
> -PYTHON_TOMLI_SETUP_TYPE = distutils
> +PYTHON_TOMLI_SETUP_TYPE = flit
>   PYTHON_TOMLI_LICENSE = MIT
>   PYTHON_TOMLI_LICENSE_FILES = LICENSE
>   
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH v2 1/1] package/pkg-python: migrate flit to new bootstrapping sequence

[Роман Донченко]

Hi,

24.03.2022 23:37, Arnout Vandecappelle пишет:

>>   else
>>   $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV)
>> -$(2)_BASE_BUILD_CMD = -m build -n -w
>> -$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py 
>> dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS)
>> +# Use flit built in wheel builder for packages that are 
>> host-python-pypa-build dependencies.
>> +# This is needed to avoid a circular with host-python-pypa-build and 
>> those dependencies.
>> +#
>> +$(2)_BASE_BUILD_CMD = $$(if \
>> +        $$(filter \
>> +            host-python-flit-core host-python-installer 
>> host-python-pep517 host-python-tomli, \
>> +            $(1)) \
>> +        , \
>> +        -m flit_core.wheel, \
>> +        -m build -n -w \
> 
>   This is adding way too much complexity to the infra just to support 
> these four packages. I think there a a couple of better options:
> 
> - don't use python infra at all for these packages; or
> - set HOST_PYTHON_FLIT_CORE_BUILD_CMDS etc. in python-flit-core.mk; or
> - add a fifth SETUP_TYPE "generic" or "custom" or something, which 
> requires $(2)_BASE_* to be set by the python-flit-core.mk;
> - in the infra change it to $(2)_BASE_BUILD_CMD ?=, set 
> HOST_PYTHON_FLIT_CORE_BASE_BUILD_CMD in python-flit-core.mk; or
> - add a fifth SETUP_TYPE "flit-bootstrap" or something like that which 
> only gets used by the problematic packages.
> 
> 
>   The latter two options have my preference because I think they are the 
> cleanest, but any of them is OK for me. The "generic" setup type and the 
> overridable _BASE_BUILD_CMD would also require an update in the manual.

IMO, the simplest implementation would be:

1. For host-python-flit-core and host-python-installer, use 
host-generic-package and define the build and install commands manually.

2. For every other package whose SETUP_TYPE is "flit", have it depend on 
just host-python-flit-core and host-python-installer, and build with 
"python -m flit_core.wheel". In other words, don't use 
host-python-pypa-build.

I don't think there are any advantages in using pypa-build over just 
running flit_core.wheel directly, and dropping it would reduce the 
number of packages that need special handling and simplify the 
dependency tree.


>> diff --git a/package/python-flit-core/python-flit-core.mk 
>> b/package/python-flit-core/python-flit-core.mk
>> index 0e058a1f17..d206a72f82 100644
>> --- a/package/python-flit-core/python-flit-core.mk
>> +++ b/package/python-flit-core/python-flit-core.mk
>> @@ -8,6 +8,6 @@ PYTHON_FLIT_CORE_VERSION = 3.7.1
>>   PYTHON_FLIT_CORE_SOURCE = flit_core-$(PYTHON_FLIT_CORE_VERSION).tar.gz
>>   PYTHON_FLIT_CORE_SITE = 
>> https://files.pythonhosted.org/packages/15/d1/d8798b83e953fd6f86ca9b50f93eec464a9305b0661469c8234e61095481 
>>
>>   PYTHON_FLIT_CORE_LICENSE = BSD-3-Clause
>> -PYTHON_FLIT_CORE_SETUP_TYPE = pep517
>> +PYTHON_FLIT_CORE_SETUP_TYPE = flit
> 
>   This means there's no package any more that uses pep517 setup type. Do 
> you think it's ever going to be used again? Or can we remove it altogether?

I think python-setuptools could be a potential user of this, if 
python-pypa-build ever becomes buildable without setuptools (which it 
very well might).

-Roman.
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH v2 1/1] package/pkg-python: migrate flit to new bootstrapping sequence

[James Hilliard]

On Thu, Mar 24, 2022 at 4:28 PM Роман Донченко <dpb@corrigendum.ru> wrote:
>
> Hi,
>
> 24.03.2022 23:37, Arnout Vandecappelle пишет:
>
> >>   else
> >>   $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV)
> >> -$(2)_BASE_BUILD_CMD = -m build -n -w
> >> -$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py
> >> dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS)
> >> +# Use flit built in wheel builder for packages that are
> >> host-python-pypa-build dependencies.
> >> +# This is needed to avoid a circular with host-python-pypa-build and
> >> those dependencies.
> >> +#
> >> +$(2)_BASE_BUILD_CMD = $$(if \
> >> +        $$(filter \
> >> +            host-python-flit-core host-python-installer
> >> host-python-pep517 host-python-tomli, \
> >> +            $(1)) \
> >> +        , \
> >> +        -m flit_core.wheel, \
> >> +        -m build -n -w \
> >
> >   This is adding way too much complexity to the infra just to support
> > these four packages. I think there a a couple of better options:
> >
> > - don't use python infra at all for these packages; or
> > - set HOST_PYTHON_FLIT_CORE_BUILD_CMDS etc. in python-flit-core.mk; or
> > - add a fifth SETUP_TYPE "generic" or "custom" or something, which
> > requires $(2)_BASE_* to be set by the python-flit-core.mk;
> > - in the infra change it to $(2)_BASE_BUILD_CMD ?=, set
> > HOST_PYTHON_FLIT_CORE_BASE_BUILD_CMD in python-flit-core.mk; or
> > - add a fifth SETUP_TYPE "flit-bootstrap" or something like that which
> > only gets used by the problematic packages.

I added a flit-bootstrap SETUP_TYPE and isolated the special casing
logic to that:
https://patchwork.ozlabs.org/project/buildroot/patch/20220324223715.1961461-1-james.hilliard1@gmail.com/

> >
> >
> >   The latter two options have my preference because I think they are the
> > cleanest, but any of them is OK for me. The "generic" setup type and the
> > overridable _BASE_BUILD_CMD would also require an update in the manual.
>
> IMO, the simplest implementation would be:
>
> 1. For host-python-flit-core and host-python-installer, use
> host-generic-package and define the build and install commands manually.

This is more complex since we only need to special case a few things.

>
> 2. For every other package whose SETUP_TYPE is "flit", have it depend on
> just host-python-flit-core and host-python-installer, and build with
> "python -m flit_core.wheel". In other words, don't use
> host-python-pypa-build.

This is not recommended by upstream AFAIU, "python -m flit_core.wheel" is
only supposed to be used for initial toolchain bootstrapping as it's
not a generic
pep517 build frontend.

>
> I don't think there are any advantages in using pypa-build over just
> running flit_core.wheel directly, and dropping it would reduce the
> number of packages that need special handling and simplify the
> dependency tree.

We will need a full pep517 build frontend anyways so this wouldn't simplify
anything IMO.

>
>
> >> diff --git a/package/python-flit-core/python-flit-core.mk
> >> b/package/python-flit-core/python-flit-core.mk
> >> index 0e058a1f17..d206a72f82 100644
> >> --- a/package/python-flit-core/python-flit-core.mk
> >> +++ b/package/python-flit-core/python-flit-core.mk
> >> @@ -8,6 +8,6 @@ PYTHON_FLIT_CORE_VERSION = 3.7.1
> >>   PYTHON_FLIT_CORE_SOURCE = flit_core-$(PYTHON_FLIT_CORE_VERSION).tar.gz
> >>   PYTHON_FLIT_CORE_SITE =
> >> https://files.pythonhosted.org/packages/15/d1/d8798b83e953fd6f86ca9b50f93eec464a9305b0661469c8234e61095481
> >>
> >>   PYTHON_FLIT_CORE_LICENSE = BSD-3-Clause
> >> -PYTHON_FLIT_CORE_SETUP_TYPE = pep517
> >> +PYTHON_FLIT_CORE_SETUP_TYPE = flit
> >
> >   This means there's no package any more that uses pep517 setup type. Do
> > you think it's ever going to be used again? Or can we remove it altogether?
>
> I think python-setuptools could be a potential user of this, if
> python-pypa-build ever becomes buildable without setuptools (which it
> very well might).

From my understanding upstream intends to migrate python-pypa-build to a flit
based build in the future. At that point setuptools could itself use
the generic pep517
SETUP_TYPE since it uses a local pep517 build backend.

>
> -Roman.
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH v2 1/1] package/pkg-python: migrate flit to new bootstrapping sequence

[Роман Донченко]

25.03.2022 1:47, James Hilliard пишет:
> On Thu, Mar 24, 2022 at 4:28 PM Роман Донченко <dpb@corrigendum.ru> wrote:
>>
>> Hi,
>>
>> 24.03.2022 23:37, Arnout Vandecappelle пишет:
>>
>>>>    else
>>>>    $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV)
>>>> -$(2)_BASE_BUILD_CMD = -m build -n -w
>>>> -$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py
>>>> dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS)
>>>> +# Use flit built in wheel builder for packages that are
>>>> host-python-pypa-build dependencies.
>>>> +# This is needed to avoid a circular with host-python-pypa-build and
>>>> those dependencies.
>>>> +#
>>>> +$(2)_BASE_BUILD_CMD = $$(if \
>>>> +        $$(filter \
>>>> +            host-python-flit-core host-python-installer
>>>> host-python-pep517 host-python-tomli, \
>>>> +            $(1)) \
>>>> +        , \
>>>> +        -m flit_core.wheel, \
>>>> +        -m build -n -w \
>>>
>>>    This is adding way too much complexity to the infra just to support
>>> these four packages. I think there a a couple of better options:
>>>
>>> - don't use python infra at all for these packages; or
>>> - set HOST_PYTHON_FLIT_CORE_BUILD_CMDS etc. in python-flit-core.mk; or
>>> - add a fifth SETUP_TYPE "generic" or "custom" or something, which
>>> requires $(2)_BASE_* to be set by the python-flit-core.mk;
>>> - in the infra change it to $(2)_BASE_BUILD_CMD ?=, set
>>> HOST_PYTHON_FLIT_CORE_BASE_BUILD_CMD in python-flit-core.mk; or
>>> - add a fifth SETUP_TYPE "flit-bootstrap" or something like that which
>>> only gets used by the problematic packages.
> 
> I added a flit-bootstrap SETUP_TYPE and isolated the special casing
> logic to that:
> https://patchwork.ozlabs.org/project/buildroot/patch/20220324223715.1961461-1-james.hilliard1@gmail.com/
> 
>>>
>>>
>>>    The latter two options have my preference because I think they are the
>>> cleanest, but any of them is OK for me. The "generic" setup type and the
>>> overridable _BASE_BUILD_CMD would also require an update in the manual.
>>
>> IMO, the simplest implementation would be:
>>
>> 1. For host-python-flit-core and host-python-installer, use
>> host-generic-package and define the build and install commands manually.
> 
> This is more complex since we only need to special case a few things.

Is it really that complex, though? Admittedly, I'm not hugely familiar 
with buildroot, but I wrote a test version of the makefile for 
flit-core, and I just had to define PYTHON_FLIT_CORE_DEPENDENCIES (= 
host-python), HOST_PYTHON_FLIT_CORE_BUILD_CMDS and 
HOST_PYTHON_FLIT_CORE_INSTALL_CMDS.
I think it's easier to understand that than to track all the special 
cases in pkg-python.mk.

>>
>> 2. For every other package whose SETUP_TYPE is "flit", have it depend on
>> just host-python-flit-core and host-python-installer, and build with
>> "python -m flit_core.wheel". In other words, don't use
>> host-python-pypa-build.
> 
> This is not recommended by upstream AFAIU, "python -m flit_core.wheel" is
> only supposed to be used for initial toolchain bootstrapping as it's
> not a generic
> pep517 build frontend.

Yes, but we don't need a generic build frontend. We already know that 
these packages use flit as the backend, and the build dependencies are 
installed by Buildroot, so the function of the frontend is reduced to 
just calling the backend - so you might as well remove the frontend and 
call the backend directly.

Also, you can see that flit_core.wheel uses the same underlying code as 
Flit's PEP 517 backend, so I don't think there can be cases where the 
latter will work, but the former won't:

https://github.com/pypa/flit/blob/3.7.1/flit_core/flit_core/wheel.py#L235
https://github.com/pypa/flit/blob/3.7.1/flit_core/flit_core/buildapi.py#L70

> 
>>
>> I don't think there are any advantages in using pypa-build over just
>> running flit_core.wheel directly, and dropping it would reduce the
>> number of packages that need special handling and simplify the
>> dependency tree.
> 
> We will need a full pep517 build frontend anyways so this wouldn't simplify
> anything IMO.

The simplification is in not having two different flit-based SETUP_TYPEs.

-Roman.
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

Re: [Buildroot] [PATCH v2 1/1] package/pkg-python: migrate flit to new bootstrapping sequence

[James Hilliard]

On Thu, Mar 24, 2022 at 5:32 PM Роман Донченко <dpb@corrigendum.ru> wrote:
>
>
>
> 25.03.2022 1:47, James Hilliard пишет:
> > On Thu, Mar 24, 2022 at 4:28 PM Роман Донченко <dpb@corrigendum.ru> wrote:
> >>
> >> Hi,
> >>
> >> 24.03.2022 23:37, Arnout Vandecappelle пишет:
> >>
> >>>>    else
> >>>>    $(2)_BASE_ENV = $$(HOST_PKG_PYTHON_PEP517_ENV)
> >>>> -$(2)_BASE_BUILD_CMD = -m build -n -w
> >>>> -$(2)_BASE_INSTALL_CMD = $(TOPDIR)/support/scripts/pyinstaller.py
> >>>> dist/* $$(HOST_PKG_PYTHON_PEP517_INSTALL_OPTS)
> >>>> +# Use flit built in wheel builder for packages that are
> >>>> host-python-pypa-build dependencies.
> >>>> +# This is needed to avoid a circular with host-python-pypa-build and
> >>>> those dependencies.
> >>>> +#
> >>>> +$(2)_BASE_BUILD_CMD = $$(if \
> >>>> +        $$(filter \
> >>>> +            host-python-flit-core host-python-installer
> >>>> host-python-pep517 host-python-tomli, \
> >>>> +            $(1)) \
> >>>> +        , \
> >>>> +        -m flit_core.wheel, \
> >>>> +        -m build -n -w \
> >>>
> >>>    This is adding way too much complexity to the infra just to support
> >>> these four packages. I think there a a couple of better options:
> >>>
> >>> - don't use python infra at all for these packages; or
> >>> - set HOST_PYTHON_FLIT_CORE_BUILD_CMDS etc. in python-flit-core.mk; or
> >>> - add a fifth SETUP_TYPE "generic" or "custom" or something, which
> >>> requires $(2)_BASE_* to be set by the python-flit-core.mk;
> >>> - in the infra change it to $(2)_BASE_BUILD_CMD ?=, set
> >>> HOST_PYTHON_FLIT_CORE_BASE_BUILD_CMD in python-flit-core.mk; or
> >>> - add a fifth SETUP_TYPE "flit-bootstrap" or something like that which
> >>> only gets used by the problematic packages.
> >
> > I added a flit-bootstrap SETUP_TYPE and isolated the special casing
> > logic to that:
> > https://patchwork.ozlabs.org/project/buildroot/patch/20220324223715.1961461-1-james.hilliard1@gmail.com/
> >
> >>>
> >>>
> >>>    The latter two options have my preference because I think they are the
> >>> cleanest, but any of them is OK for me. The "generic" setup type and the
> >>> overridable _BASE_BUILD_CMD would also require an update in the manual.
> >>
> >> IMO, the simplest implementation would be:
> >>
> >> 1. For host-python-flit-core and host-python-installer, use
> >> host-generic-package and define the build and install commands manually.
> >
> > This is more complex since we only need to special case a few things.
>
> Is it really that complex, though? Admittedly, I'm not hugely familiar
> with buildroot, but I wrote a test version of the makefile for
> flit-core, and I just had to define PYTHON_FLIT_CORE_DEPENDENCIES (=
> host-python), HOST_PYTHON_FLIT_CORE_BUILD_CMDS and
> HOST_PYTHON_FLIT_CORE_INSTALL_CMDS.
> I think it's easier to understand that than to track all the special
> cases in pkg-python.mk.

It's harder to maintain IMO since you'd then have to manually sync
generic fixes in
pkg-python.mk with multiple package makefiles.

There's really not that much special casing needed in pkg-python.mk IMO and
it's easier to do there since the special casing needs to apply to a
few packages.

>
> >>
> >> 2. For every other package whose SETUP_TYPE is "flit", have it depend on
> >> just host-python-flit-core and host-python-installer, and build with
> >> "python -m flit_core.wheel". In other words, don't use
> >> host-python-pypa-build.
> >
> > This is not recommended by upstream AFAIU, "python -m flit_core.wheel" is
> > only supposed to be used for initial toolchain bootstrapping as it's
> > not a generic
> > pep517 build frontend.
>
> Yes, but we don't need a generic build frontend. We already know that
> these packages use flit as the backend, and the build dependencies are
> installed by Buildroot, so the function of the frontend is reduced to
> just calling the backend - so you might as well remove the frontend and
> call the backend directly.

Setuptools for example would be a non-flit pep517 package.

The goal here is not just to provide flit support but rather to bring up the
necessary infrastructure for installing different types of pep517 packages
that may use different pep517 backends.

>
> Also, you can see that flit_core.wheel uses the same underlying code as
> Flit's PEP 517 backend, so I don't think there can be cases where the
> latter will work, but the former won't:
>
> https://github.com/pypa/flit/blob/3.7.1/flit_core/flit_core/wheel.py#L235
> https://github.com/pypa/flit/blob/3.7.1/flit_core/flit_core/buildapi.py#L70

The main issue is that flit_core.wheel isn't a generic pep517 frontend and it's
recommended to move to pypa-build once bootstrapped:
https://github.com/pypa/flit/blob/3.7.1/doc/bootstrap.rst

I'm also wary of using the python -m flit_core.wheel bootstrapping frontend
for packages that aren't needed for bootstrapping as it's unlikely to be tested
much in general compared with a full pep517 frontend like pypa-build.

>
> >
> >>
> >> I don't think there are any advantages in using pypa-build over just
> >> running flit_core.wheel directly, and dropping it would reduce the
> >> number of packages that need special handling and simplify the
> >> dependency tree.
> >
> > We will need a full pep517 build frontend anyways so this wouldn't simplify
> > anything IMO.
>
> The simplification is in not having two different flit-based SETUP_TYPEs.

I mean, I'm trying to share the pypa-build frontend across all pep517 based
package builds other than those where it's not possible, bringing up build here
will simplify adding support for non-flit pep517 based packages down the line
as we will already have a functional generic pep517 build frontend then.

The regular flit SETUP_TYPE is basically identical to the pep517 SETUP_TYPE
except that it has a host-python-flit-core dependency so there's very
little extra
code actually needed for the non-bootstrap flit SETUP_TYPE.

>
> -Roman.
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot