* [Buildroot] [PATCH v2 2/4] package/pkg-python: add flit package infrastructure
2021-12-05 0:11 [Buildroot] [PATCH v2 1/4] package/python-tomli: new package James Hilliard
@ 2021-12-05 0:11 ` James Hilliard
2021-12-05 0:11 ` [Buildroot] [PATCH v2 3/4] utils/scanpypi: add flit package support James Hilliard
2021-12-05 0:11 ` [Buildroot] [PATCH v2 4/4] package/python-tinycss2: bump to version 1.1.1 James Hilliard
2 siblings, 0 replies; 4+ messages in thread
From: James Hilliard @ 2021-12-05 0:11 UTC (permalink / raw)
To: buildroot; +Cc: James Hilliard, Asaf Kahlon, Thomas Petazzoni
Due to flit removing the setup.py generation from the default sdist
releases we need to add a hook that generates them before we can do
a build and install using distutils.
Installing flit/flit-core is also a bit tricky since we also need
to generate setup.py's for them as well, we can do this by adding
their build directories to path and calling the internal methods
required to do the setup.py generation.
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
package/pkg-python.mk | 30 ++++++-
...sion-parser-for-multiple-assignments.patch | 80 +++++++++++++++++++
.../python-flit-core/python-flit-core.hash | 3 +
package/python-flit-core/python-flit-core.mk | 34 ++++++++
package/python-flit/python-flit.hash | 5 ++
package/python-flit/python-flit.mk | 17 ++++
6 files changed, 166 insertions(+), 3 deletions(-)
create mode 100644 package/python-flit-core/0001-Fix-ast-version-parser-for-multiple-assignments.patch
create mode 100644 package/python-flit-core/python-flit-core.hash
create mode 100644 package/python-flit-core/python-flit-core.mk
create mode 100644 package/python-flit/python-flit.hash
create mode 100644 package/python-flit/python-flit.mk
diff --git a/package/pkg-python.mk b/package/pkg-python.mk
index e6b81bdfd3..f50c33a89a 100644
--- a/package/pkg-python.mk
+++ b/package/pkg-python.mk
@@ -120,8 +120,8 @@ ifndef $(2)_SETUP_TYPE
endif
endif
-# Distutils
-ifeq ($$($(2)_SETUP_TYPE),distutils)
+# Distutils/flit
+ifneq ($$(filter distutils flit,$$($(2)_SETUP_TYPE)),)
ifeq ($(4),target)
$(2)_BASE_ENV = $$(PKG_PYTHON_DISTUTILS_ENV)
$(2)_BASE_BUILD_TGT = build
@@ -149,7 +149,7 @@ $(2)_BASE_BUILD_OPTS =
$(2)_BASE_INSTALL_OPTS = $$(HOST_PKG_PYTHON_SETUPTOOLS_INSTALL_OPTS)
endif
else
-$$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils' or 'setuptools'")
+$$(error "Invalid $(2)_SETUP_TYPE. Valid options are 'distutils', 'setuptools' or 'flit")
endif
# Target packages need both the python interpreter on the target (for
@@ -212,6 +212,10 @@ $(2)_DEPENDENCIES += $$(if $$(filter host-python-setuptools,$(1)),,host-python-s
endif
endif # SETUP_TYPE
+ifeq ($$($(2)_SETUP_TYPE),flit)
+$(2)_DEPENDENCIES += $$(if $$(filter host-python-flit host-python-flit-core,$(1)),,host-python-flit-core)
+endif # SETUP_TYPE
+
# Python interpreter to use for building the package.
#
# We may want to specify the python interpreter to be used for building a
@@ -236,12 +240,32 @@ $(2)_PYTHON_INTERPRETER = $$(HOST_DIR)/bin/$$($(2)_NEEDS_HOST_PYTHON)
endif
endif
+ifeq ($$($(2)_SETUP_TYPE),flit)
+ifndef $(2)_FLIT_GENERATE_SETUP
+define $(2)_FLIT_GENERATE_SETUP
+ (cd $$($$(PKG)_BUILDDIR)/; \
+ $$($$(PKG)_BASE_ENV) $$($$(PKG)_ENV) \
+ $$($(2)_PYTHON_INTERPRETER) -c \
+ "import sys; \
+ sys.modules['requests'] = False; \
+ from flit.sdist import SdistBuilder; \
+ from pathlib import Path; \
+ cwd=Path.cwd(); \
+ setup=cwd.joinpath('setup.py').open('wb'); \
+ pyproject=cwd.joinpath('pyproject.toml'); \
+ builder=SdistBuilder.from_ini_path(pyproject); \
+ setup.write(builder.make_setup_py())")
+endef
+endif
+endif
+
#
# Build step. Only define it if not already defined by the package .mk
# file.
#
ifndef $(2)_BUILD_CMDS
define $(2)_BUILD_CMDS
+ $$($$(PKG)_FLIT_GENERATE_SETUP)
(cd $$($$(PKG)_BUILDDIR)/; \
$$($$(PKG)_BASE_ENV) $$($$(PKG)_ENV) \
$$($(2)_PYTHON_INTERPRETER) setup.py \
diff --git a/package/python-flit-core/0001-Fix-ast-version-parser-for-multiple-assignments.patch b/package/python-flit-core/0001-Fix-ast-version-parser-for-multiple-assignments.patch
new file mode 100644
index 0000000000..5a94e1dcab
--- /dev/null
+++ b/package/python-flit-core/0001-Fix-ast-version-parser-for-multiple-assignments.patch
@@ -0,0 +1,80 @@
+From 2cd8b5708be88b90ea2fa0fb35407a5ec2038c8e Mon Sep 17 00:00:00 2001
+From: James Hilliard <james.hilliard1@gmail.com>
+Date: Sat, 27 Nov 2021 02:36:15 -0700
+Subject: [PATCH] Fix ast version parser for multiple assignments
+
+Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
+[Upstream status:
+https://github.com/takluyver/flit/pull/474]
+---
+ flit_core/common.py | 21 +++++++++++--------
+ .../tests/samples/moduleunimportabledouble.py | 8 +++++++
+ flit_core/tests/test_common.py | 5 +++++
+ 3 files changed, 25 insertions(+), 9 deletions(-)
+ create mode 100644 flit_core/tests/samples/moduleunimportabledouble.py
+
+diff --git a/flit_core/common.py b/flit_core/common.py
+index f1f378f..86bcf4b 100644
+--- a/flit_core/common.py
++++ b/flit_core/common.py
+@@ -132,15 +132,18 @@ def get_docstring_and_version_via_ast(target):
+ for child in node.body:
+ # Only use the version from the given module if it's a simple
+ # string assignment to __version__
+- is_version_str = (
+- isinstance(child, ast.Assign)
+- and len(child.targets) == 1
+- and isinstance(child.targets[0], ast.Name)
+- and child.targets[0].id == "__version__"
+- and isinstance(child.value, ast.Str)
+- )
+- if is_version_str:
+- version = child.value.s
++ if isinstance(child, ast.Assign):
++ for target in child.targets:
++ is_version_str = (
++ isinstance(target, ast.Name)
++ and target.id == "__version__"
++ and isinstance(child.value, ast.Str)
++ )
++ if is_version_str:
++ version = child.value.s
++ break
++ else:
++ continue
+ break
+ else:
+ version = None
+diff --git a/flit_core/tests/samples/moduleunimportabledouble.py b/flit_core/tests/samples/moduleunimportabledouble.py
+new file mode 100644
+index 0000000..42d51f3
+--- /dev/null
++++ b/flit_core/tests/samples/moduleunimportabledouble.py
+@@ -0,0 +1,8 @@
++
++"""
++A sample unimportable module with double assignment
++"""
++
++raise ImportError()
++
++VERSION = __version__ = "0.1"
+diff --git a/flit_core/tests/test_common.py b/flit_core/tests/test_common.py
+index 02cfab7..42e230b 100644
+--- a/flit_core/tests/test_common.py
++++ b/flit_core/tests/test_common.py
+@@ -70,6 +70,11 @@ class ModuleTests(TestCase):
+ 'version': '0.1'}
+ )
+
++ info = get_info_from_module(Module('moduleunimportabledouble', samples_dir))
++ self.assertEqual(info, {'summary': 'A sample unimportable module with double assignment',
++ 'version': '0.1'}
++ )
++
+ info = get_info_from_module(Module('module1', samples_dir / 'constructed_version'))
+ self.assertEqual(info, {'summary': 'This module has a __version__ that requires runtime interpretation',
+ 'version': '1.2.3'}
+--
+2.33.1
+
diff --git a/package/python-flit-core/python-flit-core.hash b/package/python-flit-core/python-flit-core.hash
new file mode 100644
index 0000000000..e32a05365b
--- /dev/null
+++ b/package/python-flit-core/python-flit-core.hash
@@ -0,0 +1,3 @@
+# md5, sha256 from https://pypi.org/pypi/flit_core/json
+md5 cf656ea7dac7cfa0f714e24bbadfb013 flit_core-3.5.1.tar.gz
+sha256 3083720351a6cb00e0634a1ec0e26eae7b273174c3c6c03d5b597a14203b282e flit_core-3.5.1.tar.gz
diff --git a/package/python-flit-core/python-flit-core.mk b/package/python-flit-core/python-flit-core.mk
new file mode 100644
index 0000000000..95a215bd39
--- /dev/null
+++ b/package/python-flit-core/python-flit-core.mk
@@ -0,0 +1,34 @@
+################################################################################
+#
+# python-flit-core
+#
+################################################################################
+
+PYTHON_FLIT_CORE_VERSION = 3.5.1
+PYTHON_FLIT_CORE_SOURCE = flit_core-$(PYTHON_FLIT_CORE_VERSION).tar.gz
+PYTHON_FLIT_CORE_SITE = https://files.pythonhosted.org/packages/7e/1e/15198966abf00e590ec95fb8aa4ba3d274897fe7b182fce2867f672f6a91
+PYTHON_FLIT_CORE_SETUP_TYPE = flit
+PYTHON_FLIT_CORE_LICENSE = BSD-3-Clause
+HOST_PYTHON_FLIT_CORE_NEEDS_HOST_PYTHON = python3
+HOST_PYTHON_FLIT_CORE_DEPENDENCIES = host-python-flit
+HOST_PYTHON_FLIT_CORE_ENV = PYTHONPATH="$(PYTHON3_PATH):$(@D)"
+
+define HOST_PYTHON_FLIT_CORE_FLIT_GENERATE_SETUP
+ cd $(HOST_PYTHON_FLIT_CORE_BUILDDIR)/; \
+ $(HOST_PYTHON_FLIT_CORE_BASE_ENV) $(HOST_PYTHON_FLIT_CORE_ENV) \
+ $(HOST_PYTHON_FLIT_CORE_PYTHON_INTERPRETER) -c \
+ "import sys; \
+ sys.modules['requests'] = False; \
+ from flit.sdist import SdistBuilder; \
+ from flit_core.build_thyself import metadata, Module; \
+ from pathlib import Path; \
+ cwd=Path.cwd(); \
+ module=Module('flit_core', cwd); \
+ reqs_by_extra={'.none': metadata.requires}; \
+ extra_files=['pyproject.toml', 'build_dists.py']; \
+ setup=cwd.joinpath('setup.py').open('wb'); \
+ builder=SdistBuilder(module, metadata, cwd, reqs_by_extra, {}, extra_files); \
+ setup.write(builder.make_setup_py())"
+endef
+
+$(eval $(host-python-package))
diff --git a/package/python-flit/python-flit.hash b/package/python-flit/python-flit.hash
new file mode 100644
index 0000000000..e676b85090
--- /dev/null
+++ b/package/python-flit/python-flit.hash
@@ -0,0 +1,5 @@
+# md5, sha256 from https://pypi.org/pypi/flit/json
+md5 1ff1eb6e71cac6d873808fa12328a8c5 flit-3.5.1.tar.gz
+sha256 2e3b7377714483ecc54b236330d7bf3467b9f5b909b22333b50b6b4324162510 flit-3.5.1.tar.gz
+# Locally computed sha256 checksums
+sha256 35b4f1dec512e617077fd6980dbb43ef8c2887adc5d0185edb4c04da175dd816 LICENSE
diff --git a/package/python-flit/python-flit.mk b/package/python-flit/python-flit.mk
new file mode 100644
index 0000000000..557ee7d8d9
--- /dev/null
+++ b/package/python-flit/python-flit.mk
@@ -0,0 +1,17 @@
+################################################################################
+#
+# python-flit
+#
+################################################################################
+
+PYTHON_FLIT_VERSION = 3.5.1
+PYTHON_FLIT_SOURCE = flit-$(PYTHON_FLIT_VERSION).tar.gz
+PYTHON_FLIT_SITE = https://files.pythonhosted.org/packages/a8/bc/4bbb98f1ce0ae4e778f3486c39c487a066d69f5ffe4e762c79664dfd5df8
+PYTHON_FLIT_SETUP_TYPE = flit
+PYTHON_FLIT_LICENSE = BSD-3-Clause
+PYTHON_FLIT_LICENSE_FILES = LICENSE
+HOST_PYTHON_FLIT_NEEDS_HOST_PYTHON = python3
+HOST_PYTHON_FLIT_DEPENDENCIES = host-python-docutils host-python-tomli
+HOST_PYTHON_FLIT_ENV = PYTHONPATH="$(PYTHON3_PATH):$(@D):$(@D)/flit_core"
+
+$(eval $(host-python-package))
--
2.25.1
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply related [flat|nested] 4+ messages in thread* [Buildroot] [PATCH v2 3/4] utils/scanpypi: add flit package support
2021-12-05 0:11 [Buildroot] [PATCH v2 1/4] package/python-tomli: new package James Hilliard
2021-12-05 0:11 ` [Buildroot] [PATCH v2 2/4] package/pkg-python: add flit package infrastructure James Hilliard
@ 2021-12-05 0:11 ` James Hilliard
2021-12-05 0:11 ` [Buildroot] [PATCH v2 4/4] package/python-tinycss2: bump to version 1.1.1 James Hilliard
2 siblings, 0 replies; 4+ messages in thread
From: James Hilliard @ 2021-12-05 0:11 UTC (permalink / raw)
To: buildroot; +Cc: James Hilliard, Asaf Kahlon, Thomas Petazzoni
These packages don't have a setup.py so we instead need to parse their
pyproject.toml file.
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
utils/scanpypi | 81 ++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 65 insertions(+), 16 deletions(-)
diff --git a/utils/scanpypi b/utils/scanpypi
index 1f3326cf4d..4164a52db3 100755
--- a/utils/scanpypi
+++ b/utils/scanpypi
@@ -296,21 +296,67 @@ class BuildrootPackage():
current_dir = os.getcwd()
os.chdir(self.tmp_extract)
sys.path.insert(0, self.tmp_extract)
- s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
- imp.load_module('__main__', s_file, s_path, s_desc)
- if self.metadata_name in self.setup_args:
- pass
- elif self.metadata_name.replace('_', '-') in self.setup_args:
- self.metadata_name = self.metadata_name.replace('_', '-')
- elif self.metadata_name.replace('-', '_') in self.setup_args:
- self.metadata_name = self.metadata_name.replace('-', '_')
try:
- self.setup_metadata = self.setup_args[self.metadata_name]
- except KeyError:
- # This means setup was not called
- print('ERROR: Could not determine package metadata for {pkg}.\n'
- .format(pkg=self.real_name))
+ s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
+ imp.load_module('__main__', s_file, s_path, s_desc)
+ if self.metadata_name in self.setup_args:
+ pass
+ elif self.metadata_name.replace('_', '-') in self.setup_args:
+ self.metadata_name = self.metadata_name.replace('_', '-')
+ elif self.metadata_name.replace('-', '_') in self.setup_args:
+ self.metadata_name = self.metadata_name.replace('-', '_')
+ try:
+ self.setup_metadata = self.setup_args[self.metadata_name]
+ except KeyError:
+ # This means setup was not called
+ print('ERROR: Could not determine package metadata for {pkg}.\n'
+ .format(pkg=self.real_name))
+ raise
+ finally:
+ os.chdir(current_dir)
+ sys.path.remove(self.tmp_extract)
+
+ def load_pyproject(self):
+ """
+ Loads the corresponding pyproject.toml and store its metadata
+ """
+ from pprint import pprint
+ current_dir = os.getcwd()
+ os.chdir(self.tmp_extract)
+ sys.path.insert(0, self.tmp_extract)
+ try:
+ pyproject = open('pyproject.toml')
+ except FileNotFoundError:
raise
+ else:
+ try:
+ from pip._vendor.tomli import load as toml_load
+ except ImportError:
+ try:
+ from toml import load as _toml_load # noqa: F401
+ def toml_load(f):
+ w = io.TextIOWrapper(f, encoding="utf8", newline="")
+ try:
+ return _toml_load(w)
+ finally:
+ w.detach()
+ except ImportError:
+ print('This package needs toml')
+ raise
+ with pyproject:
+ pyproject_data = toml_load(pyproject)
+ try:
+ self.setup_metadata = pyproject_data.get('project', {})
+ self.metadata_name = self.setup_metadata.get('name', self.real_name)
+ build_backend = pyproject_data.get('build-system', {}).get('build-backend', None)
+ if build_backend is not None:
+ if build_backend == 'flit_core.buildapi':
+ self.setup_metadata['method'] = 'flit'
+ except KeyError:
+ # This means setup was not called
+ print('ERROR: Could not determine package metadata for {pkg}.\n'
+ .format(pkg=self.real_name))
+ raise
os.chdir(current_dir)
sys.path.remove(self.tmp_extract)
@@ -690,11 +736,14 @@ def main():
try:
package.load_setup()
except ImportError as err:
- if 'buildutils' in err.message:
+ if 'buildutils' in str(err):
print('This package needs buildutils')
+ continue
else:
- raise
- continue
+ try:
+ package.load_pyproject()
+ except Exception as e:
+ raise
except (AttributeError, KeyError) as error:
print('Error: Could not install package {pkg}: {error}'.format(
pkg=package.real_name, error=error))
--
2.25.1
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply related [flat|nested] 4+ messages in thread