[RFC][PATCH] Attempt to unify predefined package feed handling

[RFC][PATCH] Attempt to unify predefined package feed handling

[David Nyström]

local.conf defines 
PACKAGE_FEED_URIS = "http://www.feed_repo.org/repo/"

should we do it this way or some other way ?

[Bug 5407]



Signed-off-by: David Nyström <david.nystrom@enea.com>
---
 meta/lib/oe/package_manager.py | 76 +++++++++++++++++++++++++++++++++++++++++-
 meta/lib/oe/rootfs.py          | 15 +++++++--
 2 files changed, 87 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index d29adac..d4050c6 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -223,6 +223,7 @@ class PackageManager(object):
         self.d = d
         self.deploy_dir = None
         self.deploy_lock = None
+        self.feed_uris = None
 
     """
     Update the package manager package database.
@@ -262,6 +263,10 @@ class PackageManager(object):
     def list_installed(self, format=None):
         pass
 
+    @abstractmethod
+    def insert_feeds_uris(self):
+        pass
+
     """
     Install complementary packages based upon the list of currently installed
     packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
@@ -339,6 +344,7 @@ class RpmPM(PackageManager):
         self.providename = providename
         self.fullpkglist = list()
         self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM', True)
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
         self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm")
         self.install_dir = os.path.join(self.target_rootfs, "install")
         self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm")
@@ -358,6 +364,41 @@ class RpmPM(PackageManager):
 
         self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
 
+
+    def insert_feeds_uris(self):
+        # List must be prefered to least preferred order
+        channel_priority = 5
+        default_platform_extra = set()
+        platform_extra = set()
+        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
+        for mlib in self.ml_os_list:
+            for arch in self.ml_prefix_list[mlib]:
+                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
+                if mlib == bbextendvariant:
+                        default_platform_extra.add(plt)
+                else:
+                        platform_extra.add(plt)
+                channel_priority += 5
+
+        platform_extra = platform_extra.union(default_platform_extra)
+
+        for uri in self.feed_uris.split():
+            channel_priority *= 2
+            uri_iterator = 0
+            for canonical_arch in platform_extra:
+                arch = canonical_arch.split('-')[0]
+                if arch == "noarch" or arch == "any":
+                    continue
+
+                bb.note('Note: adding Smart channel url%d%s (%s)' %
+                        (uri_iterator, arch, channel_priority))
+                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
+                                   % (uri_iterator, arch, uri, arch))
+                self._invoke_smart('channel --set url%d-%s priority=%d' %
+                                   (uri_iterator, arch, channel_priority))
+                channel_priority -= 5
+            uri_iterator += 1
+
     '''
     Create configs for rpm and smart, and multilib is supported
     '''
@@ -944,7 +985,7 @@ class OpkgPM(PackageManager):
 
         self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
-
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
         self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
         self.opkg_args += self.d.getVar("OPKG_ARGS", True)
@@ -1050,6 +1091,24 @@ class OpkgPM(PackageManager):
                     config_file.write("src oe-%s file:%s\n" %
                                       (arch, pkgs_dir))
 
+    def insert_feeds_uris(self):
+        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
+                                  % self.target_rootfs)
+
+        with open(rootfs_config, "w+") as config_file:
+            priority = 5
+            uri_iterator = 0
+            for uri in self.feed_uris.split():
+                config_file.write("src/gz url-%d %s/ipk\n" %
+                                  (uri_iterator, uri))
+
+                for arch in self.pkg_archs.split():
+                    if arch == "noarch" or arch == "any":
+                        continue
+                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
+                                      (arch, uri_iterator, uri, arch))
+                uri_iterator += 1
+
     def update(self):
         self.deploy_dir_lock()
 
@@ -1250,6 +1309,7 @@ class DpkgPM(PackageManager):
     def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
         super(DpkgPM, self).__init__(d)
         self.target_rootfs = target_rootfs
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
         self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB', True)
         if apt_conf_dir is None:
             self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt")
@@ -1410,6 +1470,20 @@ class DpkgPM(PackageManager):
         if result is not None:
             bb.fatal(result)
 
+    def insert_feeds_uris(self):
+        sources_conf = os.path.join("%s/etc/apt/sources.list"
+                                    % self.target_rootfs)
+        arch_list = []
+        archs = self.d.getVar('PACKAGE_ARCHS', True)
+        arch_list = archs.split()
+        with open(sources_conf, "w+") as sources_file:
+            for uri in self.feed_uris.split():
+                for arch in arch_list:
+                    if arch == "noarch" or arch == "any":
+                        continue
+                    sources_file.write("deb %s/deb/%s ./\n" %
+                                       (uri, arch))
+
     def _create_configs(self, archs, base_archs):
         base_archs = re.sub("_", "-", base_archs)
 
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index be0afa6..7be2a18 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -350,7 +350,10 @@ class RpmRootfs(Rootfs):
                     bb.fatal(message)
 
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         False, True, self.d):
+            return
+        self.pm.insert_feeds_uris()
 
     def _handle_intercept_failure(self, registered_pkgs):
         rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
@@ -433,7 +436,10 @@ class DpkgRootfs(Rootfs):
         pass
 
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         False, True, self.d):
+            return
+        self.pm.insert_feeds_uris()
 
 
 class OpkgRootfs(Rootfs):
@@ -699,8 +705,11 @@ class OpkgRootfs(Rootfs):
         pass
 
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         False, True, self.d):
+            return
 
+        self.pm.insert_feeds_uris()
 
 def create_rootfs(d, manifest_dir=None):
     env_bkp = os.environ.copy()
-- 
1.8.3.2

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[Laurentiu Palcu]

Hi David,

My comments/questions below.

laurentiu

On Thu, Feb 27, 2014 at 02:28:21PM +0100, David Nyström wrote:
> local.conf defines 
> PACKAGE_FEED_URIS = "http://www.feed_repo.org/repo/"
> 
> should we do it this way or some other way ?
> 
> [Bug 5407]
> 
> 
> 
> Signed-off-by: David Nyström <david.nystrom@enea.com>
> ---
>  meta/lib/oe/package_manager.py | 76 +++++++++++++++++++++++++++++++++++++++++-
>  meta/lib/oe/rootfs.py          | 15 +++++++--
>  2 files changed, 87 insertions(+), 4 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> index d29adac..d4050c6 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -223,6 +223,7 @@ class PackageManager(object):
>          self.d = d
>          self.deploy_dir = None
>          self.deploy_lock = None
> +        self.feed_uris = None
I guess you can do:
           self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or ""
since we're using the same variable for all backends.

>  
>      """
>      Update the package manager package database.
> @@ -262,6 +263,10 @@ class PackageManager(object):
>      def list_installed(self, format=None):
>          pass
>  
> +    @abstractmethod
> +    def insert_feeds_uris(self):
> +        pass
> +
>      """
>      Install complementary packages based upon the list of currently installed
>      packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
> @@ -339,6 +344,7 @@ class RpmPM(PackageManager):
>          self.providename = providename
>          self.fullpkglist = list()
>          self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM', True)
> +        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
can be removed, see above.

>          self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm")
>          self.install_dir = os.path.join(self.target_rootfs, "install")
>          self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm")
> @@ -358,6 +364,41 @@ class RpmPM(PackageManager):
>  
>          self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
>  
> +
> +    def insert_feeds_uris(self):
shouldn't we check that self.feed_uris != "" before continuing?

> +        # List must be prefered to least preferred order
> +        channel_priority = 5
> +        default_platform_extra = set()
> +        platform_extra = set()
> +        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
> +        for mlib in self.ml_os_list:
> +            for arch in self.ml_prefix_list[mlib]:
> +                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
> +                if mlib == bbextendvariant:
> +                        default_platform_extra.add(plt)
> +                else:
> +                        platform_extra.add(plt)
> +                channel_priority += 5
> +
> +        platform_extra = platform_extra.union(default_platform_extra)
> +
> +        for uri in self.feed_uris.split():
> +            channel_priority *= 2
> +            uri_iterator = 0
> +            for canonical_arch in platform_extra:
> +                arch = canonical_arch.split('-')[0]
> +                if arch == "noarch" or arch == "any":
> +                    continue
> +
> +                bb.note('Note: adding Smart channel url%d%s (%s)' %
> +                        (uri_iterator, arch, channel_priority))
> +                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
> +                                   % (uri_iterator, arch, uri, arch))
> +                self._invoke_smart('channel --set url%d-%s priority=%d' %
> +                                   (uri_iterator, arch, channel_priority))
> +                channel_priority -= 5
> +            uri_iterator += 1
> +
>      '''
>      Create configs for rpm and smart, and multilib is supported
>      '''
> @@ -944,7 +985,7 @@ class OpkgPM(PackageManager):
>  
>          self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
>          self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
> -
> +        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
can be removed, see above.

>          self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
>          self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
>          self.opkg_args += self.d.getVar("OPKG_ARGS", True)
> @@ -1050,6 +1091,24 @@ class OpkgPM(PackageManager):
>                      config_file.write("src oe-%s file:%s\n" %
>                                        (arch, pkgs_dir))
>  
> +    def insert_feeds_uris(self):
check self.feed_uris != "" here?

> +        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
> +                                  % self.target_rootfs)
> +
> +        with open(rootfs_config, "w+") as config_file:
> +            priority = 5
apparently this is not used at all in the code below, can be removed.

> +            uri_iterator = 0
> +            for uri in self.feed_uris.split():
> +                config_file.write("src/gz url-%d %s/ipk\n" %
> +                                  (uri_iterator, uri))
> +
> +                for arch in self.pkg_archs.split():
> +                    if arch == "noarch" or arch == "any":
> +                        continue
> +                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
> +                                      (arch, uri_iterator, uri, arch))
> +                uri_iterator += 1
> +
>      def update(self):
>          self.deploy_dir_lock()
>  
> @@ -1250,6 +1309,7 @@ class DpkgPM(PackageManager):
>      def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
>          super(DpkgPM, self).__init__(d)
>          self.target_rootfs = target_rootfs
> +        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
can be removed, see above.

>          self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB', True)
>          if apt_conf_dir is None:
>              self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt")
> @@ -1410,6 +1470,20 @@ class DpkgPM(PackageManager):
>          if result is not None:
>              bb.fatal(result)
>  
> +    def insert_feeds_uris(self):
check self.feed_uris != "" here?

> +        sources_conf = os.path.join("%s/etc/apt/sources.list"
> +                                    % self.target_rootfs)
> +        arch_list = []
> +        archs = self.d.getVar('PACKAGE_ARCHS', True)
> +        arch_list = archs.split()
> +        with open(sources_conf, "w+") as sources_file:
> +            for uri in self.feed_uris.split():
> +                for arch in arch_list:
> +                    if arch == "noarch" or arch == "any":
> +                        continue
> +                    sources_file.write("deb %s/deb/%s ./\n" %
> +                                       (uri, arch))
> +
>      def _create_configs(self, archs, base_archs):
>          base_archs = re.sub("_", "-", base_archs)
>  
> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> index be0afa6..7be2a18 100644
> --- a/meta/lib/oe/rootfs.py
> +++ b/meta/lib/oe/rootfs.py
> @@ -350,7 +350,10 @@ class RpmRootfs(Rootfs):
>                      bb.fatal(message)
>  
>      def _insert_feed_uris(self):
> -        pass
> +        if base_contains("IMAGE_FEATURES", "package-management",
> +                         False, True, self.d):
> +            return
> +        self.pm.insert_feeds_uris()
For the sake of code clarity, I would suggest:

        if base_contains("IMAGE_FEATURES", "package-management",
                         True, False, self.d):
            self.pm.insert_feeds_uris()

>  
>      def _handle_intercept_failure(self, registered_pkgs):
>          rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
> @@ -433,7 +436,10 @@ class DpkgRootfs(Rootfs):
>          pass
>  
>      def _insert_feed_uris(self):
> -        pass
> +        if base_contains("IMAGE_FEATURES", "package-management",
> +                         False, True, self.d):
> +            return
> +        self.pm.insert_feeds_uris()
same here.

>  
>  
>  class OpkgRootfs(Rootfs):
> @@ -699,8 +705,11 @@ class OpkgRootfs(Rootfs):
>          pass
>  
>      def _insert_feed_uris(self):
> -        pass
> +        if base_contains("IMAGE_FEATURES", "package-management",
> +                         False, True, self.d):
> +            return
>  
> +        self.pm.insert_feeds_uris()
same here.

>  
>  def create_rootfs(d, manifest_dir=None):
>      env_bkp = os.environ.copy()
> -- 
> 1.8.3.2
> 
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[David Nyström]

On 2014-02-27 15:24, Laurentiu Palcu wrote:
> Hi David,
>
> My comments/questions below.

Thank you for your detailed comments on the RFC.
I will return with V2, where your comments are addressed when the 
general approach is Acked.

What do you think about the general approach ?
I guess it was you who added the insert_feeds_uris stubs in rootfs.py ?

Br,
David

> laurentiu
>
> On Thu, Feb 27, 2014 at 02:28:21PM +0100, David Nyström wrote:
>> local.conf defines
>> PACKAGE_FEED_URIS = "http://www.feed_repo.org/repo/"
>>
>> should we do it this way or some other way ?
>>
>> [Bug 5407]
>>
>>
>>
>> Signed-off-by: David Nyström <david.nystrom@enea.com>
>> ---
>>   meta/lib/oe/package_manager.py | 76 +++++++++++++++++++++++++++++++++++++++++-
>>   meta/lib/oe/rootfs.py          | 15 +++++++--
>>   2 files changed, 87 insertions(+), 4 deletions(-)
>>
>> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
>> index d29adac..d4050c6 100644
>> --- a/meta/lib/oe/package_manager.py
>> +++ b/meta/lib/oe/package_manager.py
>> @@ -223,6 +223,7 @@ class PackageManager(object):
>>           self.d = d
>>           self.deploy_dir = None
>>           self.deploy_lock = None
>> +        self.feed_uris = None
> I guess you can do:
>             self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or ""
> since we're using the same variable for all backends.
>
>>
>>       """
>>       Update the package manager package database.
>> @@ -262,6 +263,10 @@ class PackageManager(object):
>>       def list_installed(self, format=None):
>>           pass
>>
>> +    @abstractmethod
>> +    def insert_feeds_uris(self):
>> +        pass
>> +
>>       """
>>       Install complementary packages based upon the list of currently installed
>>       packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
>> @@ -339,6 +344,7 @@ class RpmPM(PackageManager):
>>           self.providename = providename
>>           self.fullpkglist = list()
>>           self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM', True)
>> +        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
> can be removed, see above.
>
>>           self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm")
>>           self.install_dir = os.path.join(self.target_rootfs, "install")
>>           self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm")
>> @@ -358,6 +364,41 @@ class RpmPM(PackageManager):
>>
>>           self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
>>
>> +
>> +    def insert_feeds_uris(self):
> shouldn't we check that self.feed_uris != "" before continuing?
>
>> +        # List must be prefered to least preferred order
>> +        channel_priority = 5
>> +        default_platform_extra = set()
>> +        platform_extra = set()
>> +        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
>> +        for mlib in self.ml_os_list:
>> +            for arch in self.ml_prefix_list[mlib]:
>> +                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
>> +                if mlib == bbextendvariant:
>> +                        default_platform_extra.add(plt)
>> +                else:
>> +                        platform_extra.add(plt)
>> +                channel_priority += 5
>> +
>> +        platform_extra = platform_extra.union(default_platform_extra)
>> +
>> +        for uri in self.feed_uris.split():
>> +            channel_priority *= 2
>> +            uri_iterator = 0
>> +            for canonical_arch in platform_extra:
>> +                arch = canonical_arch.split('-')[0]
>> +                if arch == "noarch" or arch == "any":
>> +                    continue
>> +
>> +                bb.note('Note: adding Smart channel url%d%s (%s)' %
>> +                        (uri_iterator, arch, channel_priority))
>> +                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
>> +                                   % (uri_iterator, arch, uri, arch))
>> +                self._invoke_smart('channel --set url%d-%s priority=%d' %
>> +                                   (uri_iterator, arch, channel_priority))
>> +                channel_priority -= 5
>> +            uri_iterator += 1
>> +
>>       '''
>>       Create configs for rpm and smart, and multilib is supported
>>       '''
>> @@ -944,7 +985,7 @@ class OpkgPM(PackageManager):
>>
>>           self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
>>           self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
>> -
>> +        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
> can be removed, see above.
>
>>           self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
>>           self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
>>           self.opkg_args += self.d.getVar("OPKG_ARGS", True)
>> @@ -1050,6 +1091,24 @@ class OpkgPM(PackageManager):
>>                       config_file.write("src oe-%s file:%s\n" %
>>                                         (arch, pkgs_dir))
>>
>> +    def insert_feeds_uris(self):
> check self.feed_uris != "" here?
>> +        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
>> +                                  % self.target_rootfs)
>> +
>> +        with open(rootfs_config, "w+") as config_file:
>> +            priority = 5
> apparently this is not used at all in the code below, can be removed.
>
>> +            uri_iterator = 0
>> +            for uri in self.feed_uris.split():
>> +                config_file.write("src/gz url-%d %s/ipk\n" %
>> +                                  (uri_iterator, uri))
>> +
>> +                for arch in self.pkg_archs.split():
>> +                    if arch == "noarch" or arch == "any":
>> +                        continue
>> +                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
>> +                                      (arch, uri_iterator, uri, arch))
>> +                uri_iterator += 1
>> +
>>       def update(self):
>>           self.deploy_dir_lock()
>>
>> @@ -1250,6 +1309,7 @@ class DpkgPM(PackageManager):
>>       def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
>>           super(DpkgPM, self).__init__(d)
>>           self.target_rootfs = target_rootfs
>> +        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
> can be removed, see above.
>
>>           self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB', True)
>>           if apt_conf_dir is None:
>>               self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt")
>> @@ -1410,6 +1470,20 @@ class DpkgPM(PackageManager):
>>           if result is not None:
>>               bb.fatal(result)
>>
>> +    def insert_feeds_uris(self):
> check self.feed_uris != "" here?
>
>> +        sources_conf = os.path.join("%s/etc/apt/sources.list"
>> +                                    % self.target_rootfs)
>> +        arch_list = []
>> +        archs = self.d.getVar('PACKAGE_ARCHS', True)
>> +        arch_list = archs.split()
>> +        with open(sources_conf, "w+") as sources_file:
>> +            for uri in self.feed_uris.split():
>> +                for arch in arch_list:
>> +                    if arch == "noarch" or arch == "any":
>> +                        continue
>> +                    sources_file.write("deb %s/deb/%s ./\n" %
>> +                                       (uri, arch))
>> +
>>       def _create_configs(self, archs, base_archs):
>>           base_archs = re.sub("_", "-", base_archs)
>>
>> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
>> index be0afa6..7be2a18 100644
>> --- a/meta/lib/oe/rootfs.py
>> +++ b/meta/lib/oe/rootfs.py
>> @@ -350,7 +350,10 @@ class RpmRootfs(Rootfs):
>>                       bb.fatal(message)
>>
>>       def _insert_feed_uris(self):
>> -        pass
>> +        if base_contains("IMAGE_FEATURES", "package-management",
>> +                         False, True, self.d):
>> +            return
>> +        self.pm.insert_feeds_uris()
> For the sake of code clarity, I would suggest:
>
>          if base_contains("IMAGE_FEATURES", "package-management",
>                           True, False, self.d):
>              self.pm.insert_feeds_uris()
>
>>
>>       def _handle_intercept_failure(self, registered_pkgs):
>>           rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
>> @@ -433,7 +436,10 @@ class DpkgRootfs(Rootfs):
>>           pass
>>
>>       def _insert_feed_uris(self):
>> -        pass
>> +        if base_contains("IMAGE_FEATURES", "package-management",
>> +                         False, True, self.d):
>> +            return
>> +        self.pm.insert_feeds_uris()
> same here.
>
>>
>>
>>   class OpkgRootfs(Rootfs):
>> @@ -699,8 +705,11 @@ class OpkgRootfs(Rootfs):
>>           pass
>>
>>       def _insert_feed_uris(self):
>> -        pass
>> +        if base_contains("IMAGE_FEATURES", "package-management",
>> +                         False, True, self.d):
>> +            return
>>
>> +        self.pm.insert_feeds_uris()
> same here.
>
>>
>>   def create_rootfs(d, manifest_dir=None):
>>       env_bkp = os.environ.copy()
>> --
>> 1.8.3.2
>>
>> _______________________________________________
>> Openembedded-core mailing list
>> Openembedded-core@lists.openembedded.org
>> http://lists.openembedded.org/mailman/listinfo/openembedded-core

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[Laurentiu Palcu]

On Thu, Feb 27, 2014 at 03:35:50PM +0100, David Nyström wrote:
> On 2014-02-27 15:24, Laurentiu Palcu wrote:
> >Hi David,
> >
> >My comments/questions below.
> 
> Thank you for your detailed comments on the RFC.
> I will return with V2, where your comments are addressed when the
> general approach is Acked.
> 
> What do you think about the general approach ?
I think the approach is ok. I don't know a simpler way than this to add
package feeds into the image.

> I guess it was you who added the insert_feeds_uris stubs in rootfs.py ?
Yes, I added the stubs in preparation for this. But I didn't know
when/if this will be done. Thanks for taking care of this.

laurentiu

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[David Nyström]

On 2014-02-27 16:18, Laurentiu Palcu wrote:
> On Thu, Feb 27, 2014 at 03:35:50PM +0100, David Nyström wrote:
>> On 2014-02-27 15:24, Laurentiu Palcu wrote:
>>> Hi David,
>>>
>>> My comments/questions below.
>> Thank you for your detailed comments on the RFC.
>> I will return with V2, where your comments are addressed when the
>> general approach is Acked.
>>
>> What do you think about the general approach ?
> I think the approach is ok. I don't know a simpler way than this to add
> package feeds into the image.

OK, there seems to be no screaming objections from other parties either.

I'll resend v2

Br,
David

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[Otavio Salvador]

[-- Attachment #1: Type: text/plain, Size: 1349 bytes --]

Hello David,

On Thu, Feb 27, 2014 at 4:59 PM, David Nyström <david.c.nystrom@gmail.com>wrote:

>
> On 2014-02-27 16:18, Laurentiu Palcu wrote:
>
>> On Thu, Feb 27, 2014 at 03:35:50PM +0100, David Nyström wrote:
>>
>>> On 2014-02-27 15:24, Laurentiu Palcu wrote:
>>>
>>>> Hi David,
>>>>
>>>> My comments/questions below.
>>>>
>>> Thank you for your detailed comments on the RFC.
>>> I will return with V2, where your comments are addressed when the
>>> general approach is Acked.
>>>
>>> What do you think about the general approach ?
>>>
>> I think the approach is ok. I don't know a simpler way than this to add
>> package feeds into the image.
>>
>
> OK, there seems to be no screaming objections from other parties either.
>
> I'll resend v2
>

I understand why you are adding this here but I think it'd be good to the
settings to be put in a package so in a product we may add extra
repositories updating the package.

I am trying to think now how this could be done (or optionally done).

What you think?

Other point, PACKAGE_FEED_URIS could be change to PACKAGE_FEED_URI as it
matches SRC_URI for example.

-- 
Otavio Salvador                             O.S. Systems
http://www.ossystems.com.br        http://code.ossystems.com.br
Mobile: +55 (53) 9981-7854            Mobile: +1 (347) 903-9750

[-- Attachment #2: Type: text/html, Size: 2742 bytes --]

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[Paul Barker]

On 1 March 2014 18:10, Otavio Salvador <otavio@ossystems.com.br> wrote:
> Hello David,
>
> On Thu, Feb 27, 2014 at 4:59 PM, David Nyström <david.c.nystrom@gmail.com>
> wrote:
>>
>>
>> On 2014-02-27 16:18, Laurentiu Palcu wrote:
>>>
>>> On Thu, Feb 27, 2014 at 03:35:50PM +0100, David Nyström wrote:
>>>>
>>>> On 2014-02-27 15:24, Laurentiu Palcu wrote:
>>>>>
>>>>> Hi David,
>>>>>
>>>>> My comments/questions below.
>>>>
>>>> Thank you for your detailed comments on the RFC.
>>>> I will return with V2, where your comments are addressed when the
>>>> general approach is Acked.
>>>>
>>>> What do you think about the general approach ?
>>>
>>> I think the approach is ok. I don't know a simpler way than this to add
>>> package feeds into the image.
>>
>>
>> OK, there seems to be no screaming objections from other parties either.
>>
>> I'll resend v2
>
>
> I understand why you are adding this here but I think it'd be good to the
> settings to be put in a package so in a product we may add extra
> repositories updating the package.
>
> I am trying to think now how this could be done (or optionally done).
>
> What you think?

+1

I'd rather have '/etc/opkg/base-feeds.conf' (for example) generated
and placed into a package which is then installed on the rootfs. That
would make it easy to distribute an update to that in the future. I'm
not sure how difficult/easy that would be.

>
> Other point, PACKAGE_FEED_URIS could be change to PACKAGE_FEED_URI as it
> matches SRC_URI for example.
>

+1

Thanks,

-- 
Paul Barker

Email: paul@paulbarker.me.uk
http://www.paulbarker.me.uk

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[Paul Eggleton]

On Saturday 01 March 2014 18:21:59 Paul Barker wrote:
> On 1 March 2014 18:10, Otavio Salvador <otavio@ossystems.com.br> wrote:
> > Hello David,
> > 
> > On Thu, Feb 27, 2014 at 4:59 PM, David Nyström <david.c.nystrom@gmail.com>
> > 
> > wrote:
> >> On 2014-02-27 16:18, Laurentiu Palcu wrote:
> >>> On Thu, Feb 27, 2014 at 03:35:50PM +0100, David Nyström wrote:
> >>>> On 2014-02-27 15:24, Laurentiu Palcu wrote:
> >>>>> Hi David,
> >>>>> 
> >>>>> My comments/questions below.
> >>>> 
> >>>> Thank you for your detailed comments on the RFC.
> >>>> I will return with V2, where your comments are addressed when the
> >>>> general approach is Acked.
> >>>> 
> >>>> What do you think about the general approach ?
> >>> 
> >>> I think the approach is ok. I don't know a simpler way than this to add
> >>> package feeds into the image.
> >> 
> >> OK, there seems to be no screaming objections from other parties either.
> >> 
> >> I'll resend v2
> > 
> > I understand why you are adding this here but I think it'd be good to the
> > settings to be put in a package so in a product we may add extra
> > repositories updating the package.
> > 
> > I am trying to think now how this could be done (or optionally done).
> > 
> > What you think?
> 
> +1
> 
> I'd rather have '/etc/opkg/base-feeds.conf' (for example) generated
> and placed into a package which is then installed on the rootfs. That
> would make it easy to distribute an update to that in the future. I'm
> not sure how difficult/easy that would be.

The only problem with this is you then can't have your feeds different per 
image.

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[David Nyström]

On 2014-03-01 19:10, Otavio Salvador wrote:
> Hello David,
>
> On Thu, Feb 27, 2014 at 4:59 PM, David Nyström <david.c.nystrom@gmail.com>wrote:
>
>>
>> On 2014-02-27 16:18, Laurentiu Palcu wrote:
>>
>>> On Thu, Feb 27, 2014 at 03:35:50PM +0100, David Nyström wrote:
>>>
>>>> On 2014-02-27 15:24, Laurentiu Palcu wrote:
>>>>
>>>>> Hi David,
>>>>>
>>>>> My comments/questions below.
>>>>>
>>>> Thank you for your detailed comments on the RFC.
>>>> I will return with V2, where your comments are addressed when the
>>>> general approach is Acked.
>>>>
>>>> What do you think about the general approach ?
>>>>
>>> I think the approach is ok. I don't know a simpler way than this to add
>>> package feeds into the image.
>>>
>>
>> OK, there seems to be no screaming objections from other parties either.
>>
>> I'll resend v2
>>
>
> I understand why you are adding this here but I think it'd be good to the
> settings to be put in a package so in a product we may add extra
> repositories updating the package.
>

Good point.

> I am trying to think now how this could be done (or optionally done).
>
> What you think?

Yes, how would we determine the repo subdirs for a specific machine 
without too much
anonymous python magic in a recipe, considering multilib setups?

Example for qemux86-64:
-- 
all
x86_64 = ${TARGET_ARCH}
core2-64 = ${??}
qemu86_64 = ${MACHINE}
-- 

If we can get a sane way to retrieve them, modifying the repo feeds via 
install/postinstall should be the easy part.


> Other point, PACKAGE_FEED_URIS could be change to PACKAGE_FEED_URI as it
> matches SRC_URI for example.

Agreed.

Re: [RFC][PATCH] Attempt to unify predefined package feed handling

[David Nyström]

On 2014-02-27 15:04, David Nyström wrote:
> local.conf defines
> PACKAGE_FEED_URIS = "http://www.feed_repo.org/repo/"
>
> should we do it this way or some other way ?
>
> [Bug 5407]
>
>
>

Oops,
Duplicate, please comment on the other thread.

[RFC][PATCH] Attempt to unify predefined package feed handling

[David Nyström]

local.conf defines 
PACKAGE_FEED_URIS = "http://www.feed_repo.org/repo/"

should we do it this way or some other way ?

[Bug 5407]



Signed-off-by: David Nyström <david.nystrom@enea.com>
---
 meta/lib/oe/package_manager.py | 76 +++++++++++++++++++++++++++++++++++++++++-
 meta/lib/oe/rootfs.py          | 15 +++++++--
 2 files changed, 87 insertions(+), 4 deletions(-)

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index d29adac..d4050c6 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -223,6 +223,7 @@ class PackageManager(object):
         self.d = d
         self.deploy_dir = None
         self.deploy_lock = None
+        self.feed_uris = None
 
     """
     Update the package manager package database.
@@ -262,6 +263,10 @@ class PackageManager(object):
     def list_installed(self, format=None):
         pass
 
+    @abstractmethod
+    def insert_feeds_uris(self):
+        pass
+
     """
     Install complementary packages based upon the list of currently installed
     packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
@@ -339,6 +344,7 @@ class RpmPM(PackageManager):
         self.providename = providename
         self.fullpkglist = list()
         self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM', True)
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
         self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm")
         self.install_dir = os.path.join(self.target_rootfs, "install")
         self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm")
@@ -358,6 +364,41 @@ class RpmPM(PackageManager):
 
         self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
 
+
+    def insert_feeds_uris(self):
+        # List must be prefered to least preferred order
+        channel_priority = 5
+        default_platform_extra = set()
+        platform_extra = set()
+        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
+        for mlib in self.ml_os_list:
+            for arch in self.ml_prefix_list[mlib]:
+                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
+                if mlib == bbextendvariant:
+                        default_platform_extra.add(plt)
+                else:
+                        platform_extra.add(plt)
+                channel_priority += 5
+
+        platform_extra = platform_extra.union(default_platform_extra)
+
+        for uri in self.feed_uris.split():
+            channel_priority *= 2
+            uri_iterator = 0
+            for canonical_arch in platform_extra:
+                arch = canonical_arch.split('-')[0]
+                if arch == "noarch" or arch == "any":
+                    continue
+
+                bb.note('Note: adding Smart channel url%d%s (%s)' %
+                        (uri_iterator, arch, channel_priority))
+                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
+                                   % (uri_iterator, arch, uri, arch))
+                self._invoke_smart('channel --set url%d-%s priority=%d' %
+                                   (uri_iterator, arch, channel_priority))
+                channel_priority -= 5
+            uri_iterator += 1
+
     '''
     Create configs for rpm and smart, and multilib is supported
     '''
@@ -944,7 +985,7 @@ class OpkgPM(PackageManager):
 
         self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
-
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
         self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
         self.opkg_args += self.d.getVar("OPKG_ARGS", True)
@@ -1050,6 +1091,24 @@ class OpkgPM(PackageManager):
                     config_file.write("src oe-%s file:%s\n" %
                                       (arch, pkgs_dir))
 
+    def insert_feeds_uris(self):
+        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
+                                  % self.target_rootfs)
+
+        with open(rootfs_config, "w+") as config_file:
+            priority = 5
+            uri_iterator = 0
+            for uri in self.feed_uris.split():
+                config_file.write("src/gz url-%d %s/ipk\n" %
+                                  (uri_iterator, uri))
+
+                for arch in self.pkg_archs.split():
+                    if arch == "noarch" or arch == "any":
+                        continue
+                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
+                                      (arch, uri_iterator, uri, arch))
+                uri_iterator += 1
+
     def update(self):
         self.deploy_dir_lock()
 
@@ -1250,6 +1309,7 @@ class DpkgPM(PackageManager):
     def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
         super(DpkgPM, self).__init__(d)
         self.target_rootfs = target_rootfs
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True)
         self.deploy_dir = self.d.getVar('DEPLOY_DIR_DEB', True)
         if apt_conf_dir is None:
             self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt")
@@ -1410,6 +1470,20 @@ class DpkgPM(PackageManager):
         if result is not None:
             bb.fatal(result)
 
+    def insert_feeds_uris(self):
+        sources_conf = os.path.join("%s/etc/apt/sources.list"
+                                    % self.target_rootfs)
+        arch_list = []
+        archs = self.d.getVar('PACKAGE_ARCHS', True)
+        arch_list = archs.split()
+        with open(sources_conf, "w+") as sources_file:
+            for uri in self.feed_uris.split():
+                for arch in arch_list:
+                    if arch == "noarch" or arch == "any":
+                        continue
+                    sources_file.write("deb %s/deb/%s ./\n" %
+                                       (uri, arch))
+
     def _create_configs(self, archs, base_archs):
         base_archs = re.sub("_", "-", base_archs)
 
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index be0afa6..7be2a18 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -350,7 +350,10 @@ class RpmRootfs(Rootfs):
                     bb.fatal(message)
 
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         False, True, self.d):
+            return
+        self.pm.insert_feeds_uris()
 
     def _handle_intercept_failure(self, registered_pkgs):
         rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
@@ -433,7 +436,10 @@ class DpkgRootfs(Rootfs):
         pass
 
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         False, True, self.d):
+            return
+        self.pm.insert_feeds_uris()
 
 
 class OpkgRootfs(Rootfs):
@@ -699,8 +705,11 @@ class OpkgRootfs(Rootfs):
         pass
 
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         False, True, self.d):
+            return
 
+        self.pm.insert_feeds_uris()
 
 def create_rootfs(d, manifest_dir=None):
     env_bkp = os.environ.copy()
-- 
1.8.3.2