[layerindex-web][PATCH 1/3] utils.py: split out parse_conf() from parse_layer_conf()

[layerindex-web][PATCH 1/3] utils.py: split out parse_conf() from parse_layer_conf()

[Paul Eggleton]

If we want to parse a configuration file (e.g. a distro conf file) then
we need convenient access to bitbake's conf parsing code, so create a
parse_conf() function to provide that.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/utils.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/layerindex/utils.py b/layerindex/utils.py
index 50734fc..3b45b07 100644
--- a/layerindex/utils.py
+++ b/layerindex/utils.py
@@ -160,6 +160,15 @@ def is_layer_valid(layerdir):
         return False
     return True
 
+def parse_conf(conf_file, d):
+    if hasattr(bb.parse, "handle"):
+        # Newer BitBake
+        data = bb.parse.handle(conf_file, d, include=True)
+    else:
+        # Older BitBake (1.18 and below)
+        data = bb.cooker._parse(conf_file, d)
+    return data
+
 def parse_layer_conf(layerdir, data, logger=None):
     conf_file = os.path.join(layerdir, "conf", "layer.conf")
 
@@ -169,12 +178,7 @@ def parse_layer_conf(layerdir, data, logger=None):
         return
 
     data.setVar('LAYERDIR', str(layerdir))
-    if hasattr(bb, "cookerdata"):
-        # Newer BitBake
-        data = bb.cookerdata.parse_config_file(conf_file, data)
-    else:
-        # Older BitBake (1.18 and below)
-        data = bb.cooker._parse(conf_file, data)
+    data = parse_conf(conf_file, data)
     data.expandVarref('LAYERDIR')
 
 def runcmd(cmd, destdir=None, printerr=True, logger=None):
-- 
2.9.3

[layerindex-web][PATCH 2/3] update_layer.py: use DISTRO_NAME as primary distro short description

[Paul Eggleton]

When reading conf/distro/*.conf to create distro records, attempt to
parse the config file and use DISTRO_NAME (if set) to populate the
description field for the record. If that's not set then fall back to
the less commonly used meta-comment that we supported previously.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/update_layer.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py
index 705227b..9fe96fb 100644
--- a/layerindex/update_layer.py
+++ b/layerindex/update_layer.py
@@ -133,7 +133,7 @@ def update_machine_conf_file(path, machine):
                 break
     machine.description = desc
 
-def update_distro_conf_file(path, distro):
+def update_distro_conf_file(path, distro, d):
     logger.debug('Updating distro %s' % path)
     desc = ""
     with open(path, 'r') as f:
@@ -144,7 +144,18 @@ def update_distro_conf_file(path, distro):
                 desc = line[14:].strip()
                 desc = re.sub(r'Distribution configuration for( running)*( an)*( the)*', '', desc)
                 break
-    distro.description = desc
+
+    distro_name = ''
+    try:
+        d = utils.parse_conf(path, d)
+        distro_name = d.getVar('DISTRO_NAME', True)
+    except Exception as e:
+        logger.warn('Error parsing distro configuration file %s: %s' % (path, str(e)))
+
+    if distro_name:
+        distro.description = distro_name
+    else:
+        distro.description = desc
 
 def main():
     if LooseVersion(git.__version__) < '0.3.1':
@@ -531,7 +542,7 @@ def main():
                                 distro = Distro()
                                 distro.layerbranch = layerbranch
                                 distro.name = filename
-                                update_distro_conf_file(os.path.join(repodir, path), distro)
+                                update_distro_conf_file(os.path.join(repodir, path), distro, config_data_copy)
                                 distro.save()
                             elif typename == 'bbclass':
                                 bbclass = BBClass()
@@ -568,7 +579,7 @@ def main():
                                 results = layerdistros.filter(name=filename)
                                 if results:
                                     distro = results[0]
-                                    update_distro_conf_file(os.path.join(repodir, path), distro)
+                                    update_distro_conf_file(os.path.join(repodir, path), distro, config_data_copy)
                                     distro.save()
 
                             deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
@@ -642,7 +653,7 @@ def main():
                                 distro = Distro()
                                 distro.layerbranch = layerbranch
                                 distro.name = filename
-                                update_distro_conf_file(fullpath, distro)
+                                update_distro_conf_file(fullpath, distro, config_data_copy)
                                 distro.save()
                             elif typename == 'bbclass':
                                 bbclass = BBClass()
-- 
2.9.3

[layerindex-web][PATCH 3/3] layerindex: add "No update" status to LayerItem

[Paul Eggleton]

Add a status for a layer indicating it should not be updated. I don't
expect this to be widely used (and is only settable from the admin
interface) but would be useful if you have a legacy sub-layer that you
want to prevent from being visible on certain branches - it will prevent
the update script from doing anything with the layer and thus avoid
branch records from being auto-created on branches where you've deleted
it.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/forms.py                                   |  2 +-
 .../migrations/0007_layeritem_status_noupdate.py      | 19 +++++++++++++++++++
 layerindex/models.py                                  |  1 +
 layerindex/restviews.py                               |  6 +++---
 layerindex/update.py                                  |  1 +
 layerindex/views.py                                   |  4 ++--
 6 files changed, 27 insertions(+), 6 deletions(-)
 create mode 100644 layerindex/migrations/0007_layeritem_status_noupdate.py

diff --git a/layerindex/forms.py b/layerindex/forms.py
index 0e7902d..26e676a 100644
--- a/layerindex/forms.py
+++ b/layerindex/forms.py
@@ -179,7 +179,7 @@ class AdvancedRecipeSearchForm(forms.Form):
     field = forms.ChoiceField(choices=FIELD_CHOICES)
     match_type = forms.ChoiceField(choices=MATCH_TYPE_CHOICES)
     value = forms.CharField(max_length=255, required=False)
-    layer = forms.ModelChoiceField(queryset=LayerItem.objects.filter(classic=False).filter(status='P').order_by('name'), empty_label="(any)", required=False)
+    layer = forms.ModelChoiceField(queryset=LayerItem.objects.filter(classic=False).filter(status__in=['P', 'X']).order_by('name'), empty_label="(any)", required=False)
 
 
 class RecipeChangesetForm(forms.ModelForm):
diff --git a/layerindex/migrations/0007_layeritem_status_noupdate.py b/layerindex/migrations/0007_layeritem_status_noupdate.py
new file mode 100644
index 0000000..e917b0e
--- /dev/null
+++ b/layerindex/migrations/0007_layeritem_status_noupdate.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layerindex', '0006_change_branch_meta'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='layeritem',
+            name='status',
+            field=models.CharField(default='N', choices=[('N', 'New'), ('P', 'Published'), ('X', 'No update')], max_length=1),
+        ),
+    ]
diff --git a/layerindex/models.py b/layerindex/models.py
index b60ad01..fb2e026 100644
--- a/layerindex/models.py
+++ b/layerindex/models.py
@@ -65,6 +65,7 @@ class LayerItem(models.Model):
     LAYER_STATUS_CHOICES = (
         ('N', 'New'),
         ('P', 'Published'),
+        ('X', 'No update'),
     )
     LAYER_TYPE_CHOICES = (
         ('A', 'Base'),
diff --git a/layerindex/restviews.py b/layerindex/restviews.py
index b0c8a7a..d947bbf 100644
--- a/layerindex/restviews.py
+++ b/layerindex/restviews.py
@@ -22,7 +22,7 @@ class LayerItemSerializer(serializers.ModelSerializer):
         model = LayerItem
 
 class LayerItemViewSet(ParametricSearchableModelViewSet):
-    queryset = LayerItem.objects.filter(status='P')
+    queryset = LayerItem.objects.filter(status__in=['P', 'X'])
     serializer_class = LayerItemSerializer
 
 class LayerBranchSerializer(serializers.ModelSerializer):
@@ -30,7 +30,7 @@ class LayerBranchSerializer(serializers.ModelSerializer):
         model = LayerBranch
 
 class LayerBranchViewSet(ParametricSearchableModelViewSet):
-    queryset = LayerBranch.objects.filter(layer__status='P')
+    queryset = LayerBranch.objects.filter(layer__status__in=['P', 'X'])
     serializer_class = LayerBranchSerializer
 
 class LayerDependencySerializer(serializers.ModelSerializer):
@@ -38,7 +38,7 @@ class LayerDependencySerializer(serializers.ModelSerializer):
         model = LayerDependency
 
 class LayerDependencyViewSet(ParametricSearchableModelViewSet):
-    queryset = LayerDependency.objects.filter(layerbranch__layer__status='P')
+    queryset = LayerDependency.objects.filter(layerbranch__layer__status__in=['P', 'X'])
     serializer_class = LayerDependencySerializer
 
 class RecipeSerializer(serializers.ModelSerializer):
diff --git a/layerindex/update.py b/layerindex/update.py
index e5987bb..d5c56cd 100755
--- a/layerindex/update.py
+++ b/layerindex/update.py
@@ -157,6 +157,7 @@ def main():
             logger.error('No layers matching specified query "%s"' % options.layers)
             sys.exit(1)
     else:
+        # We deliberately exclude status == 'X' ("no update") here
         layerquery = LayerItem.objects.filter(classic=False).filter(status='P')
         if layerquery.count() == 0:
             logger.info("No published layers to update")
diff --git a/layerindex/views.py b/layerindex/views.py
index 311a86b..65a536a 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -279,7 +279,7 @@ class LayerListView(ListView):
 
     def get_queryset(self):
         _check_url_branch(self.kwargs)
-        return LayerBranch.objects.filter(branch__name=self.kwargs['branch']).filter(layer__status='P').order_by('layer__layer_type', '-layer__index_preference', 'layer__name')
+        return LayerBranch.objects.filter(branch__name=self.kwargs['branch']).filter(layer__status__in=['P', 'X']).order_by('layer__layer_type', '-layer__index_preference', 'layer__name')
 
     def get_context_data(self, **kwargs):
         context = super(LayerListView, self).get_context_data(**kwargs)
@@ -459,7 +459,7 @@ class DuplicatesView(TemplateView):
         context['classes'] = self.get_classes(layer_ids)
         context['url_branch'] = self.kwargs['branch']
         context['this_url_name'] = resolve(self.request.path_info).url_name
-        context['layers'] = LayerBranch.objects.filter(branch__name=self.kwargs['branch']).filter(layer__status='P').order_by( 'layer__name')
+        context['layers'] = LayerBranch.objects.filter(branch__name=self.kwargs['branch']).filter(layer__status__in=['P', 'X']).order_by( 'layer__name')
         context['showlayers'] = layer_ids
         return context
 
-- 
2.9.3