[layerindex-web][PATCH 0/2] Implement CSV export for layer recipes

[layerindex-web][PATCH 0/2] Implement CSV export for layer recipes

[Paul Eggleton]

Ross asked for the ability to export a layer's recipes in CSV format,
and here it is - as well as cleanup of some old cruft that was supposed
to do a similar thing but wasn't finished.


The following changes since commit 837359e6f18aac985c2967344d035d70c05e427a:

  detail: Fix layer description not wrapping properly (2018-05-04 23:52:51 +1200)

are available in the Git repository at:

  git://git.yoctoproject.org/layerindex-web paule/recipe-csv-export
  http://git.yoctoproject.org/cgit.cgi//log/?h=paule/recipe-csv-export

Paul Eggleton (2):
  Drop old raw recipe export code
  Add CSV export for layer recipes

 layerindex/urls.py                  |  8 +-------
 layerindex/urls_branch.py           |  5 ++++-
 layerindex/views.py                 | 29 ++++++++++++++++++++++-------
 templates/layerindex/detail.html    | 14 +++++++-------
 templates/layerindex/rawrecipes.txt |  2 --
 5 files changed, 34 insertions(+), 24 deletions(-)
 delete mode 100644 templates/layerindex/rawrecipes.txt

-- 
2.14.3

[layerindex-web][PATCH 1/2] Drop old raw recipe export code

[Paul Eggleton]

We're about to replace this with a proper CSV export function, so we
don't need this dead code hanging around anymore.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/urls.py                  | 8 +-------
 layerindex/urls_branch.py           | 2 +-
 layerindex/views.py                 | 7 -------
 templates/layerindex/rawrecipes.txt | 2 --
 4 files changed, 2 insertions(+), 17 deletions(-)
 delete mode 100644 templates/layerindex/rawrecipes.txt

diff --git a/layerindex/urls.py b/layerindex/urls.py
index b0b74314..f80606fc 100644
--- a/layerindex/urls.py
+++ b/layerindex/urls.py
@@ -8,7 +8,7 @@ from django.conf.urls import *
 from django.views.generic import TemplateView, DetailView, ListView, RedirectView
 from django.views.defaults import page_not_found
 from django.core.urlresolvers import reverse_lazy
-from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view
+from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view
 from layerindex.models import LayerItem, Recipe, RecipeChangeset
 from rest_framework import routers
 from . import restviews
@@ -101,12 +101,6 @@ urlpatterns = [
             name="bulk_change_delete"),
     url(r'^branch/(?P<branch>[-\w]+)/',
         include('layerindex.urls_branch')),
-    #url(r'^raw/recipes.txt$',
-    #    PlainTextListView.as_view(
-    #        queryset=Recipe.objects.order_by('pn', 'layerbranch__layer'),
-    #        context_object_name='recipe_list',
-    #        template_name='layerindex/rawrecipes.txt'),
-    #        name='recipe_list_raw'),
     url(r'^updates/$',
         UpdateListView.as_view(
             template_name='layerindex/updatelist.html'),
diff --git a/layerindex/urls_branch.py b/layerindex/urls_branch.py
index a71af1ce..0e41435e 100644
--- a/layerindex/urls_branch.py
+++ b/layerindex/urls_branch.py
@@ -7,7 +7,7 @@
 from django.conf.urls import *
 from django.views.defaults import page_not_found
 from django.core.urlresolvers import reverse_lazy
-from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView
+from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView
 
 urlpatterns = [
     url(r'^$', 
diff --git a/layerindex/views.py b/layerindex/views.py
index bc3cddfa..dbf08497 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -771,13 +771,6 @@ class ClassSearchView(ListView):
         context['this_url_name'] = resolve(self.request.path_info).url_name
         return context
 
-class PlainTextListView(ListView):
-    def render_to_response(self, context):
-        "Returns a plain text response rendering of the template"
-        template = get_template(self.template_name)
-        return HttpResponse(template.render(context),
-                                 content_type='text/plain')
-
 class HistoryListView(ListView):
     context_object_name = "revisions"
     paginate_by = 50
diff --git a/templates/layerindex/rawrecipes.txt b/templates/layerindex/rawrecipes.txt
deleted file mode 100644
index c1911c9c..00000000
--- a/templates/layerindex/rawrecipes.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-{% for recipe in recipe_list %}{{ recipe.layerbranch.layer.name }}	{{ recipe.pn }}	{{ recipe.pv }}	{{ recipe.full_path }}
-{% endfor %}
-- 
2.14.3

[layerindex-web][PATCH 2/2] Add CSV export for layer recipes

[Paul Eggleton]

Add the ability to export the recipe listing for a layer to a CSV file
for importing into external tools. At the moment we include name,
version and license, but there is a parameter that lets you specify the
fields to include in the URL if desired.

Implements [YOCTO #12722].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
---
 layerindex/urls_branch.py        |  5 ++++-
 layerindex/views.py              | 22 ++++++++++++++++++++++
 templates/layerindex/detail.html | 14 +++++++-------
 3 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/layerindex/urls_branch.py b/layerindex/urls_branch.py
index 0e41435e..2809147b 100644
--- a/layerindex/urls_branch.py
+++ b/layerindex/urls_branch.py
@@ -7,7 +7,7 @@
 from django.conf.urls import *
 from django.views.defaults import page_not_found
 from django.core.urlresolvers import reverse_lazy
-from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView
+from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view
 
 urlpatterns = [
     url(r'^$', 
@@ -20,6 +20,9 @@ urlpatterns = [
         LayerDetailView.as_view(
             template_name='layerindex/detail.html'),
             name='layer_item'),
+    url(r'^layer/(?P<slug>[-\w]+)/recipes/csv/$',
+        layer_export_recipes_csv_view,
+        name='layer_export_recipes_csv'),
     url(r'^recipes/$',
         RecipeSearchView.as_view(
             template_name='layerindex/recipes.html'),
diff --git a/layerindex/views.py b/layerindex/views.py
index dbf08497..06d35261 100644
--- a/layerindex/views.py
+++ b/layerindex/views.py
@@ -1022,3 +1022,25 @@ class StatsView(TemplateView):
                 machine_count=Count('layerbranch__machine', distinct=True),
                 distro_count=Count('layerbranch__distro', distinct=True))
         return context
+
+
+def layer_export_recipes_csv_view(request, branch, slug):
+    import csv
+    layer = get_object_or_404(LayerItem, name=slug)
+    layerbranch = layer.get_layerbranch(branch)
+
+    response = HttpResponse(content_type='text/csv')
+    response['Content-Disposition'] = 'attachment; filename="recipes_%s_%s.csv"' % (layer.name, layerbranch.branch.name)
+
+    fieldlist = request.GET.get('fields', 'pn,pv,license').split(',')
+    recipe_fields = [f.name for f in Recipe._meta.get_fields() if not (f.auto_created and f.is_relation)]
+    for field in fieldlist:
+        if field not in recipe_fields:
+            return HttpResponse('Field %s is invalid' % field)
+
+    writer = csv.writer(response)
+    for recipe in layerbranch.sorted_recipes():
+        values = [getattr(recipe, field) for field in fieldlist]
+        writer.writerow(values)
+
+    return response
diff --git a/templates/layerindex/detail.html b/templates/layerindex/detail.html
index 220d475b..67c21126 100644
--- a/templates/layerindex/detail.html
+++ b/templates/layerindex/detail.html
@@ -199,13 +199,13 @@
                         <div class="navbar-inner">
                             <a class="brand pull-left">{{ layeritem.name }} recipes <span class="muted">({{ layerbranch.sorted_recipes.count }})</span></a>
 
-                            <ul class="nav pull-right">
-                                <li>
-                                    <form action="" class="navbar-search pull-right" id="filter-form">
-                                        <input type="text" placeholder="Search recipes" class="search-query" id="filter">
-                                    </form>
-                                </li>
-                            </ul>
+                            <div class="pull-right">
+                                <a href="{% url 'layer_export_recipes_csv' layerbranch.branch.name layerbranch.layer.name %}" class="btn"><i class="icon-file"></i> Export CSV</a>
+
+                                <form action="" class="navbar-search nav-spacer pull-right" id="filter-form">
+                                    <input type="text" placeholder="Search recipes" class="search-query" id="filter">
+                                </form>
+                            </div>
                         </div>
                     </div>
 
-- 
2.14.3

Re: [layerindex-web][PATCH 2/2] Add CSV export for layer recipes

[Mark Hatle]

On 5/6/18 10:35 PM, Paul Eggleton wrote:
> Add the ability to export the recipe listing for a layer to a CSV file
> for importing into external tools. At the moment we include name,
> version and license, but there is a parameter that lets you specify the
> fields to include in the URL if desired.
> 
> Implements [YOCTO #12722].

With the bitbake layer library I'm working on, it should be possible to do this
directly out of bitbake.

This wouldn't limit it to just recipes either, but any data.  Loading (querying)
the json data from the layer index is easy, and the library would know how to
filter to the right branch/layer/etc.

Something to consider as we go forward with this and other work.

--Mark

> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
> ---
>  layerindex/urls_branch.py        |  5 ++++-
>  layerindex/views.py              | 22 ++++++++++++++++++++++
>  templates/layerindex/detail.html | 14 +++++++-------
>  3 files changed, 33 insertions(+), 8 deletions(-)
> 
> diff --git a/layerindex/urls_branch.py b/layerindex/urls_branch.py
> index 0e41435e..2809147b 100644
> --- a/layerindex/urls_branch.py
> +++ b/layerindex/urls_branch.py
> @@ -7,7 +7,7 @@
>  from django.conf.urls import *
>  from django.views.defaults import page_not_found
>  from django.core.urlresolvers import reverse_lazy
> -from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView
> +from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view
>  
>  urlpatterns = [
>      url(r'^$', 
> @@ -20,6 +20,9 @@ urlpatterns = [
>          LayerDetailView.as_view(
>              template_name='layerindex/detail.html'),
>              name='layer_item'),
> +    url(r'^layer/(?P<slug>[-\w]+)/recipes/csv/$',
> +        layer_export_recipes_csv_view,
> +        name='layer_export_recipes_csv'),
>      url(r'^recipes/$',
>          RecipeSearchView.as_view(
>              template_name='layerindex/recipes.html'),
> diff --git a/layerindex/views.py b/layerindex/views.py
> index dbf08497..06d35261 100644
> --- a/layerindex/views.py
> +++ b/layerindex/views.py
> @@ -1022,3 +1022,25 @@ class StatsView(TemplateView):
>                  machine_count=Count('layerbranch__machine', distinct=True),
>                  distro_count=Count('layerbranch__distro', distinct=True))
>          return context
> +
> +
> +def layer_export_recipes_csv_view(request, branch, slug):
> +    import csv
> +    layer = get_object_or_404(LayerItem, name=slug)
> +    layerbranch = layer.get_layerbranch(branch)
> +
> +    response = HttpResponse(content_type='text/csv')
> +    response['Content-Disposition'] = 'attachment; filename="recipes_%s_%s.csv"' % (layer.name, layerbranch.branch.name)
> +
> +    fieldlist = request.GET.get('fields', 'pn,pv,license').split(',')
> +    recipe_fields = [f.name for f in Recipe._meta.get_fields() if not (f.auto_created and f.is_relation)]
> +    for field in fieldlist:
> +        if field not in recipe_fields:
> +            return HttpResponse('Field %s is invalid' % field)
> +
> +    writer = csv.writer(response)
> +    for recipe in layerbranch.sorted_recipes():
> +        values = [getattr(recipe, field) for field in fieldlist]
> +        writer.writerow(values)
> +
> +    return response
> diff --git a/templates/layerindex/detail.html b/templates/layerindex/detail.html
> index 220d475b..67c21126 100644
> --- a/templates/layerindex/detail.html
> +++ b/templates/layerindex/detail.html
> @@ -199,13 +199,13 @@
>                          <div class="navbar-inner">
>                              <a class="brand pull-left">{{ layeritem.name }} recipes <span class="muted">({{ layerbranch.sorted_recipes.count }})</span></a>
>  
> -                            <ul class="nav pull-right">
> -                                <li>
> -                                    <form action="" class="navbar-search pull-right" id="filter-form">
> -                                        <input type="text" placeholder="Search recipes" class="search-query" id="filter">
> -                                    </form>
> -                                </li>
> -                            </ul>
> +                            <div class="pull-right">
> +                                <a href="{% url 'layer_export_recipes_csv' layerbranch.branch.name layerbranch.layer.name %}" class="btn"><i class="icon-file"></i> Export CSV</a>
> +
> +                                <form action="" class="navbar-search nav-spacer pull-right" id="filter-form">
> +                                    <input type="text" placeholder="Search recipes" class="search-query" id="filter">
> +                                </form>
> +                            </div>
>                          </div>
>                      </div>
>  
>