* [RFC] WIP unidentified layer / custom images and non git layers
@ 2016-10-07 16:47 Michael Wood
2016-10-07 16:47 ` [PATCH] wip fix unidentified layer Michael Wood
0 siblings, 1 reply; 3+ messages in thread
From: Michael Wood @ 2016-10-07 16:47 UTC (permalink / raw)
To: toaster
I've been trying to get the buildinfohelper and localhostbecontrollert to behave correctly when it has both non git layers and toaster-custom-image layers.
There seems to be a number of problems that this causes with the unidentified layer showing up being one sympton. I think I've found most of the issues but it would be good if this patch could be well tested and to see (if?) other things break too as this is a /very/ fagile bit of the code.
Michael Wood (1):
wip fix unidentified layer
bitbake/lib/bb/ui/buildinfohelper.py | 11 ++----
.../toaster/bldcontrol/localhostbecontroller.py | 44 +++++++++++++++-------
bitbake/lib/toaster/orm/models.py | 3 +-
bitbake/lib/toaster/toastergui/api.py | 4 +-
4 files changed, 39 insertions(+), 23 deletions(-)
--
2.7.4
^ permalink raw reply [flat|nested] 3+ messages in thread
* [PATCH] wip fix unidentified layer
2016-10-07 16:47 [RFC] WIP unidentified layer / custom images and non git layers Michael Wood
@ 2016-10-07 16:47 ` Michael Wood
2016-10-25 5:04 ` sujith h
0 siblings, 1 reply; 3+ messages in thread
From: Michael Wood @ 2016-10-07 16:47 UTC (permalink / raw)
To: toaster
WIP fix unidentified layer and try to stop the buildinfohelper and
localhostbecontroller from getting confused between non layers and the
toaster-custom-images layer
---
bitbake/lib/bb/ui/buildinfohelper.py | 11 ++----
.../toaster/bldcontrol/localhostbecontroller.py | 44 +++++++++++++++-------
bitbake/lib/toaster/orm/models.py | 3 +-
bitbake/lib/toaster/toastergui/api.py | 4 +-
4 files changed, 39 insertions(+), 23 deletions(-)
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 5b69660..82e38ae 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -42,7 +42,7 @@ from orm.models import Variable, VariableHistory
from orm.models import Package, Package_File, Target_Installed_Package, Target_File
from orm.models import Task_Dependency, Package_Dependency
from orm.models import Recipe_Dependency, Provides
-from orm.models import Project, CustomImagePackage, CustomImageRecipe
+from orm.models import Project, CustomImagePackage
from orm.models import signal_runbuilds
from bldcontrol.models import BuildEnvironment, BuildRequest
@@ -361,11 +361,6 @@ class ORMWrapper(object):
def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information):
if isinstance(layer_obj, Layer_Version):
- # Special case the toaster-custom-images layer which is created
- # on the fly so don't update the values which may cause the layer
- # to be duplicated on a future get_or_create
- if layer_obj.layer.name == CustomImageRecipe.LAYER_NAME:
- return layer_obj
# We already found our layer version for this build so just
# update it with the new build information
logger.debug("We found our layer from toaster")
@@ -629,6 +624,8 @@ class ORMWrapper(object):
recipe = self._cached_get(
Recipe,
name=built_recipe.name,
+ layer_version__layer__name=
+ built_recipe.layer_version.layer.name,
layer_version__build=None,
layer_version__release=
built_recipe.layer_version.release,
@@ -637,7 +634,7 @@ class ORMWrapper(object):
)
except (Recipe.DoesNotExist,
Recipe.MultipleObjectsReturned) as e:
- logger.info("We did not find one recipe for the"
+ logger.info("We did not find one recipe for the "
"configuration data package %s %s" % (p, e))
continue
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index a64e89b..50c0b5d 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -198,8 +198,6 @@ class LocalhostBEController(BuildEnvironmentController):
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
- logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
-
# 5. create custom layer and add custom recipes to it
layerpath = os.path.join(self.be.builddir,
CustomImageRecipe.LAYER_NAME)
@@ -222,19 +220,35 @@ class LocalhostBEController(BuildEnvironmentController):
with open(config, "w") as conf:
conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
+ BRLayer.objects.get_or_create(req=target.req,
+ name=layer.name,
+ dirpath=layerpath,
+ giturl="file://%s" % layerpath)
+
# Update the Layer_Version dirpath that has our base_recipe in
# to be able to read the base recipe to then generate the
# custom recipe.
+ logger.info(customrecipe.base_recipe.layer_version)
+
br_layer_base_recipe = layers.get(
layer_version=customrecipe.base_recipe.layer_version)
- br_layer_base_dirpath = \
- os.path.join(self.be.sourcedir,
- self.getGitCloneDirectory(
- br_layer_base_recipe.giturl,
- br_layer_base_recipe.commit),
- customrecipe.base_recipe.layer_version.dirpath
- )
+ # If the layer is one that we've cloned we know where it lives
+ if br_layer_base_recipe.giturl and br_layer_base_recipe.commit:
+ layer_path = self.getGitCloneDirectory(
+ br_layer_base_recipe.giturl,
+ br_layer_base_recipe.commit)
+ # Otherwise it's a local layer
+ elif br_layer_base_recipe.local_source_dir:
+ layer_path = br_layer_base_recipe.local_source_dir
+ else:
+ logger.error("Unable to workout the dir path for the custom"
+ " image recipe")
+
+ br_layer_base_dirpath = os.path.join(
+ self.be.sourcedir,
+ layer_path,
+ customrecipe.base_recipe.layer_version.dirpath)
customrecipe.base_recipe.layer_version.dirpath = \
br_layer_base_dirpath
@@ -249,21 +263,25 @@ class LocalhostBEController(BuildEnvironmentController):
# Update the layer and recipe objects
customrecipe.layer_version.dirpath = layerpath
+ customrecipe.layer_version.layer.local_source_dir = layerpath
+ customrecipe.layer_version.layer.save()
customrecipe.layer_version.save()
customrecipe.file_path = recipe_path
customrecipe.save()
# create *Layer* objects needed for build machinery to work
- BRLayer.objects.get_or_create(req=target.req,
- name=layer.name,
- dirpath=layerpath,
- giturl="file://%s" % layerpath)
+
+
+ nongitlayerlist.append(layerpath)
+
if os.path.isdir(layerpath):
layerlist.append(layerpath)
self.islayerset = True
layerlist.extend(nongitlayerlist)
+
+ logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
return layerlist
def readServerLogFile(self):
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index a7de57c..2c1edc9 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -1631,7 +1631,8 @@ class CustomImageRecipe(Recipe):
if base_recipe_path:
base_recipe = open(base_recipe_path, 'r').read()
else:
- raise IOError("Based on recipe file not found")
+ raise IOError("Based on recipe file not found: %s" %
+ base_recipe_path)
# Add a special case for when the recipe we have based a custom image
# recipe on requires another recipe.
diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/toastergui/api.py
index ae1f150..e4c3b84 100644
--- a/bitbake/lib/toaster/toastergui/api.py
+++ b/bitbake/lib/toaster/toastergui/api.py
@@ -293,8 +293,7 @@ class XhrCustomRecipe(View):
# create layer 'Custom layer' and verion if needed
layer = Layer.objects.get_or_create(
name=CustomImageRecipe.LAYER_NAME,
- summary="Layer for custom recipes",
- vcs_url="file:///toaster_created_layer")[0]
+ summary="Layer for custom recipes")[0]
# Check if we have a layer version already
# We don't use get_or_create here because the dirpath will change
@@ -306,6 +305,7 @@ class XhrCustomRecipe(View):
lver, created = Layer_Version.objects.get_or_create(
project=params['project'],
layer=layer,
+ layer_source=LayerSource.TYPE_LOCAL,
dirpath="toaster_created_layer")
# Add a dependency on our layer to the base recipe's layer
--
2.7.4
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH] wip fix unidentified layer
2016-10-07 16:47 ` [PATCH] wip fix unidentified layer Michael Wood
@ 2016-10-25 5:04 ` sujith h
0 siblings, 0 replies; 3+ messages in thread
From: sujith h @ 2016-10-25 5:04 UTC (permalink / raw)
To: Michael Wood; +Cc: toaster
[-- Attachment #1: Type: text/plain, Size: 9738 bytes --]
Hi,
I have tested this patch and it works for me.
Thanks,
Sujith H
On Fri, Oct 7, 2016 at 10:17 PM, Michael Wood <michael.g.wood@intel.com>
wrote:
> WIP fix unidentified layer and try to stop the buildinfohelper and
> localhostbecontroller from getting confused between non layers and the
> toaster-custom-images layer
> ---
> bitbake/lib/bb/ui/buildinfohelper.py | 11 ++----
> .../toaster/bldcontrol/localhostbecontroller.py | 44
> +++++++++++++++-------
> bitbake/lib/toaster/orm/models.py | 3 +-
> bitbake/lib/toaster/toastergui/api.py | 4 +-
> 4 files changed, 39 insertions(+), 23 deletions(-)
>
> diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/
> buildinfohelper.py
> index 5b69660..82e38ae 100644
> --- a/bitbake/lib/bb/ui/buildinfohelper.py
> +++ b/bitbake/lib/bb/ui/buildinfohelper.py
> @@ -42,7 +42,7 @@ from orm.models import Variable, VariableHistory
> from orm.models import Package, Package_File, Target_Installed_Package,
> Target_File
> from orm.models import Task_Dependency, Package_Dependency
> from orm.models import Recipe_Dependency, Provides
> -from orm.models import Project, CustomImagePackage, CustomImageRecipe
> +from orm.models import Project, CustomImagePackage
> from orm.models import signal_runbuilds
>
> from bldcontrol.models import BuildEnvironment, BuildRequest
> @@ -361,11 +361,6 @@ class ORMWrapper(object):
>
> def get_update_layer_version_object(self, build_obj, layer_obj,
> layer_version_information):
> if isinstance(layer_obj, Layer_Version):
> - # Special case the toaster-custom-images layer which is
> created
> - # on the fly so don't update the values which may cause the
> layer
> - # to be duplicated on a future get_or_create
> - if layer_obj.layer.name == CustomImageRecipe.LAYER_NAME:
> - return layer_obj
> # We already found our layer version for this build so just
> # update it with the new build information
> logger.debug("We found our layer from toaster")
> @@ -629,6 +624,8 @@ class ORMWrapper(object):
> recipe = self._cached_get(
> Recipe,
> name=built_recipe.name,
> + layer_version__layer__name=
> + built_recipe.layer_version.layer.name,
> layer_version__build=None,
> layer_version__release=
> built_recipe.layer_version.release,
> @@ -637,7 +634,7 @@ class ORMWrapper(object):
> )
> except (Recipe.DoesNotExist,
> Recipe.MultipleObjectsReturned) as e:
> - logger.info("We did not find one recipe for the"
> + logger.info("We did not find one recipe for the "
> "configuration data package %s %s" % (p,
> e))
> continue
>
> diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> index a64e89b..50c0b5d 100644
> --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> @@ -198,8 +198,6 @@ class LocalhostBEController(
> BuildEnvironmentController):
> if name != "bitbake":
> layerlist.append(localdirpath.rstrip("/"))
>
> - logger.debug("localhostbecontroller: current layer list %s " %
> pformat(layerlist))
> -
> # 5. create custom layer and add custom recipes to it
> layerpath = os.path.join(self.be.builddir,
> CustomImageRecipe.LAYER_NAME)
> @@ -222,19 +220,35 @@ class LocalhostBEController(
> BuildEnvironmentController):
> with open(config, "w") as conf:
> conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES +=
> "${LAYERDIR}/recipes/*.bb"\n')
>
> + BRLayer.objects.get_or_create(req=target.req,
> + name=layer.name,
> + dirpath=layerpath,
> + giturl="file://%s" % layerpath)
> +
> # Update the Layer_Version dirpath that has our base_recipe in
> # to be able to read the base recipe to then generate the
> # custom recipe.
> + logger.info(customrecipe.base_recipe.layer_version)
> +
> br_layer_base_recipe = layers.get(
> layer_version=customrecipe.base_recipe.layer_version)
>
> - br_layer_base_dirpath = \
> - os.path.join(self.be.sourcedir,
> - self.getGitCloneDirectory(
> - br_layer_base_recipe.giturl,
> - br_layer_base_recipe.commit),
> - customrecipe.base_recipe.
> layer_version.dirpath
> - )
> + # If the layer is one that we've cloned we know where it lives
> + if br_layer_base_recipe.giturl and
> br_layer_base_recipe.commit:
> + layer_path = self.getGitCloneDirectory(
> + br_layer_base_recipe.giturl,
> + br_layer_base_recipe.commit)
> + # Otherwise it's a local layer
> + elif br_layer_base_recipe.local_source_dir:
> + layer_path = br_layer_base_recipe.local_source_dir
> + else:
> + logger.error("Unable to workout the dir path for the
> custom"
> + " image recipe")
> +
> + br_layer_base_dirpath = os.path.join(
> + self.be.sourcedir,
> + layer_path,
> + customrecipe.base_recipe.layer_version.dirpath)
>
> customrecipe.base_recipe.layer_version.dirpath = \
> br_layer_base_dirpath
> @@ -249,21 +263,25 @@ class LocalhostBEController(
> BuildEnvironmentController):
>
> # Update the layer and recipe objects
> customrecipe.layer_version.dirpath = layerpath
> + customrecipe.layer_version.layer.local_source_dir = layerpath
> + customrecipe.layer_version.layer.save()
> customrecipe.layer_version.save()
>
> customrecipe.file_path = recipe_path
> customrecipe.save()
>
> # create *Layer* objects needed for build machinery to work
> - BRLayer.objects.get_or_create(req=target.req,
> - name=layer.name,
> - dirpath=layerpath,
> - giturl="file://%s" % layerpath)
> +
> +
> + nongitlayerlist.append(layerpath)
> +
> if os.path.isdir(layerpath):
> layerlist.append(layerpath)
>
> self.islayerset = True
> layerlist.extend(nongitlayerlist)
> +
> + logger.debug("localhostbecontroller: current layer list %s " %
> pformat(layerlist))
> return layerlist
>
> def readServerLogFile(self):
> diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/
> models.py
> index a7de57c..2c1edc9 100644
> --- a/bitbake/lib/toaster/orm/models.py
> +++ b/bitbake/lib/toaster/orm/models.py
> @@ -1631,7 +1631,8 @@ class CustomImageRecipe(Recipe):
> if base_recipe_path:
> base_recipe = open(base_recipe_path, 'r').read()
> else:
> - raise IOError("Based on recipe file not found")
> + raise IOError("Based on recipe file not found: %s" %
> + base_recipe_path)
>
> # Add a special case for when the recipe we have based a custom
> image
> # recipe on requires another recipe.
> diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/
> toastergui/api.py
> index ae1f150..e4c3b84 100644
> --- a/bitbake/lib/toaster/toastergui/api.py
> +++ b/bitbake/lib/toaster/toastergui/api.py
> @@ -293,8 +293,7 @@ class XhrCustomRecipe(View):
> # create layer 'Custom layer' and verion if needed
> layer = Layer.objects.get_or_create(
> name=CustomImageRecipe.LAYER_NAME,
> - summary="Layer for custom recipes",
> - vcs_url="file:///toaster_created_layer")[0]
> + summary="Layer for custom recipes")[0]
>
> # Check if we have a layer version already
> # We don't use get_or_create here because the dirpath will
> change
> @@ -306,6 +305,7 @@ class XhrCustomRecipe(View):
> lver, created = Layer_Version.objects.get_or_create(
> project=params['project'],
> layer=layer,
> + layer_source=LayerSource.TYPE_LOCAL,
> dirpath="toaster_created_layer")
>
> # Add a dependency on our layer to the base recipe's layer
> --
> 2.7.4
>
> --
> _______________________________________________
> toaster mailing list
> toaster@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/toaster
>
--
സുജിത് ഹരിദാസന്
Bangalore
<Project>Contributor to KDE project
<Project>Contributor to Yocto project
http://fci.wikia.com/wiki/Anti-DRM-Campaign
<Blog> http://sujithh.info
C-x C-c
[-- Attachment #2: Type: text/html, Size: 13100 bytes --]
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2016-10-25 5:05 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-10-07 16:47 [RFC] WIP unidentified layer / custom images and non git layers Michael Wood
2016-10-07 16:47 ` [PATCH] wip fix unidentified layer Michael Wood
2016-10-25 5:04 ` sujith h
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.