From: "Shakeel, Muhammad" <muhammad_shakeel@mentor.com>
To: openembedded-core@lists.openembedded.org
Cc: Christopher Larson <chris_larson@mentor.com>
Subject: [PATCH 1/2 v3] archiver class: Use tasks instead of pre/post funcs with sstate
Date: Tue, 22 Jan 2013 17:10:52 +0500 [thread overview]
Message-ID: <1358856653-17049-1-git-send-email-muhammad_shakeel@mentor.com> (raw)
From: Muhammad Shakeel <muhammad_shakeel@mentor.com>
* Add tasks to move sources, script/logs and diff/env files in
deploy directory.
* Enable SSTATE for 'do_archive_scripts_logs' task
* Enable SSTATE for 'do_dumpdata_create_diff_gz' task
* SSTATE is not used for sources/patches archiver task because source
archive package can result into a very large file. It will be an
unnecessary overhead to keep sources in DL_DIR and cached-binaries.
* If 'SOURCE_ARCHIVE_PACKAGE_TYPE' is 'srpm' then use pre/post functions
because in this case we do not want to use tasks to move sources/logs
in DEPLOY_DIR. 'do_package_write_rpm' is responsible for handling
archiver packages.
[YOCTO #3449]
Signed-off-by: Muhammad Shakeel <muhammad_shakeel@mentor.com>
Signed-off-by: Noor Ahsan <noor_ahsan@mentor.com>
Signed-off-by: Christopher Larson <chris_larson@mentor.com>
---
meta/classes/archive-configured-source.bbclass | 47 ++++++++++++++++++++++--
meta/classes/archive-original-source.bbclass | 47 ++++++++++++++++++++++--
meta/classes/archive-patched-source.bbclass | 47 ++++++++++++++++++++++--
meta/classes/archiver.bbclass | 40 ++++++++++++++------
4 files changed, 161 insertions(+), 20 deletions(-)
diff --git a/meta/classes/archive-configured-source.bbclass b/meta/classes/archive-configured-source.bbclass
index 1eaaf4c..ae70be3 100644
--- a/meta/classes/archive-configured-source.bbclass
+++ b/meta/classes/archive-configured-source.bbclass
@@ -8,10 +8,51 @@
inherit archiver
# Get archiving package with configured sources including patches
-do_configure[postfuncs] += "do_archive_configured_sources "
+addtask do_archive_configured_sources after do_configure
# Get archiving package with temp(logs) and scripts(.bb and inc files)
-do_package_write_rpm[prefuncs] += "do_archive_scripts_logs "
+addtask do_archive_scripts_logs after do_package_write_rpm
# Get dump date and create diff file
-do_package_write_rpm[postfuncs] += "do_dumpdata_create_diff_gz "
+addtask do_dumpdata_create_diff_gz after do_package_write_rpm before do_build
+
+python () {
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) != 'srpm':
+ """
+ If package type is not 'srpm' then add tasks to move archive packages of
+ configured sources and scripts/logs in ${DEPLOY_DIR}/sources.
+ """
+ d.appendVarFlag('do_compile', 'deps', ['do_archive_configured_sources'])
+ d.appendVarFlag('do_build', 'recrdeptask', ' do_archive_configured_sources')
+ d.appendVarFlag('do_build', 'deps', ['do_archive_scripts_logs'])
+
+ else:
+ d.prependVarFlag('do_configure', 'postfuncs', "do_archive_configured_sources")
+ d.prependVarFlag('do_package_write_rpm', 'prefuncs', "do_archive_scripts_logs")
+}
+
+ARCHIVE_SSTATE_OUTDIR = "${DEPLOY_DIR}/sources/"
+ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR = "${WORKDIR}/script-logs/"
+ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR = "${WORKDIR}/diffgz-envdata/"
+
+SSTATETASKS += "do_archive_scripts_logs"
+do_archive_scripts_logs[sstate-name] = "archive_scripts_logs"
+do_archive_scripts_logs[sstate-inputdirs] = "${ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR}"
+do_archive_scripts_logs[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
+
+python do_archive_scripts_logs_setscene () {
+ sstate_setscene(d)
+}
+
+addtask do_archive_scripts_logs_setscene
+
+SSTATETASKS += "do_dumpdata_create_diff_gz"
+do_dumpdata_create_diff_gz[sstate-name] = "dumpdata_create_diff_gz"
+do_dumpdata_create_diff_gz[sstate-inputdirs] = "${ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR}"
+do_dumpdata_create_diff_gz[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
+
+python do_dumpdata_create_diff_gz_setscene () {
+ sstate_setscene(d)
+}
+
+addtask do_dumpdata_create_diff_gz_setscene
diff --git a/meta/classes/archive-original-source.bbclass b/meta/classes/archive-original-source.bbclass
index 1b3f8d0..f668f3e 100644
--- a/meta/classes/archive-original-source.bbclass
+++ b/meta/classes/archive-original-source.bbclass
@@ -8,10 +8,51 @@
inherit archiver
# Get original sources archiving package with patches
-do_unpack[postfuncs] += "do_archive_original_sources_patches "
+addtask do_archive_original_sources_patches after do_unpack
# Get archiving package with temp(logs) and scripts(.bb and inc files)
-do_package_write_rpm[prefuncs] += "do_archive_scripts_logs "
+addtask do_archive_scripts_logs after do_package_write_rpm
# Get dump date and create diff file
-do_package_write_rpm[postfuncs] += "do_dumpdata_create_diff_gz "
+addtask do_dumpdata_create_diff_gz after do_package_write_rpm before do_build
+
+python () {
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) != 'srpm':
+ """
+ If package type is not 'srpm' then add tasks to move archive packages of
+ original sources and scripts/logs in ${DEPLOY_DIR}/sources.
+ """
+ d.appendVarFlag('do_patch', 'deps', ['do_archive_original_sources_patches'])
+ d.appendVarFlag('do_build', 'recrdeptask', ' do_archive_original_sources_patches')
+ d.appendVarFlag('do_build', 'deps', ['do_archive_scripts_logs'])
+
+ else:
+ d.prependVarFlag('do_unpack', 'postfuncs', "do_archive_original_sources_patches")
+ d.prependVarFlag('do_package_write_rpm', 'prefuncs', "do_archive_scripts_logs")
+}
+
+ARCHIVE_SSTATE_OUTDIR = "${DEPLOY_DIR}/sources/"
+ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR = "${WORKDIR}/script-logs/"
+ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR = "${WORKDIR}/diffgz-envdata/"
+
+SSTATETASKS += "do_archive_scripts_logs"
+do_archive_scripts_logs[sstate-name] = "archive_scripts_logs"
+do_archive_scripts_logs[sstate-inputdirs] = "${ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR}"
+do_archive_scripts_logs[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
+
+python do_archive_scripts_logs_setscene () {
+ sstate_setscene(d)
+}
+
+addtask do_archive_scripts_logs_setscene
+
+SSTATETASKS += "do_dumpdata_create_diff_gz"
+do_dumpdata_create_diff_gz[sstate-name] = "dumpdata_create_diff_gz"
+do_dumpdata_create_diff_gz[sstate-inputdirs] = "${ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR}"
+do_dumpdata_create_diff_gz[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
+
+python do_dumpdata_create_diff_gz_setscene () {
+ sstate_setscene(d)
+}
+
+addtask do_dumpdata_create_diff_gz_setscene
diff --git a/meta/classes/archive-patched-source.bbclass b/meta/classes/archive-patched-source.bbclass
index 40b2dcb..daadaae 100644
--- a/meta/classes/archive-patched-source.bbclass
+++ b/meta/classes/archive-patched-source.bbclass
@@ -8,10 +8,51 @@
inherit archiver
# Get archiving package with patched sources including patches
-do_patch[postfuncs] += "do_archive_patched_sources "
+addtask do_archive_patched_sources after do_patch
# Get archiving package with logs(temp) and scripts(.bb and .inc files)
-do_package_write_rpm[prefuncs] += "do_archive_scripts_logs "
+addtask do_archive_scripts_logs after do_package_write_rpm
# Get dump date and create diff file
-do_package_write_rpm[postfuncs] += "do_dumpdata_create_diff_gz "
+addtask do_dumpdata_create_diff_gz after do_package_write_rpm before do_build
+
+python () {
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) != 'srpm':
+ """
+ If package type is not 'srpm' then add tasks to move archive packages of
+ patched sources and scripts/logs in ${DEPLOY_DIR}/sources.
+ """
+ d.appendVarFlag('do_configure', 'deps', ['do_archive_patched_sources'])
+ d.appendVarFlag('do_build', 'recrdeptask', ' do_archive_patched_sources')
+ d.appendVarFlag('do_build', 'deps', ['do_archive_scripts_logs'])
+
+ else:
+ d.prependVarFlag('do_patch', 'postfuncs', "do_archive_patched_sources")
+ d.prependVarFlag('do_package_write_rpm', 'prefuncs', "do_archive_scripts_logs")
+}
+
+ARCHIVE_SSTATE_OUTDIR = "${DEPLOY_DIR}/sources/"
+ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR = "${WORKDIR}/script-logs/"
+ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR = "${WORKDIR}/diffgz-envdata/"
+
+SSTATETASKS += "do_archive_scripts_logs"
+do_archive_scripts_logs[sstate-name] = "archive_scripts_logs"
+do_archive_scripts_logs[sstate-inputdirs] = "${ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR}"
+do_archive_scripts_logs[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
+
+python do_archive_scripts_logs_setscene () {
+ sstate_setscene(d)
+}
+
+addtask do_archive_scripts_logs_setscene
+
+SSTATETASKS += "do_dumpdata_create_diff_gz"
+do_dumpdata_create_diff_gz[sstate-name] = "dumpdata_create_diff_gz"
+do_dumpdata_create_diff_gz[sstate-inputdirs] = "${ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR}"
+do_dumpdata_create_diff_gz[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
+
+python do_dumpdata_create_diff_gz_setscene () {
+ sstate_setscene(d)
+}
+
+addtask do_dumpdata_create_diff_gz_setscene
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index a4a2158..4323827 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -94,9 +94,11 @@ def get_bb_inc(d):
work_dir = d.getVar('WORKDIR', True)
bbfile = d.getVar('FILE', True)
bbdir = os.path.dirname(bbfile)
- script_logs = os.path.join(work_dir, 'script-logs')
+ target_sys = d.getVar('TARGET_SYS', True)
+ pf = d.getVar('PF', True)
+ licenses = get_licenses(d)
+ script_logs = os.path.join(work_dir, 'script-logs/'+ target_sys + '/' + licenses + '/' + pf + '/script-logs')
bb_inc = os.path.join(script_logs, 'bb_inc')
- bb.mkdirhier(script_logs)
bb.mkdirhier(bb_inc)
def find_file(dir, file):
@@ -124,6 +126,18 @@ def get_bb_inc(d):
for bbincfile in bbinc:
shutil.copy(bbincfile, bb_inc)
+ return script_logs
+
+def get_logs(d):
+ """
+ create a directory "script-logs" in ${WORKDIR}
+ """
+ work_dir = d.getVar('WORKDIR', True)
+ target_sys = d.getVar('TARGET_SYS', True)
+ pf = d.getVar('PF', True)
+ licenses = get_licenses(d)
+ script_logs = os.path.join(work_dir, 'script-logs/'+ target_sys + '/' + licenses + '/' + pf + '/script-logs')
+
try:
bb.mkdirhier(os.path.join(script_logs, 'temp'))
oe.path.copytree(os.path.join(work_dir, 'temp'), os.path.join(script_logs, 'temp'))
@@ -309,7 +323,8 @@ def archive_logs(d, logdir, bbinc=False):
work_dir = d.getVar('WORKDIR', True)
log_dir = os.path.basename(logdir)
tarname = pf + '-' + log_dir + ".tar.gz"
- tarname = do_tarball(work_dir, log_dir, tarname)
+ archive_dir = os.path.join( logdir, '..' )
+ tarname = do_tarball(archive_dir, log_dir, tarname)
if bbinc:
shutil.rmtree(logdir, ignore_errors=True)
return tarname
@@ -414,6 +429,7 @@ def archive_scripts_logs(d):
archive scripts and logs. scripts include .bb and .inc files and
logs include stuff in "temp".
"""
+ import shutil
if tar_filter(d):
return
@@ -421,18 +437,20 @@ def archive_scripts_logs(d):
temp_dir = os.path.join(work_dir, 'temp')
source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
if source_archive_log_with_scripts == 'logs_with_scripts':
+ logdir = get_logs(d)
logdir = get_bb_inc(d)
- tarlog = archive_logs(d, logdir, True)
elif source_archive_log_with_scripts == 'logs':
- if os.path.exists(temp_dir):
- tarlog = archive_logs(d, temp_dir, False)
+ logdir = get_logs(d)
else:
return
- if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) != 'srpm':
- move_tarball_deploy(d, [tarlog])
+ tarlog = archive_logs(d, logdir, True)
- else:
+ if d.getVar('SOURCE_ARCHIVE_PACKAGE_TYPE', True) == 'srpm':
+ if os.path.exists(work_dir+ '/' + tarlog):
+ os.remove(work_dir+ '/' + tarlog)
+ shutil.move(os.path.join(logdir, '..', tarlog), work_dir)
+ shutil.rmtree(os.path.join(work_dir,'script-logs'))
store_package(d, tarlog)
def dumpdata(d):
@@ -449,7 +467,7 @@ def dumpdata(d):
pf = d.getVar('PF', True)
target_sys = d.getVar('TARGET_SYS', True)
licenses = get_licenses(d)
- dumpdir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+ dumpdir = os.path.join(workdir, 'diffgz-envdata/'+ target_sys + '/' + licenses + '/' + pf )
if not os.path.exists(dumpdir):
bb.mkdirhier(dumpdir)
@@ -480,7 +498,7 @@ def create_diff_gz(d):
pf = d.getVar('PF', True)
licenses = get_licenses(d)
target_sys = d.getVar('TARGET_SYS', True)
- diff_dir = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
+ diff_dir = os.path.join(work_dir, 'diffgz-envdata/'+ target_sys + '/' + licenses + '/' + pf )
diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
f = open(os.path.join(work_dir,'temp/exclude-from-file'), 'a')
--
1.7.9.5
next reply other threads:[~2013-01-22 12:26 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2013-01-22 12:10 Shakeel, Muhammad [this message]
2013-01-22 15:41 ` [PATCH 1/2 v3] archiver class: Use tasks instead of pre/post funcs with sstate Richard Purdie
2013-02-18 14:54 ` Shakeel, Muhammad
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1358856653-17049-1-git-send-email-muhammad_shakeel@mentor.com \
--to=muhammad_shakeel@mentor.com \
--cc=chris_larson@mentor.com \
--cc=openembedded-core@lists.openembedded.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.