* [PATCH 1/5] Added the PR service.
2011-05-26 11:55 [PATCH 0/5] network based PR service(revised) Lianhao Lu
@ 2011-05-26 11:55 ` Lianhao Lu
2011-05-26 11:55 ` [PATCH 2/5] conf/bitbake.conf: Added variables for " Lianhao Lu
` (4 subsequent siblings)
5 siblings, 0 replies; 16+ messages in thread
From: Lianhao Lu @ 2011-05-26 11:55 UTC (permalink / raw)
To: openembedded-core
Added the initial implementation of the server side PR service.
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
bitbake/bin/bitbake-prserv | 53 +++++++++++
bitbake/lib/prserv/__init__.py | 11 ++
bitbake/lib/prserv/db.py | 100 ++++++++++++++++++++
bitbake/lib/prserv/serv.py | 198 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 362 insertions(+), 0 deletions(-)
create mode 100755 bitbake/bin/bitbake-prserv
create mode 100644 bitbake/lib/prserv/__init__.py
create mode 100644 bitbake/lib/prserv/db.py
create mode 100644 bitbake/lib/prserv/serv.py
diff --git a/bitbake/bin/bitbake-prserv b/bitbake/bin/bitbake-prserv
new file mode 100755
index 0000000..14073ca
--- /dev/null
+++ b/bitbake/bin/bitbake-prserv
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+import os
+import sys,logging
+import optparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib'))
+
+import prserv
+import prserv.serv
+
+__version__="1.0.0"
+
+PRHOST_DEFAULT=''
+PRPORT_DEFAULT=8585
+
+def main():
+ parser = optparse.OptionParser(
+ version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__),
+ usage = "%prog [options]")
+
+ parser.add_option("-f", "--file", help="database filename(default prserv.db)", action="store",
+ dest="dbfile", type="string", default="prserv.db")
+ parser.add_option("-l", "--log", help="log filename(default prserv.log)", action="store",
+ dest="logfile", type="string", default="prserv.log")
+ parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
+ action = "store", type="string", dest="loglevel", default = "WARNING")
+ parser.add_option("--start", help="start daemon",
+ action="store_true", dest="start", default="True")
+ parser.add_option("--stop", help="stop daemon",
+ action="store_false", dest="start")
+ parser.add_option("--host", help="ip address to bind", action="store",
+ dest="host", type="string", default=PRHOST_DEFAULT)
+ parser.add_option("--port", help="port number(default 8585)", action="store",
+ dest="port", type="int", default=PRPORT_DEFAULT)
+
+ options, args = parser.parse_args(sys.argv)
+
+ prserv.init_logger(os.path.abspath(options.logfile),options.loglevel)
+
+ if options.start:
+ prserv.serv.start_daemon(options)
+ else:
+ prserv.serv.stop_daemon()
+
+if __name__ == "__main__":
+ try:
+ ret = main()
+ except Exception:
+ ret = 1
+ import traceback
+ traceback.print_exc(5)
+ sys.exit(ret)
+
diff --git a/bitbake/lib/prserv/__init__.py b/bitbake/lib/prserv/__init__.py
new file mode 100644
index 0000000..2837e13
--- /dev/null
+++ b/bitbake/lib/prserv/__init__.py
@@ -0,0 +1,11 @@
+__version__ = "1.0.0"
+
+import os, time
+import sys,logging
+
+def init_logger(logfile, loglevel):
+ numeric_level = getattr(logging, loglevel.upper(), None)
+ if not isinstance(numeric_level, int):
+ raise ValueError('Invalid log level: %s' % loglevel)
+ logging.basicConfig(level=numeric_level, filename=logfile)
+
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
new file mode 100644
index 0000000..bbee931
--- /dev/null
+++ b/bitbake/lib/prserv/db.py
@@ -0,0 +1,100 @@
+import logging
+import os.path
+import errno
+import sys
+import warnings
+import sqlite3
+
+try:
+ import sqlite3
+except ImportError:
+ from pysqlite2 import dbapi2 as sqlite3
+
+sqlversion = sqlite3.sqlite_version_info
+if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
+ raise Exception("sqlite3 version 3.3.0 or later is required.")
+
+class NotFoundError(StandardError):
+ pass
+
+class PRTable():
+ def __init__(self,cursor,table):
+ self.cursor = cursor
+ self.table = table
+
+ #create the table
+ self._execute("CREATE TABLE IF NOT EXISTS %s \
+ (version TEXT NOT NULL, \
+ checksum TEXT NOT NULL, \
+ value INTEGER, \
+ PRIMARY KEY (version,checksum));"
+ % table)
+
+ def _execute(self, *query):
+ """Execute a query, waiting to acquire a lock if necessary"""
+ count = 0
+ while True:
+ try:
+ return self.cursor.execute(*query)
+ except sqlite3.OperationalError as exc:
+ if 'database is locked' in str(exc) and count < 500:
+ count = count + 1
+ continue
+ raise
+ except sqlite3.IntegrityError as exc:
+ print "Integrity error %s" % str(exc)
+ break
+
+ def getValue(self, version, checksum):
+ data=self._execute("SELECT value FROM %s WHERE version=? AND checksum=?;" % self.table,
+ (version,checksum))
+ row=data.fetchone()
+ if row != None:
+ return row[0]
+ else:
+ #no value found, try to insert
+ self._execute("INSERT INTO %s VALUES (?, ?, (select ifnull(max(value)+1,0) from %s where version=?));"
+ % (self.table,self.table),
+ (version,checksum,version))
+ data=self._execute("SELECT value FROM %s WHERE version=? AND checksum=?;" % self.table,
+ (version,checksum))
+ row=data.fetchone()
+ if row != None:
+ return row[0]
+ else:
+ raise NotFoundError
+
+class PRData(object):
+ """Object representing the PR database"""
+ def __init__(self, filename):
+ self.filename=os.path.abspath(filename)
+ #build directory hierarchy
+ try:
+ os.makedirs(os.path.dirname(self.filename))
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise e
+ self.connection=sqlite3.connect(self.filename, timeout=5,
+ isolation_level=None)
+ self.cursor=self.connection.cursor()
+ self._tables={}
+
+ def __del__(self):
+ print "PRData: closing DB %s" % self.filename
+ self.connection.close()
+
+ def __getitem__(self,tblname):
+ if not isinstance(tblname, basestring):
+ raise TypeError("tblname argument must be a string, not '%s'" %
+ type(tblname))
+ if tblname in self._tables:
+ return self._tables[tblname]
+ else:
+ tableobj = self._tables[tblname] = PRTable(self.cursor, tblname)
+ return tableobj
+
+ def __delitem__(self, tblname):
+ if tblname in self._tables:
+ del self._tables[tblname]
+ logging.info("drop table %s" % (tblname))
+ self.cursor.execute("DROP TABLE IF EXISTS %s;" % tblname)
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
new file mode 100644
index 0000000..ecafe4f
--- /dev/null
+++ b/bitbake/lib/prserv/serv.py
@@ -0,0 +1,198 @@
+import os,sys,logging
+import signal,time, atexit
+from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+import xmlrpclib,sqlite3
+
+import bb.server.xmlrpc
+import prserv
+import prserv.db
+
+if sys.hexversion < 0x020600F0:
+ print("Sorry, python 2.6 or later is required.")
+ sys.exit(1)
+
+class Handler(SimpleXMLRPCRequestHandler):
+ def _dispatch(self,method,params):
+ try:
+ value=self.server.funcs[method](*params)
+ except:
+ import traceback
+ traceback.print_exc()
+ raise
+ return value
+
+class PRServer(SimpleXMLRPCServer):
+ pidfile="/tmp/PRServer.pid"
+ def __init__(self, dbfile, logfile, interface, daemon=True):
+ ''' constructor '''
+ SimpleXMLRPCServer.__init__(self, interface,
+ requestHandler=SimpleXMLRPCRequestHandler,
+ logRequests=False, allow_none=True)
+ self.dbfile=dbfile
+ self.daemon=daemon
+ self.logfile=logfile
+ self.host, self.port = self.socket.getsockname()
+ self.db=prserv.db.PRData(dbfile)
+ self.table=self.db["PRMAIN"]
+
+ self.register_function(self.getPR, "getPR")
+ self.register_function(self.quit, "quit")
+ self.register_function(self.ping, "ping")
+ self.register_introspection_functions()
+
+ def ping(self):
+ return not self.quit
+
+ def getPR(self, version, checksum):
+ try:
+ return self.table.getValue(version,checksum)
+ except prserv.NotFoundError:
+ logging.error("can not find value for (%s, %s)",version,checksum)
+ return None
+ except sqlite3.Error as exc:
+ logging.error(str(exc))
+ return None
+
+ def quit(self):
+ self.quit=True
+ return
+
+ def _serve_forever(self):
+ self.quit = False
+ self.timeout = 0.5
+ while not self.quit:
+ self.handle_request()
+
+ logging.info("PRServer: stopping...")
+ self.server_close()
+ return
+
+ def start(self):
+ if self.daemon is True:
+ logging.info("PRServer: starting daemon...")
+ self.daemonize()
+ else:
+ logging.info("PRServer: starting...")
+ self._serve_forever()
+
+ def delpid(self):
+ os.remove(PRServer.pidfile)
+
+ def daemonize(self):
+ """
+ See Advanced Programming in the UNIX, Sec 13.3
+ """
+ os.umask(0)
+
+ try:
+ pid = os.fork()
+ if pid > 0:
+ sys.exit(0)
+ except OSError,e:
+ sys.stderr.write("1st fork failed: %d %s\n" % (e.errno, e.strerror))
+ sys.exit(1)
+
+ os.setsid()
+ """
+ fork again to make sure the daemon is not session leader,
+ which prevents it from acquiring controlling terminal
+ """
+ try:
+ pid = os.fork()
+ if pid > 0: #parent
+ sys.exit(0)
+ except OSError,e:
+ sys.stderr.write("2nd fork failed: %d %s\n" % (e.errno, e.strerror))
+ sys.exit(1)
+
+ os.chdir("/")
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+ si = file('/dev/null', 'r')
+ so = file(self.logfile, 'a+')
+ se = so
+ os.dup2(si.fileno(),sys.stdin.fileno())
+ os.dup2(so.fileno(),sys.stdout.fileno())
+ os.dup2(se.fileno(),sys.stderr.fileno())
+
+ # write pidfile
+ atexit.register(self.delpid)
+ pid = str(os.getpid())
+ pf = file(PRServer.pidfile, 'w+')
+ pf.write("%s\n" % pid)
+ pf.write("%s\n" % self.host)
+ pf.write("%s\n" % self.port)
+ pf.close()
+
+ self._serve_forever()
+
+class PRServerConnection():
+ def __init__(self, host, port):
+ self.connection = bb.server.xmlrpc._create_server(host, port)
+ self.host = host
+ self.port = port
+
+ def terminate(self):
+ # Don't wait for server indefinitely
+ import socket
+ socket.setdefaulttimeout(2)
+ try:
+ self.connection.quit()
+ except:
+ pass
+
+ def getPR(self, version, checksum):
+ return self.connection.getPR(version, checksum)
+
+ def ping(self):
+ return self.connection.ping()
+
+def start_daemon(options):
+ try:
+ pf = file(PRServer.pidfile,'r')
+ pid = int(pf.readline().strip())
+ pf.close()
+ except IOError:
+ pid = None
+
+ if pid:
+ sys.stderr.write("pidfile %s already exist. Daemon already running?\n"
+ % PRServer.pidfile)
+ sys.exit(1)
+
+ server = PRServer(options.dbfile, interface=(options.host, options.port),
+ logfile=os.path.abspath(options.logfile))
+ server.start()
+
+def stop_daemon():
+ try:
+ pf = file(PRServer.pidfile,'r')
+ pid = int(pf.readline().strip())
+ host = pf.readline().strip()
+ port = int(pf.readline().strip())
+ pf.close()
+ except IOError:
+ pid = None
+
+ if not pid:
+ sys.stderr.write("pidfile %s does not exist. Daemon not running?\n"
+ % PRServer.pidfile)
+ sys.exit(1)
+
+ PRServerConnection(host,port).terminate()
+ time.sleep(0.5)
+
+ try:
+ while 1:
+ os.kill(pid,signal.SIGTERM)
+ time.sleep(0.1)
+ except OSError, err:
+ err = str(err)
+ if err.find("No such process") > 0:
+ if os.path.exists(PRServer.pidfile):
+ os.remove(PRServer.pidfile)
+ else:
+ print err
+ sys.exit(1)
+
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 2/5] conf/bitbake.conf: Added variables for PR service.
2011-05-26 11:55 [PATCH 0/5] network based PR service(revised) Lianhao Lu
2011-05-26 11:55 ` [PATCH 1/5] Added the PR service Lianhao Lu
@ 2011-05-26 11:55 ` Lianhao Lu
2011-05-26 11:59 ` Phil Blundell
2011-05-26 11:55 ` [PATCH 3/5] classes/package(prserv).bbclass: Added PR service support Lianhao Lu
` (3 subsequent siblings)
5 siblings, 1 reply; 16+ messages in thread
From: Lianhao Lu @ 2011-05-26 11:55 UTC (permalink / raw)
To: openembedded-core
1. change BB_SIGNATURE_HANDLER from basic to basichash.
2. Added following variables for PR service:
USE_PR_SERV: flag of whether to use the network PR service
PRAUTOINX: search index for the network PR service
PKGV/PKGR: version and revision used in package feed
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
meta/conf/bitbake.conf | 18 ++++++++++++++++--
1 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 35fd6f8..06f5fd6 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -153,9 +153,17 @@ PR = "${@bb.parse.BBHandler.vars_from_file(bb.data.getVar('FILE',d),d)[2] or 'r0
PF = "${PN}-${EXTENDPE}${PV}-${PR}"
EXTENDPE = "${@['','${PE\x7d_'][bb.data.getVar('PE',d,1) > 0]}"
EXTENDPEVER = "${@['','${PE\x7d:'][bb.data.getVar('PE',d,1) > 0]}"
-EXTENDPV = "${EXTENDPEVER}${PV}-${PR}"
+EXTENDPV = "${EXTENDPEVER}${PV}-${PKGR}"
P = "${PN}-${PV}"
+EXTENDPRAUTO = "${@['.${PRAUTO\x7d',''][bb.data.getVar('PRAUTO',d,1) is None]}"
+PRAUTOINX = "${PF}${DISTRO_PR}"
+
+PKGV ?= "${PV}"
+PKGR ?= "${PR}${DISTRO_PR}${EXTENDPRAUTO}"
+EXTENDPKGVER = "${@['','${PKGE\x7d:'][bb.data.getVar('PKGE',d,1) > 0]}"
+EXTENDPKGV ?= "${EXTENDPKGVER}${PKGV}-${PKGR}"
+
# Base package name
# Automatically derives "foo" from "foo-native", "foo-cross" or "foo-initial"
# otherwise it is the same as PN and P
@@ -163,6 +171,11 @@ SPECIAL_PKGSUFFIX = "-native -cross -initial -intermediate -nativesdk -crosssdk
BPN = "${@base_prune_suffix(bb.data.getVar('PN', d, True), bb.data.getVar('SPECIAL_PKGSUFFIX', d, True).split(), d)}"
BP = "${BPN}-${PV}"
+#
+# network based PR service
+#
+USE_PR_SERV = "${@[1,0][(bb.data.getVar('PRSERV_HOST',d,1) is None) or (bb.data.getVar('PRSERV_PORT',d,1) is None)]}"
+
# Package info.
SECTION = "base"
@@ -649,6 +662,7 @@ PCMCIA_MANAGER ?= "pcmcia-cs"
DEFAULT_TASK_PROVIDER ?= "task-base"
MACHINE_TASK_PROVIDER ?= "${DEFAULT_TASK_PROVIDER}"
IMAGE_ROOTFS_SIZE ?= "65536"
+DISTRO_PR ?= ''
# Forcefully set CACHE now so future changes to things like
# MACHINE don't change the path to the cache
@@ -707,7 +721,7 @@ DISTRO[unexport] = "1"
TRANSLATED_TARGET_ARCH ??= ${TARGET_ARCH}
# Setup our default hash policy
-BB_SIGNATURE_HANDLER ?= "basic"
+BB_SIGNATURE_HANDLER ?= "basichash"
BB_HASHTASK_WHITELIST ?= "(.*-cross$|.*-native$|.*-cross-initial$|.*-cross-intermediate$|^virtual:native:.*|^virtual:nativesdk:.*)"
BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM USER FILESPATH USERNAME STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE"
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 2/5] conf/bitbake.conf: Added variables for PR service.
2011-05-26 11:55 ` [PATCH 2/5] conf/bitbake.conf: Added variables for " Lianhao Lu
@ 2011-05-26 11:59 ` Phil Blundell
2011-05-26 12:43 ` Lu, Lianhao
0 siblings, 1 reply; 16+ messages in thread
From: Phil Blundell @ 2011-05-26 11:59 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
On Thu, 2011-05-26 at 19:55 +0800, Lianhao Lu wrote:
> -EXTENDPV = "${EXTENDPEVER}${PV}-${PR}"
> +EXTENDPV = "${EXTENDPEVER}${PV}-${PKGR}"
That looks a bit weird. Is it really correct to be mixing PV and PKGR
like that?
FWIW, oe master has:
EXTENDPE = "${@int('${PE}') and '${PE}_' or ''}"
EXTENDPEVER = "${@int('${PE}') and '${PE}:' or ''}"
EXTENDPV = "${EXTENDPEVER}${PV}-${PR}${DISTRO_PR}"
PKGV ?= "${PV}"
PKGR ?= "${PR}${DISTRO_PR}"
EXTENDPKGEVER = "${@['','${PKGE\x7d:'][bb.data.getVar('PKGE',d,1) > 0]}"
EXTENDPKGV ?= "${EXTENDPKGEVER}${PKGV}-${PKGR}"
and, although there are a couple of uses of ${EXTENDPV} that should
perhaps be ${EXTENDPKGV}, this arrangement seems sensible otherwise.
p.
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 2/5] conf/bitbake.conf: Added variables for PR service.
2011-05-26 11:59 ` Phil Blundell
@ 2011-05-26 12:43 ` Lu, Lianhao
2011-05-26 12:47 ` Phil Blundell
0 siblings, 1 reply; 16+ messages in thread
From: Lu, Lianhao @ 2011-05-26 12:43 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
Phil Blundell wrote on 2011-05-26:
> On Thu, 2011-05-26 at 19:55 +0800, Lianhao Lu wrote:
>> -EXTENDPV = "${EXTENDPEVER}${PV}-${PR}"
>> +EXTENDPV = "${EXTENDPEVER}${PV}-${PKGR}"
>
> That looks a bit weird. Is it really correct to be mixing PV and PKGR like that?
>
> FWIW, oe master has:
>
> EXTENDPE = "${@int('${PE}') and '${PE}_' or ''}"
> EXTENDPEVER = "${@int('${PE}') and '${PE}:' or ''}"
> EXTENDPV = "${EXTENDPEVER}${PV}-${PR}${DISTRO_PR}"
>
> PKGV ?= "${PV}"
> PKGR ?= "${PR}${DISTRO_PR}"
>
> EXTENDPKGEVER = "${@['','${PKGE\x7d:'][bb.data.getVar('PKGE',d,1) > 0]}"
> EXTENDPKGV ?= "${EXTENDPKGEVER}${PKGV}-${PKGR}"
>
> and, although there are a couple of uses of ${EXTENDPV} that should
> perhaps be ${EXTENDPKGV}, this arrangement seems sensible otherwise.
>
The problem is that in OE-core the default -deb/-dbg packages are all using EXTENDPV, as well as some other recipes. Do you mean we should make them all using EXTENDPKGV instead of EXTENDPV? If this is the way, I think PKGE should be used instead of PE in package_xxx.bbclass, though currently I don't see any recipe in OE using PKGE, but some of them do use PE.
Best Regards,
Lianhao
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 2/5] conf/bitbake.conf: Added variables for PR service.
2011-05-26 12:43 ` Lu, Lianhao
@ 2011-05-26 12:47 ` Phil Blundell
2011-05-26 13:02 ` Lu, Lianhao
0 siblings, 1 reply; 16+ messages in thread
From: Phil Blundell @ 2011-05-26 12:47 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
On Thu, 2011-05-26 at 20:43 +0800, Lu, Lianhao wrote:
> The problem is that in OE-core the default -deb/-dbg packages are all
> using EXTENDPV, as well as some other recipes. Do you mean we should
> make them all using EXTENDPKGV instead of EXTENDPV?
As far as I can tell, yes, that would be the right thing to do. That's
what I was thinking of when I mentioned the "couple of uses" before.
The only reason I hesitate is that, although the current behaviour looks
like it should be wrong, nobody has actually complained about it and
that makes me wonder if I have misunderstood something.
I guess someone needs to do a few tests to figure out what exactly is
going on there. But I am fairly sure that creating a hybrid PV+PKGR is
not a good thing to do; it should be either one thing or the other, and
as far as I know everything on the output side should be using PKGxx.
p.
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [PATCH 2/5] conf/bitbake.conf: Added variables for PR service.
2011-05-26 12:47 ` Phil Blundell
@ 2011-05-26 13:02 ` Lu, Lianhao
2011-05-26 13:07 ` Phil Blundell
0 siblings, 1 reply; 16+ messages in thread
From: Lu, Lianhao @ 2011-05-26 13:02 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
Phil Blundell wrote on 2011-05-26:
> On Thu, 2011-05-26 at 20:43 +0800, Lu, Lianhao wrote:
>> The problem is that in OE-core the default -deb/-dbg packages are
>> all using EXTENDPV, as well as some other recipes. Do you mean we
>> should make them all using EXTENDPKGV instead of EXTENDPV?
>
> As far as I can tell, yes, that would be the right thing to do.
> That's what I was thinking of when I mentioned the "couple of uses" before.
> The only reason I hesitate is that, although the current behaviour
> looks like it should be wrong, nobody has actually complained about it
> and that makes me wonder if I have misunderstood something.
>
> I guess someone needs to do a few tests to figure out what exactly is
> going on there. But I am fairly sure that creating a hybrid PV+PKGR
> is not a good thing to do; it should be either one thing or the other,
> and as far as I know everything on the output side should be using PKGxx.
>
Maybe all the recipes are using the PKGV default value (?= ${PV}, so this just happens to make EXTENDPKGV equal to EXTENDPV. In what situation we should use EXTENDPV but not EXTENDPKGV then?
Best Regards,
Lianhao
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 2/5] conf/bitbake.conf: Added variables for PR service.
2011-05-26 13:02 ` Lu, Lianhao
@ 2011-05-26 13:07 ` Phil Blundell
2011-05-26 14:08 ` Richard Purdie
0 siblings, 1 reply; 16+ messages in thread
From: Phil Blundell @ 2011-05-26 13:07 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
On Thu, 2011-05-26 at 21:02 +0800, Lu, Lianhao wrote:
> Maybe all the recipes are using the PKGV default value (?= ${PV}, so
> this just happens to make EXTENDPKGV equal to EXTENDPV.
Yes, could be. The most common situation where PKGV != PV is when
something like gitpkgv.bbclass is in use, which doesn't happen all that
often. The main effect of using the wrong vars would be to make the
-doc/-dbg/-dev packages of gitpkgv-using recipes uninstallable, and I
guess I could believe that this might be happening without anybody
having noticed.
> In what situation we should use EXTENDPV but not EXTENDPKGV then?
Good question. I suspect the answer is probably "none".
p.
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 2/5] conf/bitbake.conf: Added variables for PR service.
2011-05-26 13:07 ` Phil Blundell
@ 2011-05-26 14:08 ` Richard Purdie
0 siblings, 0 replies; 16+ messages in thread
From: Richard Purdie @ 2011-05-26 14:08 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
On Thu, 2011-05-26 at 14:07 +0100, Phil Blundell wrote:
> On Thu, 2011-05-26 at 21:02 +0800, Lu, Lianhao wrote:
> > Maybe all the recipes are using the PKGV default value (?= ${PV}, so
> > this just happens to make EXTENDPKGV equal to EXTENDPV.
>
> Yes, could be. The most common situation where PKGV != PV is when
> something like gitpkgv.bbclass is in use, which doesn't happen all that
> often. The main effect of using the wrong vars would be to make the
> -doc/-dbg/-dev packages of gitpkgv-using recipes uninstallable, and I
> guess I could believe that this might be happening without anybody
> having noticed.
>
> > In what situation we should use EXTENDPV but not EXTENDPKGV then?
>
> Good question. I suspect the answer is probably "none".
For what its worth I agree with your conclusions, it probably is broken
and nobody has noticed.
Lets keep this simpler and just have one variable :)
Cheers,
Richard
^ permalink raw reply [flat|nested] 16+ messages in thread
* [PATCH 3/5] classes/package(prserv).bbclass: Added PR service support.
2011-05-26 11:55 [PATCH 0/5] network based PR service(revised) Lianhao Lu
2011-05-26 11:55 ` [PATCH 1/5] Added the PR service Lianhao Lu
2011-05-26 11:55 ` [PATCH 2/5] conf/bitbake.conf: Added variables for " Lianhao Lu
@ 2011-05-26 11:55 ` Lianhao Lu
2011-05-26 11:55 ` [PATCH 4/5] classes/package_xxx.class: " Lianhao Lu
` (2 subsequent siblings)
5 siblings, 0 replies; 16+ messages in thread
From: Lianhao Lu @ 2011-05-26 11:55 UTC (permalink / raw)
To: openembedded-core
1. Added package_get_auto_pr to PACKAGEFUNCS to get the auto
incremented value(PRAUTO) from remote PR service.
2. use PKGV/PKGR for pkgdata which will be used by package_write_xxx.
3. Added supporting functions in prserv.bbclass.
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
meta/classes/package.bbclass | 51 +++++++++++++++++++++++++++++------------
meta/classes/prserv.bbclass | 29 +++++++++++++++++++++++
2 files changed, 65 insertions(+), 15 deletions(-)
create mode 100644 meta/classes/prserv.bbclass
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 2c6d30c..a1b9482 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -7,34 +7,37 @@
#
# There are the following default steps but PACKAGEFUNCS can be extended:
#
-# a) perform_packagecopy - Copy D into PKGD
+# a) package_get_auto_pr - get PRAUTO from remote PR service
#
-# b) package_do_split_locales - Split out the locale files, updates FILES and PACKAGES
+# b) perform_packagecopy - Copy D into PKGD
#
-# c) split_and_strip_files - split the files into runtime and debug and strip them.
+# c) package_do_split_locales - Split out the locale files, updates FILES and PACKAGES
+#
+# d) split_and_strip_files - split the files into runtime and debug and strip them.
# Debug files include debug info split, and associated sources that end up in -dbg packages
#
-# d) populate_packages - Split the files in PKGD into separate packages in PKGDEST/<pkgname>
+# e) populate_packages - Split the files in PKGD into separate packages in PKGDEST/<pkgname>
# Also triggers the binary stripping code to put files in -dbg packages.
#
-# e) package_do_filedeps - Collect perfile run-time dependency metadata
+# f) package_do_filedeps - Collect perfile run-time dependency metadata
# The data is stores in FILER{PROVIDES,DEPENDS}_file_pkg variables with
# a list of affected files in FILER{PROVIDES,DEPENDS}FLIST_pkg
#
-# f) package_do_shlibs - Look at the shared libraries generated and autotmatically add any
+# g) package_do_shlibs - Look at the shared libraries generated and autotmatically add any
# depenedencies found. Also stores the package name so anyone else using this library
# knows which package to depend on.
#
-# g) package_do_pkgconfig - Keep track of which packages need and provide which .pc files
+# h) package_do_pkgconfig - Keep track of which packages need and provide which .pc files
#
-# h) read_shlibdeps - Reads the stored shlibs information into the metadata
+# i) read_shlibdeps - Reads the stored shlibs information into the metadata
#
-# i) package_depchains - Adds automatic dependencies to -dbg and -dev packages
+# j) package_depchains - Adds automatic dependencies to -dbg and -dev packages
#
-# j) emit_pkgdata - saves the packaging data into PKGDATA_DIR for use in later
+# k) emit_pkgdata - saves the packaging data into PKGDATA_DIR for use in later
# packaging steps
inherit packagedata
+inherit prserv
PKGD = "${WORKDIR}/package"
PKGDEST = "${WORKDIR}/packages-split"
@@ -326,6 +329,15 @@ def runtime_mapping_rename (varname, d):
# Package functions suitable for inclusion in PACKAGEFUNCS
#
+python package_get_auto_pr() {
+ if d.getVar('USE_PR_SERV', True):
+ auto_pr=prserv_get_pr_auto(d)
+ if auto_pr is None:
+ bb.fatal("Can NOT get auto PR revision from remote PR service")
+ return
+ d.setVar('PRAUTO',str(auto_pr))
+}
+
python package_do_split_locales() {
if (bb.data.getVar('PACKAGE_NO_LOCALE', d, True) == '1'):
bb.debug(1, "package requested not splitting locales")
@@ -773,6 +785,8 @@ python emit_pkgdata() {
write_if_exists(sf, pkg, 'PN')
write_if_exists(sf, pkg, 'PV')
write_if_exists(sf, pkg, 'PR')
+ write_if_exists(sf, pkg, 'PKGV')
+ write_if_exists(sf, pkg, 'PKGR')
write_if_exists(sf, pkg, 'DESCRIPTION')
write_if_exists(sf, pkg, 'SUMMARY')
write_if_exists(sf, pkg, 'RDEPENDS')
@@ -911,9 +925,9 @@ python package_do_shlibs() {
workdir = bb.data.getVar('WORKDIR', d, True)
- ver = bb.data.getVar('PV', d, True)
+ ver = bb.data.getVar('PKGV', d, True)
if not ver:
- bb.error("PV not defined")
+ bb.error("PKGV not defined")
return
pkgdest = bb.data.getVar('PKGDEST', d, True)
@@ -1025,6 +1039,12 @@ python package_do_shlibs() {
needs_ldconfig = False
bb.debug(2, "calculating shlib provides for %s" % pkg)
+ pkgver = bb.data.getVar('PKGV_' + pkg, d, True)
+ if not pkgver:
+ pkgver = bb.data.getVar('PV_' + pkg, d, True)
+ if not pkgver:
+ pkgver = ver
+
needed[pkg] = []
sonames = list()
renames = list()
@@ -1048,10 +1068,10 @@ python package_do_shlibs() {
fd = open(shlibs_file, 'w')
for s in sonames:
fd.write(s + '\n')
- shlib_provider[s] = (pkg, ver)
+ shlib_provider[s] = (pkg, pkgver)
fd.close()
fd = open(shver_file, 'w')
- fd.write(ver + '\n')
+ fd.write(pkgver + '\n')
fd.close()
if needs_ldconfig and use_ldconfig:
bb.debug(1, 'adding ldconfig call to postinst for %s' % pkg)
@@ -1348,7 +1368,8 @@ python package_depchains() {
}
PACKAGE_PREPROCESS_FUNCS ?= ""
-PACKAGEFUNCS ?= "perform_packagecopy \
+PACKAGEFUNCS ?= "package_get_auto_pr \
+ perform_packagecopy \
${PACKAGE_PREPROCESS_FUNCS} \
package_do_split_locales \
split_and_strip_files \
diff --git a/meta/classes/prserv.bbclass b/meta/classes/prserv.bbclass
new file mode 100644
index 0000000..de46ff6
--- /dev/null
+++ b/meta/classes/prserv.bbclass
@@ -0,0 +1,29 @@
+def prserv_make_conn(d):
+ import prserv.serv
+ host=d.getVar("PRSERV_HOST",True)
+ port=d.getVar("PRSERV_PORT",True)
+ try:
+ conn=None
+ conn=prserv.serv.PRServerConnection(host,int(port))
+ d.setVar("__PRSERV_CONN",conn)
+ except Exception, exc:
+ bb.fatal("Connecting to PR service %s:%s failed: %s" % (host, port, str(exc)))
+
+ return conn
+
+def prserv_get_pr_auto(d):
+ if not d.getVar('USE_PR_SERV', True):
+ bb.warn("Not using network based PR service")
+ return None
+
+ conn=d.getVar("__PRSERV_CONN", True)
+ if conn is None:
+ conn=prserv_make_conn(d)
+ if conn is None:
+ return None
+
+ version=d.getVar("PF", True)
+ checksum=d.getVar("BB_TASKHASH", True)
+ auto_rev=conn.getPR(version,checksum)
+ bb.debug(1,"prserv_get_pr_auto: version: %s checksum: %s result %d" % (version, checksum, auto_rev))
+ return auto_rev
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 4/5] classes/package_xxx.class: Added PR service support.
2011-05-26 11:55 [PATCH 0/5] network based PR service(revised) Lianhao Lu
` (2 preceding siblings ...)
2011-05-26 11:55 ` [PATCH 3/5] classes/package(prserv).bbclass: Added PR service support Lianhao Lu
@ 2011-05-26 11:55 ` Lianhao Lu
2011-05-26 11:55 ` [PATCH 5/5] meta-yocto/local.conf.sample: Added PRSERV_HOST and PRSERV_PORT Lianhao Lu
2011-05-26 12:07 ` [PATCH 0/5] network based PR service(revised) Koen Kooi
5 siblings, 0 replies; 16+ messages in thread
From: Lianhao Lu @ 2011-05-26 11:55 UTC (permalink / raw)
To: openembedded-core
Use PKGV/PKGR to build various package feed in tasks of pacakge_write_xxx.
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
meta/classes/package_deb.bbclass | 6 +++---
meta/classes/package_ipk.bbclass | 6 +++---
meta/classes/package_rpm.bbclass | 10 +++++-----
meta/classes/package_tar.bbclass | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index 000d9ee..44e8419 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -272,7 +272,7 @@ python do_package_deb () {
except ValueError:
pass
if not g and bb.data.getVar('ALLOW_EMPTY', localdata) != "1":
- bb.note("Not creating empty archive for %s-%s-%s" % (pkg, bb.data.getVar('PV', localdata, True), bb.data.getVar('PR', localdata, True)))
+ bb.note("Not creating empty archive for %s-%s-%s" % (pkg, bb.data.getVar('PKGV', localdata, True), bb.data.getVar('PKGR', localdata, True)))
bb.utils.unlockfile(lf)
continue
@@ -290,9 +290,9 @@ python do_package_deb () {
fields = []
pe = bb.data.getVar('PE', d, True)
if pe and int(pe) > 0:
- fields.append(["Version: %s:%s-%s\n", ['PE', 'PV', 'PR']])
+ fields.append(["Version: %s:%s-%s\n", ['PE', 'PKGV', 'PKGR']])
else:
- fields.append(["Version: %s-%s\n", ['PV', 'PR']])
+ fields.append(["Version: %s-%s\n", ['PKGV', 'PKGR']])
fields.append(["Description: %s\n", ['DESCRIPTION']])
fields.append(["Section: %s\n", ['SECTION']])
fields.append(["Priority: %s\n", ['PRIORITY']])
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index a3dfc73..177be55 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -241,7 +241,7 @@ python do_package_ipk () {
except ValueError:
pass
if not g and bb.data.getVar('ALLOW_EMPTY', localdata) != "1":
- bb.note("Not creating empty archive for %s-%s-%s" % (pkg, bb.data.getVar('PV', localdata, 1), bb.data.getVar('PR', localdata, 1)))
+ bb.note("Not creating empty archive for %s-%s-%s" % (pkg, bb.data.getVar('PKGV', localdata, 1), bb.data.getVar('PKGR', localdata, 1)))
bb.utils.unlockfile(lf)
continue
@@ -256,9 +256,9 @@ python do_package_ipk () {
fields = []
pe = bb.data.getVar('PE', d, 1)
if pe and int(pe) > 0:
- fields.append(["Version: %s:%s-%s\n", ['PE', 'PV', 'PR']])
+ fields.append(["Version: %s:%s-%s\n", ['PE', 'PKGV', 'PKGR']])
else:
- fields.append(["Version: %s-%s\n", ['PV', 'PR']])
+ fields.append(["Version: %s-%s\n", ['PKGV', 'PKGR']])
fields.append(["Description: %s\n", ['DESCRIPTION']])
fields.append(["Section: %s\n", ['SECTION']])
fields.append(["Priority: %s\n", ['PRIORITY']])
diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index 1d8c686..6bfa877 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -330,7 +330,7 @@ python write_specfile () {
if dep and ver:
if '-' in ver:
subd = oe.packagedata.read_subpkgdata_dict(dep, d)
- pv = subd['PV']
+ pv = subd['PKGV']
reppv = pv.replace('-', '+')
ver = ver.replace(pv, reppv)
newdeps_dict[dep] = ver
@@ -383,8 +383,8 @@ python write_specfile () {
# Construct the SPEC file...
srcname = bb.data.getVar('PN', d, True)
srcsummary = (bb.data.getVar('SUMMARY', d, True) or bb.data.getVar('DESCRIPTION', d, True) or ".")
- srcversion = bb.data.getVar('PV', d, True).replace('-', '+')
- srcrelease = bb.data.getVar('PR', d, True)
+ srcversion = bb.data.getVar('PKGV', d, True).replace('-', '+')
+ srcrelease = bb.data.getVar('PKGR', d, True)
srcepoch = (bb.data.getVar('PE', d, True) or "")
srclicense = bb.data.getVar('LICENSE', d, True)
srcsection = bb.data.getVar('SECTION', d, True)
@@ -438,8 +438,8 @@ python write_specfile () {
splitname = pkgname
splitsummary = (bb.data.getVar('SUMMARY', localdata, True) or bb.data.getVar('DESCRIPTION', localdata, True) or ".")
- splitversion = (bb.data.getVar('PV', localdata, True) or "").replace('-', '+')
- splitrelease = (bb.data.getVar('PR', localdata, True) or "")
+ splitversion = (bb.data.getVar('PKGV', localdata, True) or "").replace('-', '+')
+ splitrelease = (bb.data.getVar('PKGR', localdata, True) or "")
splitepoch = (bb.data.getVar('PE', localdata, True) or "")
splitlicense = (bb.data.getVar('LICENSE', localdata, True) or "")
splitsection = (bb.data.getVar('SECTION', localdata, True) or "")
diff --git a/meta/classes/package_tar.bbclass b/meta/classes/package_tar.bbclass
index e546eb7..a806e45 100644
--- a/meta/classes/package_tar.bbclass
+++ b/meta/classes/package_tar.bbclass
@@ -3,7 +3,7 @@ inherit package
IMAGE_PKGTYPE ?= "tar"
python package_tar_fn () {
- fn = os.path.join(bb.data.getVar('DEPLOY_DIR_TAR', d), "%s-%s-%s.tar.gz" % (bb.data.getVar('PKG', d), bb.data.getVar('PV', d), bb.data.getVar('PR', d)))
+ fn = os.path.join(bb.data.getVar('DEPLOY_DIR_TAR', d), "%s-%s-%s.tar.gz" % (bb.data.getVar('PKG', d), bb.data.getVar('PKGV', d), bb.data.getVar('PKGR', d)))
fn = bb.data.expand(fn, d)
bb.data.setVar('PKGFN', fn, d)
}
@@ -83,7 +83,7 @@ python do_package_tar () {
os.chdir(root)
from glob import glob
if not glob('*'):
- bb.note("Not creating empty archive for %s-%s-%s" % (pkg, bb.data.getVar('PV', localdata, 1), bb.data.getVar('PR', localdata, 1)))
+ bb.note("Not creating empty archive for %s-%s-%s" % (pkg, bb.data.getVar('PKGV', localdata, 1), bb.data.getVar('PKGR', localdata, 1)))
continue
ret = os.system("tar -czf %s %s" % (tarfn, '.'))
if ret != 0:
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* [PATCH 5/5] meta-yocto/local.conf.sample: Added PRSERV_HOST and PRSERV_PORT.
2011-05-26 11:55 [PATCH 0/5] network based PR service(revised) Lianhao Lu
` (3 preceding siblings ...)
2011-05-26 11:55 ` [PATCH 4/5] classes/package_xxx.class: " Lianhao Lu
@ 2011-05-26 11:55 ` Lianhao Lu
2011-05-26 12:07 ` [PATCH 0/5] network based PR service(revised) Koen Kooi
5 siblings, 0 replies; 16+ messages in thread
From: Lianhao Lu @ 2011-05-26 11:55 UTC (permalink / raw)
To: openembedded-core
Setting PRSERV_HOST and PRSERV_PORT would trigger the poky to use remote
PR network service. Leaving them unset allow the user to build image
without the PR network service
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>
---
meta-yocto/conf/local.conf.sample | 5 +++++
1 files changed, 5 insertions(+), 0 deletions(-)
diff --git a/meta-yocto/conf/local.conf.sample b/meta-yocto/conf/local.conf.sample
index a9b4998..359e510 100644
--- a/meta-yocto/conf/local.conf.sample
+++ b/meta-yocto/conf/local.conf.sample
@@ -216,3 +216,8 @@ NO32LIBS = "1"
# GNOME, SCREEN, XTERM and KONSOLE
#TERMCMD = "${KONSOLE_TERMCMD}"
#TERMCMDRUN = "${KONSOLE_TERMCMDRUN}"
+
+
+# The network based PR service host and port
+#PRSERV_HOST = "localhost"
+#PRSERV_PORT = "8585"
--
1.7.0.4
^ permalink raw reply related [flat|nested] 16+ messages in thread* Re: [PATCH 0/5] network based PR service(revised)
2011-05-26 11:55 [PATCH 0/5] network based PR service(revised) Lianhao Lu
` (4 preceding siblings ...)
2011-05-26 11:55 ` [PATCH 5/5] meta-yocto/local.conf.sample: Added PRSERV_HOST and PRSERV_PORT Lianhao Lu
@ 2011-05-26 12:07 ` Koen Kooi
2011-05-26 12:29 ` Lu, Lianhao
5 siblings, 1 reply; 16+ messages in thread
From: Koen Kooi @ 2011-05-26 12:07 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
Op 26 mei 2011, om 13:55 heeft Lianhao Lu het volgende geschreven:
> This is a modification of the original patch to enable network based PR service.
>
> The main difference between this series of patch to the original one is that
> this one uses the PKGR/PKGV during the package feed creation. It also made the
> PR service disabled by default(by leaving the PRSERV_HOST and PRSERV_PORT
> commented out) and revised some function names.
>
> The following features from the previous patch comments are not included in this
> series:
> 1) automatically launch PR service in localhost use case.
> 2) respect OVERIDE to allow setting PRSERV_HOST/PORT per recipe basis
> 3) write permission control in the PR service.
> 4) way to sync up local PR database witht the central server.
Does it support going thru HTTP_PROXY like the bitbake fetchers? It's not a must-have, but it would be nice if I can have the internal builds consistent with angstrom upstream.
regards,
Koen
^ permalink raw reply [flat|nested] 16+ messages in thread* Re: [PATCH 0/5] network based PR service(revised)
2011-05-26 12:07 ` [PATCH 0/5] network based PR service(revised) Koen Kooi
@ 2011-05-26 12:29 ` Lu, Lianhao
0 siblings, 0 replies; 16+ messages in thread
From: Lu, Lianhao @ 2011-05-26 12:29 UTC (permalink / raw)
To: Patches and discussions about the oe-core layer
Koen Kooi wrote on 2011-05-26:
>
> Op 26 mei 2011, om 13:55 heeft Lianhao Lu het volgende geschreven:
>
>> This is a modification of the original patch to enable network based PR
>> service.
>>
>> The main difference between this series of patch to the original one is
>> that this one uses the PKGR/PKGV during the package feed creation. It
>> also made the PR service disabled by default(by leaving the PRSERV_HOST
>> and PRSERV_PORT commented out) and revised some function names.
>>
>> The following features from the previous patch comments are not
>> included in this
>> series:
>> 1) automatically launch PR service in localhost use case.
>> 2) respect OVERIDE to allow setting PRSERV_HOST/PORT per recipe
>> basis
>> 3) write permission control in the PR service.
>> 4) way to sync up local PR database witht the central server.
>
> Does it support going thru HTTP_PROXY like the bitbake fetchers? It's
> not a must-have, but it would be nice if I can have the internal
> builds consistent with angstrom upstream.
>
It doesn't support now. But I think we can have this by implementing a custom transport, by leverage the urllib2.
Best Regards,
Lianhao
^ permalink raw reply [flat|nested] 16+ messages in thread