From: Joshua Watt <jpewhacker@gmail.com> To: openembedded-core@lists.openembedded.org, bitbake-devel@lists.openembedded.org Subject: [OE-core][PATCH v7 2/3] bitbake: hashserv: Add hash equivalence reference server Date: Fri, 4 Jan 2019 10:20:14 -0600 [thread overview] Message-ID: <20190104162015.456-3-JPEWhacker@gmail.com> (raw) In-Reply-To: <20190104162015.456-1-JPEWhacker@gmail.com> Implements a reference implementation of the hash equivalence server. This server has minimal dependencies (and no dependencies outside of the standard Python library), and implements the minimum required to be a conforming hash equivalence server. [YOCTO #13030] Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> --- bitbake/bin/bitbake-hashserv | 67 ++++++++++++++ bitbake/bin/bitbake-selftest | 2 + bitbake/lib/hashserv/__init__.py | 152 +++++++++++++++++++++++++++++++ bitbake/lib/hashserv/tests.py | 141 ++++++++++++++++++++++++++++ 4 files changed, 362 insertions(+) create mode 100755 bitbake/bin/bitbake-hashserv create mode 100644 bitbake/lib/hashserv/__init__.py create mode 100644 bitbake/lib/hashserv/tests.py diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv new file mode 100755 index 00000000000..c49397b73a5 --- /dev/null +++ b/bitbake/bin/bitbake-hashserv @@ -0,0 +1,67 @@ +#! /usr/bin/env python3 +# +# Copyright (C) 2018 Garmin Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import os +import sys +import logging +import argparse +import sqlite3 + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) + +import hashserv + +VERSION = "1.0.0" + +DEFAULT_HOST = '' +DEFAULT_PORT = 8686 + +def main(): + parser = argparse.ArgumentParser(description='HTTP Equivalence Reference Server. Version=%s' % VERSION) + parser.add_argument('--address', default=DEFAULT_HOST, help='Bind address (default "%(default)s")') + parser.add_argument('--port', type=int, default=DEFAULT_PORT, help='Bind port (default %(default)d)') + parser.add_argument('--prefix', default='', help='HTTP path prefix (default "%(default)s")') + parser.add_argument('--database', default='./hashserv.db', help='Database file (default "%(default)s")') + parser.add_argument('--log', default='WARNING', help='Set logging level') + + args = parser.parse_args() + + logger = logging.getLogger('hashserv') + + level = getattr(logging, args.log.upper(), None) + if not isinstance(level, int): + raise ValueError('Invalid log level: %s' % args.log) + + logger.setLevel(level) + console = logging.StreamHandler() + console.setLevel(level) + logger.addHandler(console) + + db = sqlite3.connect(args.database) + + server = hashserv.create_server((args.address, args.port), db, args.prefix) + server.serve_forever() + return 0 + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) + diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest index c970dcae90c..99f1af910f4 100755 --- a/bitbake/bin/bitbake-selftest +++ b/bitbake/bin/bitbake-selftest @@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib import unittest try: import bb + import hashserv import layerindexlib except RuntimeError as exc: sys.exit(str(exc)) @@ -35,6 +36,7 @@ tests = ["bb.tests.codeparser", "bb.tests.parse", "bb.tests.persist_data", "bb.tests.utils", + "hashserv.tests", "layerindexlib.tests.layerindexobj", "layerindexlib.tests.restapi", "layerindexlib.tests.cooker"] diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py new file mode 100644 index 00000000000..46bca7cab32 --- /dev/null +++ b/bitbake/lib/hashserv/__init__.py @@ -0,0 +1,152 @@ +# Copyright (C) 2018 Garmin Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from http.server import BaseHTTPRequestHandler, HTTPServer +import contextlib +import urllib.parse +import sqlite3 +import json +import traceback +import logging +from datetime import datetime + +logger = logging.getLogger('hashserv') + +class HashEquivalenceServer(BaseHTTPRequestHandler): + def log_message(self, f, *args): + logger.debug(f, *args) + + def do_GET(self): + try: + p = urllib.parse.urlparse(self.path) + + if p.path != self.prefix + '/v1/equivalent': + self.send_error(404) + return + + query = urllib.parse.parse_qs(p.query, strict_parsing=True) + method = query['method'][0] + taskhash = query['taskhash'][0] + + d = None + with contextlib.closing(self.db.cursor()) as cursor: + cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1', + {'method': method, 'taskhash': taskhash}) + + row = cursor.fetchone() + + if row is not None: + logger.debug('Found equivalent task %s', row['taskhash']) + d = {k: row[k] for k in ('taskhash', 'method', 'unihash')} + + self.send_response(200) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.end_headers() + self.wfile.write(json.dumps(d).encode('utf-8')) + except: + logger.exception('Error in GET') + self.send_error(400, explain=traceback.format_exc()) + return + + def do_POST(self): + try: + p = urllib.parse.urlparse(self.path) + + if p.path != self.prefix + '/v1/equivalent': + self.send_error(404) + return + + length = int(self.headers['content-length']) + data = json.loads(self.rfile.read(length).decode('utf-8')) + + with contextlib.closing(self.db.cursor()) as cursor: + cursor.execute(''' + SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND outhash=:outhash + ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END, + created ASC + LIMIT 1 + ''', {k: data[k] for k in ('method', 'outhash', 'taskhash')}) + + row = cursor.fetchone() + + if row is None or row['taskhash'] != data['taskhash']: + unihash = data['unihash'] + if row is not None: + unihash = row['unihash'] + + insert_data = { + 'method': data['method'], + 'outhash': data['outhash'], + 'taskhash': data['taskhash'], + 'unihash': unihash, + 'created': datetime.now() + } + + for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'): + if k in data: + insert_data[k] = data[k] + + cursor.execute('''INSERT INTO tasks_v1 (%s) VALUES (%s)''' % ( + ', '.join(sorted(insert_data.keys())), + ', '.join(':' + k for k in sorted(insert_data.keys()))), + insert_data) + + logger.info('Adding taskhash %s with unihash %s', data['taskhash'], unihash) + cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE id=:id', {'id': cursor.lastrowid}) + row = cursor.fetchone() + + self.db.commit() + + d = {k: row[k] for k in ('taskhash', 'method', 'unihash')} + + self.send_response(200) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.end_headers() + self.wfile.write(json.dumps(d).encode('utf-8')) + except: + logger.exception('Error in POST') + self.send_error(400, explain=traceback.format_exc()) + return + +def create_server(addr, db, prefix=''): + class Handler(HashEquivalenceServer): + pass + + Handler.prefix = prefix + Handler.db = db + db.row_factory = sqlite3.Row + + with contextlib.closing(db.cursor()) as cursor: + cursor.execute(''' + CREATE TABLE IF NOT EXISTS tasks_v1 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + method TEXT NOT NULL, + outhash TEXT NOT NULL, + taskhash TEXT NOT NULL, + unihash TEXT NOT NULL, + created DATETIME, + + -- Optional fields + owner TEXT, + PN TEXT, + PV TEXT, + PR TEXT, + task TEXT, + outhash_siginfo TEXT + ) + ''') + + logger.info('Starting server on %s', addr) + return HTTPServer(addr, Handler) diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py new file mode 100644 index 00000000000..806b54c5ebd --- /dev/null +++ b/bitbake/lib/hashserv/tests.py @@ -0,0 +1,141 @@ +#! /usr/bin/env python3 +# +# Copyright (C) 2018 Garmin Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import unittest +import threading +import sqlite3 +import hashlib +import urllib.request +import json +from . import create_server + +class TestHashEquivalenceServer(unittest.TestCase): + def setUp(self): + # Start an in memory hash equivalence server in the background bound to + # an ephemeral port + db = sqlite3.connect(':memory:', check_same_thread=False) + self.server = create_server(('localhost', 0), db) + self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1] + self.server_thread = threading.Thread(target=self.server.serve_forever) + self.server_thread.start() + + def tearDown(self): + # Shutdown server + s = getattr(self, 'server', None) + if s is not None: + self.server.shutdown() + self.server_thread.join() + self.server.server_close() + + def send_get(self, path): + url = '%s/%s' % (self.server_addr, path) + request = urllib.request.Request(url) + response = urllib.request.urlopen(request) + return json.loads(response.read().decode('utf-8')) + + def send_post(self, path, data): + headers = {'content-type': 'application/json'} + url = '%s/%s' % (self.server_addr, path) + request = urllib.request.Request(url, json.dumps(data).encode('utf-8'), headers) + response = urllib.request.urlopen(request) + return json.loads(response.read().decode('utf-8')) + + def test_create_hash(self): + # Simple test that hashes can be created + taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertIsNone(d, msg='Found unexpected task, %r' % d) + + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash, + }) + self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') + + def test_create_equivalent(self): + # Tests that a second reported task with the same outhash will be + # assigned the same unihash + taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4' + outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8' + unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash, + }) + self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') + + # Report a different task with the same outhash. The returned unihash + # should match the first task + taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4' + unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash2, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash2, + }) + self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') + + def test_duplicate_taskhash(self): + # Tests that duplicate reports of the same taskhash with different + # outhash & unihash always return the unihash from the first reported + # taskhash + taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash = '218e57509998197d570e2c98512d0105985dffc9' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash, + }) + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertEqual(d['unihash'], unihash) + + outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d' + unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash2, + 'unihash': unihash2 + }) + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertEqual(d['unihash'], unihash) + + outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4' + unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash3, + 'unihash': unihash3 + }) + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertEqual(d['unihash'], unihash) + + -- 2.20.1
WARNING: multiple messages have this Message-ID (diff)
From: Joshua Watt <jpewhacker@gmail.com> To: openembedded-core@lists.openembedded.org, bitbake-devel@lists.openembedded.org Subject: [PATCH v7 2/3] bitbake: hashserv: Add hash equivalence reference server Date: Fri, 4 Jan 2019 10:20:14 -0600 [thread overview] Message-ID: <20190104162015.456-3-JPEWhacker@gmail.com> (raw) In-Reply-To: <20190104162015.456-1-JPEWhacker@gmail.com> Implements a reference implementation of the hash equivalence server. This server has minimal dependencies (and no dependencies outside of the standard Python library), and implements the minimum required to be a conforming hash equivalence server. [YOCTO #13030] Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> --- bitbake/bin/bitbake-hashserv | 67 ++++++++++++++ bitbake/bin/bitbake-selftest | 2 + bitbake/lib/hashserv/__init__.py | 152 +++++++++++++++++++++++++++++++ bitbake/lib/hashserv/tests.py | 141 ++++++++++++++++++++++++++++ 4 files changed, 362 insertions(+) create mode 100755 bitbake/bin/bitbake-hashserv create mode 100644 bitbake/lib/hashserv/__init__.py create mode 100644 bitbake/lib/hashserv/tests.py diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv new file mode 100755 index 00000000000..c49397b73a5 --- /dev/null +++ b/bitbake/bin/bitbake-hashserv @@ -0,0 +1,67 @@ +#! /usr/bin/env python3 +# +# Copyright (C) 2018 Garmin Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import os +import sys +import logging +import argparse +import sqlite3 + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) + +import hashserv + +VERSION = "1.0.0" + +DEFAULT_HOST = '' +DEFAULT_PORT = 8686 + +def main(): + parser = argparse.ArgumentParser(description='HTTP Equivalence Reference Server. Version=%s' % VERSION) + parser.add_argument('--address', default=DEFAULT_HOST, help='Bind address (default "%(default)s")') + parser.add_argument('--port', type=int, default=DEFAULT_PORT, help='Bind port (default %(default)d)') + parser.add_argument('--prefix', default='', help='HTTP path prefix (default "%(default)s")') + parser.add_argument('--database', default='./hashserv.db', help='Database file (default "%(default)s")') + parser.add_argument('--log', default='WARNING', help='Set logging level') + + args = parser.parse_args() + + logger = logging.getLogger('hashserv') + + level = getattr(logging, args.log.upper(), None) + if not isinstance(level, int): + raise ValueError('Invalid log level: %s' % args.log) + + logger.setLevel(level) + console = logging.StreamHandler() + console.setLevel(level) + logger.addHandler(console) + + db = sqlite3.connect(args.database) + + server = hashserv.create_server((args.address, args.port), db, args.prefix) + server.serve_forever() + return 0 + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) + diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest index c970dcae90c..99f1af910f4 100755 --- a/bitbake/bin/bitbake-selftest +++ b/bitbake/bin/bitbake-selftest @@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib import unittest try: import bb + import hashserv import layerindexlib except RuntimeError as exc: sys.exit(str(exc)) @@ -35,6 +36,7 @@ tests = ["bb.tests.codeparser", "bb.tests.parse", "bb.tests.persist_data", "bb.tests.utils", + "hashserv.tests", "layerindexlib.tests.layerindexobj", "layerindexlib.tests.restapi", "layerindexlib.tests.cooker"] diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py new file mode 100644 index 00000000000..46bca7cab32 --- /dev/null +++ b/bitbake/lib/hashserv/__init__.py @@ -0,0 +1,152 @@ +# Copyright (C) 2018 Garmin Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from http.server import BaseHTTPRequestHandler, HTTPServer +import contextlib +import urllib.parse +import sqlite3 +import json +import traceback +import logging +from datetime import datetime + +logger = logging.getLogger('hashserv') + +class HashEquivalenceServer(BaseHTTPRequestHandler): + def log_message(self, f, *args): + logger.debug(f, *args) + + def do_GET(self): + try: + p = urllib.parse.urlparse(self.path) + + if p.path != self.prefix + '/v1/equivalent': + self.send_error(404) + return + + query = urllib.parse.parse_qs(p.query, strict_parsing=True) + method = query['method'][0] + taskhash = query['taskhash'][0] + + d = None + with contextlib.closing(self.db.cursor()) as cursor: + cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1', + {'method': method, 'taskhash': taskhash}) + + row = cursor.fetchone() + + if row is not None: + logger.debug('Found equivalent task %s', row['taskhash']) + d = {k: row[k] for k in ('taskhash', 'method', 'unihash')} + + self.send_response(200) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.end_headers() + self.wfile.write(json.dumps(d).encode('utf-8')) + except: + logger.exception('Error in GET') + self.send_error(400, explain=traceback.format_exc()) + return + + def do_POST(self): + try: + p = urllib.parse.urlparse(self.path) + + if p.path != self.prefix + '/v1/equivalent': + self.send_error(404) + return + + length = int(self.headers['content-length']) + data = json.loads(self.rfile.read(length).decode('utf-8')) + + with contextlib.closing(self.db.cursor()) as cursor: + cursor.execute(''' + SELECT taskhash, method, unihash FROM tasks_v1 WHERE method=:method AND outhash=:outhash + ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END, + created ASC + LIMIT 1 + ''', {k: data[k] for k in ('method', 'outhash', 'taskhash')}) + + row = cursor.fetchone() + + if row is None or row['taskhash'] != data['taskhash']: + unihash = data['unihash'] + if row is not None: + unihash = row['unihash'] + + insert_data = { + 'method': data['method'], + 'outhash': data['outhash'], + 'taskhash': data['taskhash'], + 'unihash': unihash, + 'created': datetime.now() + } + + for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'): + if k in data: + insert_data[k] = data[k] + + cursor.execute('''INSERT INTO tasks_v1 (%s) VALUES (%s)''' % ( + ', '.join(sorted(insert_data.keys())), + ', '.join(':' + k for k in sorted(insert_data.keys()))), + insert_data) + + logger.info('Adding taskhash %s with unihash %s', data['taskhash'], unihash) + cursor.execute('SELECT taskhash, method, unihash FROM tasks_v1 WHERE id=:id', {'id': cursor.lastrowid}) + row = cursor.fetchone() + + self.db.commit() + + d = {k: row[k] for k in ('taskhash', 'method', 'unihash')} + + self.send_response(200) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.end_headers() + self.wfile.write(json.dumps(d).encode('utf-8')) + except: + logger.exception('Error in POST') + self.send_error(400, explain=traceback.format_exc()) + return + +def create_server(addr, db, prefix=''): + class Handler(HashEquivalenceServer): + pass + + Handler.prefix = prefix + Handler.db = db + db.row_factory = sqlite3.Row + + with contextlib.closing(db.cursor()) as cursor: + cursor.execute(''' + CREATE TABLE IF NOT EXISTS tasks_v1 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + method TEXT NOT NULL, + outhash TEXT NOT NULL, + taskhash TEXT NOT NULL, + unihash TEXT NOT NULL, + created DATETIME, + + -- Optional fields + owner TEXT, + PN TEXT, + PV TEXT, + PR TEXT, + task TEXT, + outhash_siginfo TEXT + ) + ''') + + logger.info('Starting server on %s', addr) + return HTTPServer(addr, Handler) diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py new file mode 100644 index 00000000000..806b54c5ebd --- /dev/null +++ b/bitbake/lib/hashserv/tests.py @@ -0,0 +1,141 @@ +#! /usr/bin/env python3 +# +# Copyright (C) 2018 Garmin Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import unittest +import threading +import sqlite3 +import hashlib +import urllib.request +import json +from . import create_server + +class TestHashEquivalenceServer(unittest.TestCase): + def setUp(self): + # Start an in memory hash equivalence server in the background bound to + # an ephemeral port + db = sqlite3.connect(':memory:', check_same_thread=False) + self.server = create_server(('localhost', 0), db) + self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1] + self.server_thread = threading.Thread(target=self.server.serve_forever) + self.server_thread.start() + + def tearDown(self): + # Shutdown server + s = getattr(self, 'server', None) + if s is not None: + self.server.shutdown() + self.server_thread.join() + self.server.server_close() + + def send_get(self, path): + url = '%s/%s' % (self.server_addr, path) + request = urllib.request.Request(url) + response = urllib.request.urlopen(request) + return json.loads(response.read().decode('utf-8')) + + def send_post(self, path, data): + headers = {'content-type': 'application/json'} + url = '%s/%s' % (self.server_addr, path) + request = urllib.request.Request(url, json.dumps(data).encode('utf-8'), headers) + response = urllib.request.urlopen(request) + return json.loads(response.read().decode('utf-8')) + + def test_create_hash(self): + # Simple test that hashes can be created + taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' + outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' + unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertIsNone(d, msg='Found unexpected task, %r' % d) + + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash, + }) + self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') + + def test_create_equivalent(self): + # Tests that a second reported task with the same outhash will be + # assigned the same unihash + taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4' + outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8' + unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash, + }) + self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') + + # Report a different task with the same outhash. The returned unihash + # should match the first task + taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4' + unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash2, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash2, + }) + self.assertEqual(d['unihash'], unihash, 'Server returned bad unihash') + + def test_duplicate_taskhash(self): + # Tests that duplicate reports of the same taskhash with different + # outhash & unihash always return the unihash from the first reported + # taskhash + taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a' + outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e' + unihash = '218e57509998197d570e2c98512d0105985dffc9' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash, + 'unihash': unihash, + }) + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertEqual(d['unihash'], unihash) + + outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d' + unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash2, + 'unihash': unihash2 + }) + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertEqual(d['unihash'], unihash) + + outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4' + unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603' + d = self.send_post('v1/equivalent', { + 'taskhash': taskhash, + 'method': 'TestMethod', + 'outhash': outhash3, + 'unihash': unihash3 + }) + + d = self.send_get('v1/equivalent?method=TestMethod&taskhash=%s' % taskhash) + self.assertEqual(d['unihash'], unihash) + + -- 2.20.1
next prev parent reply other threads:[~2019-01-04 16:20 UTC|newest] Thread overview: 158+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-07-16 20:37 [RFC 0/9] Hash Equivalency Server Joshua Watt 2018-07-16 20:37 ` [RFC 1/9] bitbake-worker: Pass taskhash as runtask parameter Joshua Watt 2018-07-16 20:37 ` [RFC 2/9] siggen: Split out stampfile hash fetch Joshua Watt 2018-07-16 20:37 ` [RFC 3/9] siggen: Split out task depend ID Joshua Watt 2018-07-16 20:37 ` [RFC 4/9] runqueue: Track task dependency ID Joshua Watt 2018-07-16 20:37 ` [RFC 5/9] runqueue: Pass dependency ID to task Joshua Watt 2018-07-16 20:37 ` [RFC 6/9] runqueue: Pass dependency ID to hash validate Joshua Watt 2018-07-16 20:37 ` [RFC 7/9] classes/sstate: Handle depid in hash check Joshua Watt 2018-07-16 20:37 ` [RFC 8/9] hashserver: Add initial reference server Joshua Watt 2018-07-17 12:11 ` Richard Purdie 2018-07-17 12:11 ` [bitbake-devel] " Richard Purdie 2018-07-17 13:44 ` Joshua Watt 2018-07-17 13:44 ` [bitbake-devel] " Joshua Watt 2018-07-18 13:53 ` Joshua Watt 2018-07-18 13:53 ` [bitbake-devel] " Joshua Watt 2018-07-16 20:37 ` [RFC 9/9] sstate: Implement hash equivalence sstate Joshua Watt 2018-08-09 22:08 ` [RFC v2 00/16] Hash Equivalency Server Joshua Watt 2018-08-09 22:08 ` [RFC v2 01/16] bitbake: fork: Add os.fork() wrappers Joshua Watt 2018-08-09 22:08 ` [RFC v2 02/16] bitbake: persist_data: Fix leaking cursors causing deadlock Joshua Watt 2018-08-09 22:08 ` [RFC v2 03/16] bitbake: persist_data: Add key constraints Joshua Watt 2018-08-09 22:08 ` [RFC v2 04/16] bitbake: persist_data: Enable Write Ahead Log Joshua Watt 2018-08-09 22:08 ` [RFC v2 05/16] bitbake: persist_data: Disable enable_shared_cache Joshua Watt 2018-08-09 22:08 ` [RFC v2 06/16] bitbake: persist_data: Close databases across fork Joshua Watt 2018-08-09 22:08 ` [RFC v2 07/16] bitbake: tests/persist_data: Add tests Joshua Watt 2018-08-09 22:08 ` [RFC v2 08/16] bitbake: bitbake-worker: Pass taskhash as runtask parameter Joshua Watt 2018-08-09 22:08 ` [RFC v2 09/16] bitbake: siggen: Split out stampfile hash fetch Joshua Watt 2018-08-09 22:08 ` [RFC v2 10/16] bitbake: siggen: Split out task depend ID Joshua Watt 2018-08-09 22:08 ` [RFC v2 11/16] bitbake: runqueue: Track task dependency ID Joshua Watt 2018-08-09 22:08 ` [RFC v2 12/16] bitbake: runqueue: Pass dependency ID to task Joshua Watt 2018-08-09 22:08 ` [RFC v2 13/16] bitbake: runqueue: Pass dependency ID to hash validate Joshua Watt 2018-08-09 22:08 ` [RFC v2 14/16] classes/sstate: Handle depid in hash check Joshua Watt 2018-08-09 22:08 ` [RFC v2 15/16] bitbake: hashserv: Add hash equivalence reference server Joshua Watt 2018-08-09 22:08 ` [RFC v2 16/16] sstate: Implement hash equivalence sstate Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 00/17] Hash Equivalency Server Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 01/17] bitbake: fork: Add os.fork() wrappers Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 02/17] bitbake: persist_data: Fix leaking cursors causing deadlock Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 03/17] bitbake: persist_data: Add key constraints Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 04/17] bitbake: persist_data: Enable Write Ahead Log Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 05/17] bitbake: persist_data: Disable enable_shared_cache Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 06/17] bitbake: persist_data: Close databases across fork Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 07/17] bitbake: tests/persist_data: Add tests Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 08/17] bitbake: bitbake-worker: Pass taskhash as runtask parameter Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 09/17] bitbake: siggen: Split out stampfile hash fetch Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 10/17] bitbake: siggen: Split out task depend ID Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-05 22:50 ` [OE-core][PATCH " Richard Purdie 2018-12-05 22:50 ` [bitbake-devel] [PATCH " Richard Purdie 2018-12-06 14:58 ` [OE-core][PATCH " Joshua Watt 2018-12-06 14:58 ` [bitbake-devel] [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 11/17] bitbake: runqueue: Track task dependency ID Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 12/17] bitbake: runqueue: Pass dependency ID to task Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 13/17] bitbake: runqueue: Pass dependency ID to hash validate Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-05 22:52 ` [OE-core][PATCH " Richard Purdie 2018-12-05 22:52 ` [bitbake-devel] [PATCH " Richard Purdie 2018-12-04 3:42 ` [OE-core][PATCH v3 14/17] classes/sstate: Handle depid in hash check Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 15/17] bitbake: hashserv: Add hash equivalence reference server Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 16/17] sstate: Implement hash equivalence sstate Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-04 3:42 ` [OE-core][PATCH v3 17/17] classes/image-buildinfo: Remove unused argument Joshua Watt 2018-12-04 3:42 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 00/10] Hash Equivalency Server Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 01/10] bitbake: fork: Add os.fork() wrappers Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 02/10] bitbake: persist_data: Close databases across fork Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 03/10] bitbake: tests/persist_data: Add tests Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 04/10] bitbake: siggen: Split out task unique hash Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 05/10] bitbake: runqueue: Track " Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 06/10] bitbake: runqueue: Pass unique hash to task Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 07/10] bitbake: runqueue: Pass unique hash to hash validate Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 16:24 ` [OE-core] " Richard Purdie 2018-12-18 16:24 ` Richard Purdie 2018-12-18 16:31 ` [OE-core] " Joshua Watt 2018-12-18 16:31 ` Joshua Watt 2018-12-18 15:30 ` [OE-core][PATCH v4 08/10] classes/sstate: Handle unihash in hash check Joshua Watt 2018-12-18 15:30 ` [PATCH " Joshua Watt 2018-12-18 15:31 ` [OE-core][PATCH v4 09/10] bitbake: hashserv: Add hash equivalence reference server Joshua Watt 2018-12-18 15:31 ` [PATCH " Joshua Watt 2018-12-18 15:31 ` [OE-core][PATCH v4 10/10] sstate: Implement hash equivalence sstate Joshua Watt 2018-12-18 15:31 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 0/8] Hash Equivalency Server Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 1/8] bitbake: tests/persist_data: Add tests Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 2/8] bitbake: siggen: Split out task unique hash Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 3/8] bitbake: runqueue: Track " Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2019-01-05 7:49 ` [OE-core] " Alejandro Hernandez 2019-01-05 7:49 ` Alejandro Hernandez 2019-01-06 3:09 ` [OE-core] " Joshua Watt 2019-01-06 3:09 ` Joshua Watt 2019-01-07 6:52 ` [OE-core] " Alejandro Hernandez 2019-01-07 6:52 ` Alejandro Hernandez 2019-01-07 16:16 ` [OE-core] " akuster808 2019-01-07 16:16 ` akuster808 2019-01-07 16:40 ` [OE-core] " Joshua Watt 2019-01-07 16:40 ` Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 4/8] bitbake: runqueue: Pass unique hash to task Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 5/8] bitbake: runqueue: Pass unique hash to hash validate Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 6/8] classes/sstate: Handle unihash in hash check Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 7/8] bitbake: hashserv: Add hash equivalence reference server Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:10 ` [OE-core][PATCH v5 8/8] sstate: Implement hash equivalence sstate Joshua Watt 2018-12-19 3:10 ` [PATCH " Joshua Watt 2018-12-19 3:33 ` ✗ patchtest: failure for Hash Equivalency Server (rev3) Patchwork 2019-01-04 2:42 ` [OE-core][PATCH v6 0/3] Hash Equivalency Server Joshua Watt 2019-01-04 2:42 ` [PATCH " Joshua Watt 2019-01-04 2:42 ` [OE-core][PATCH v6 1/3] classes/sstate: Handle unihash in hash check Joshua Watt 2019-01-04 2:42 ` [PATCH " Joshua Watt 2019-01-04 7:01 ` [OE-core][PATCH " Richard Purdie 2019-01-04 7:01 ` [bitbake-devel] [PATCH " Richard Purdie 2019-01-04 2:42 ` [OE-core][PATCH v6 2/3] bitbake: hashserv: Add hash equivalence reference server Joshua Watt 2019-01-04 2:42 ` [PATCH " Joshua Watt 2019-01-04 2:42 ` [OE-core][PATCH v6 3/3] sstate: Implement hash equivalence sstate Joshua Watt 2019-01-04 2:42 ` [PATCH " Joshua Watt 2019-01-04 16:20 ` [OE-core][PATCH v7 0/3] Hash Equivalency Server Joshua Watt 2019-01-04 16:20 ` [PATCH " Joshua Watt 2019-01-04 16:20 ` [OE-core][PATCH v7 1/3] classes/sstate: Handle unihash in hash check Joshua Watt 2019-01-04 16:20 ` [PATCH " Joshua Watt 2019-01-04 16:20 ` Joshua Watt [this message] 2019-01-04 16:20 ` [PATCH v7 2/3] bitbake: hashserv: Add hash equivalence reference server Joshua Watt 2019-01-04 16:20 ` [OE-core][PATCH v7 3/3] sstate: Implement hash equivalence sstate Joshua Watt 2019-01-04 16:20 ` [PATCH " Joshua Watt 2019-01-08 6:29 ` [OE-core][PATCH " Jacob Kroon 2019-01-08 6:29 ` [bitbake-devel] [PATCH " Jacob Kroon 2019-01-09 17:09 ` [OE-core][PATCH " Joshua Watt 2019-01-09 17:09 ` [bitbake-devel] [PATCH " Joshua Watt 2019-01-11 20:39 ` [OE-core][PATCH " Peter Kjellerstedt 2019-01-11 20:39 ` [bitbake-devel] [PATCH " Peter Kjellerstedt 2019-01-04 16:33 ` ✗ patchtest: failure for Hash Equivalency Server (rev5) Patchwork 2019-01-04 3:03 ` ✗ patchtest: failure for Hash Equivalency Server (rev4) Patchwork 2018-12-18 16:03 ` ✗ patchtest: failure for Hash Equivalency Server (rev2) Patchwork 2018-12-04 4:05 ` ✗ patchtest: failure for Hash Equivalency Server Patchwork
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20190104162015.456-3-JPEWhacker@gmail.com \ --to=jpewhacker@gmail.com \ --cc=bitbake-devel@lists.openembedded.org \ --cc=openembedded-core@lists.openembedded.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.