* [OSSTEST PATCH 00/25] Reporting improvements
@ 2015-06-17 16:44 Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
` (25 more replies)
0 siblings, 26 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Campbell
Most of the early part of this series is straightforward (and acked)
and could go in whenever.
Patches 15-24 introduce sg-report-host-history. It's probably easiest
to simply review the final version of the script, which can be found
below. The history is preserved mostly for archeological purposes.
Ian.
#!/usr/bin/perl -w
# This is part of "osstest", an automated testing framework for Xen.
# Copyright (C) 2009-2013 Citrix Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
use strict qw(vars);
use DBI;
use Osstest;
use IO::Handle;
use HTML::Entities;
use POSIX;
use Osstest::Executive qw(:DEFAULT :colours);
our $limit= 200;
our $flightlimit;
our $htmlout = ".";
our @blessings;
open DEBUG, ">/dev/null";
my $namecond= "(name = 'host' or name like '%_host')";
csreadconfig();
while (@ARGV && $ARGV[0] =~ m/^-/) {
$_= shift @ARGV;
last if m/^--?$/;
if (m/^--(limit)\=([1-9]\d*)$/) {
$$1= $2;
} elsif (m/^--flight-limit\=([1-9]\d*)$/) {
$flightlimit= $1;
} elsif (m/^--blessings?=(.*)$/) {
push @blessings, split ',', $1;
} elsif (m/^--html-dir=(.*)$/) {
$htmlout= $1;
} elsif (m/^--debug/) {
open DEBUG, ">&2" or die $!;
DEBUG->autoflush(1);
} else {
die "$_ ?";
}
}
@blessings= qw(real) if !@blessings;
@ARGV or die $!;
our $flightcond;
sub computeflightsrange () {
if (!$flightlimit) {
my $flagscond =
'('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
my $nhostsq = db_prepare(<<END);
SELECT count(*)
FROM resources r
WHERE restype='host'
AND EXISTS (SELECT 1
FROM hostflags f
WHERE f.hostname=r.resname
AND $flagscond)
END
$nhostsq->execute();
my ($nhosts) = $nhostsq->fetchrow_array();
print DEBUG "COUNTED $nhosts hosts\n";
$flightlimit = $nhosts * $limit * 2;
}
my $minflightsq = db_prepare(<<END);
SELECT flight
FROM (
SELECT flight
FROM flights
ORDER BY flight DESC
LIMIT $flightlimit
) f
ORDER BY flight ASC
LIMIT 1
END
$minflightsq->execute();
my ($minflight) = $minflightsq->fetchrow_array();
$minflight //= 0;
$flightcond = "(flight > $minflight)";
}
sub jobquery ($$) {
my ($q, $jr) = @_;
$q->execute($jr->{flight}, $jr->{job});
return $q->fetchrow_hashref();
}
our %hosts;
sub mainquery () {
our $valcond = join " OR ", map { "val = ?" } keys %hosts;
our @params = keys %hosts;
our $runvarq //= db_prepare(<<END);
SELECT flight, job, name, val
FROM runvars
WHERE $namecond
AND ($valcond)
AND $flightcond
ORDER BY flight DESC
LIMIT ($limit * 3 + 100) * ?
END
push @params, scalar keys %hosts;
$runvarq->execute(@params);
print DEBUG "FIRST PASS\n";
while (my $jr= $runvarq->fetchrow_hashref()) {
print DEBUG "JOB $jr->{flight}.$jr->{job} ";
push @{ $hosts{$jr->{val}} }, $jr;
}
}
sub reporthost ($) {
my ($hostname) = @_;
die if $hostname =~ m/[^-_.+0-9a-z]/;
my $html_file= "$htmlout/$hostname.html";
open H, "> $html_file.new" or die "$html_file $!";
my $title= "host history $hostname\n";
$title= encode_entities($title);
print H "<html><head><title>$title</title></head><body>\n";
print H "<h1>$title</h1>\n";
print H "<table rules=all><tr>\n";
print H "<th>alloc testid</th><th>alloc completed</th>\n";
print H "<th>job finished</th>\n";
print H "<th>role</th>\n";
print H "<th>flight</th>\n";
print H "<th>branch</th><th>intended</th><th>blessing</th>\n";
print H "<th>job</th><th>failure</th>\n";
print H "</tr>\n";
our $endedq //= db_prepare(<<END);
SELECT finished, testid, status AS laststepstatus
FROM steps
WHERE flight=? AND job=? AND finished IS NOT NULL
ORDER BY finished DESC
LIMIT 1
END
our $infoq //= db_prepare(<<END);
SELECT blessing, branch, intended, status
FROM flights
JOIN jobs USING (flight)
WHERE flight=? AND job=?
END
our $allocdq //= db_prepare(<<END);
SELECT testid, finished, status
FROM steps
WHERE flight=? AND job=?
AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
ORDER BY finished ASC
LIMIT 1
END
my $inrows = $hosts{$hostname};
print DEBUG "FOUND ", (scalar @$inrows), " ROWS for $hostname\n";
my @rows;
foreach my $jr (@$inrows) {
print DEBUG "JOB $jr->{flight}.$jr->{job}\n";
my $endedrow = jobquery($endedq, $jr);
if (!$endedrow) {
print DEBUG "no-finished\n";
next;
}
print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
print DEBUG ".\n";
push @rows, { %$jr, %$endedrow };
}
@rows = sort { $b->{finished} <=> $a->{finished} } @rows;
$#rows = $limit-1 if @rows > $limit;
my $alternate = 0;
foreach my $jr (@rows) {
my $ir = jobquery($infoq, $jr);
my $ar = jobquery($allocdq, $jr);
my $altcolour = report_altcolour($alternate);
print H "<tr $altcolour>";
if (!defined $ar->{testid}) {
print H "<td bgcolor=\"$red\"></td>";
print H "<td>?</td>";
} else {
if ($ar->{status} eq 'pass') {
print H "<td>$ar->{testid}</td>";
print H "<td>", (show_abs_time $ar->{finished}), "</td>";
} elsif ($ar->{status} eq 'running') {
print H "<td bgcolor=\"$blue\">$ar->{testid}</td>";
print H "<td>(incomplete)</td>";
} else {
print H "<td bgcolor=\"$red\">$ar->{testid}</td>";
print H "<td>$ar->{status}</td>";
}
}
print H "\n";
print H "<td>", (show_abs_time $jr->{finished}), "</td>\n";
print H "<td>", $jr->{name}, "</td>\n";
my $url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}";
print H "<td><a href=\"$url\">$jr->{flight}</a></td>\n";
$url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}/".
encode_entities($jr->{job})."/";
print H "<td>$ir->{branch}</td>";
print H "<td>$ir->{intended}</td>";
print H "<td>";
print H $ir->{blessing} unless $ir->{blessing} eq 'running';
print H "</td>";
print H "<td><a href=\"$url\">$jr->{job}</td>\n";
my $ri = report_run_getinfo({ %$jr, %$ir });
print H "<td bgcolor=\"$ri->{Colour}\">$ri->{Summary}</td>\n";
print H "</tr>\n\n";
$alternate ^= 1;
}
print H "</table></body></html>\n";
close H or die $!;
rename "$html_file.new", "$html_file" or die "$html_file $!";
}
db_retry($dbh_tests, [qw(flights resources)], sub {
computeflightsrange();
});
$dbh_tests->do("SET LOCAL enable_seqscan=false");
# Otherwise the PostgreSQL query planner likes to do a complete scan
# of the runvars table, rather than walking backwards through the
# flights until it has what we've told it is enough.
foreach my $host (@ARGV) {
if ($host =~ m/^flight:/) {
my $flight=$'; #';
db_retry($dbh_tests, [qw(flights)], sub {
our $hostsinflightq //= db_prepare(<<END);
SELECT DISTINCT val
FROM runvars
WHERE flight=?
AND (name = 'host' or name like '%_host')
END
$hostsinflightq->execute($flight);
while (my $row = $hostsinflightq->fetchrow_hashref()) {
$hosts{$row->{val}} = [ ];
}
});
} elsif ($host !~ m/:/) {
$hosts{$host} = [ ];
} else {
die "$host ?";
}
}
exit 0 unless %hosts;
db_retry($dbh_tests, [qw(flights)], sub {
mainquery();
});
foreach my $host (sort keys %hosts) {
db_retry($dbh_tests, [qw(flights)], sub {
reporthost $host;
});
}
^ permalink raw reply [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
@ 2015-06-17 16:44 ` Ian Jackson
2015-06-18 15:33 ` Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare Ian Jackson
` (24 subsequent siblings)
25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
No callers, so no functional change, as yet.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
Osstest.pm | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/Osstest.pm b/Osstest.pm
index e8bd77b..7d6949f 100644
--- a/Osstest.pm
+++ b/Osstest.pm
@@ -35,7 +35,7 @@ BEGIN {
getmethod
postfork
$dbh_tests db_retry db_retry_retry db_retry_abort
- db_begin_work
+ db_begin_work db_prepare
ensuredir get_filecontents_core_quiet system_checked
nonempty visible_undef show_abs_time
);
@@ -268,6 +268,13 @@ sub db_retry ($$$;$$) {
return $r;
}
+sub db_prepare ($) {
+ # caller must ensure global filehandle DEBUG is open
+ my ($stmt) = @_;
+ print ::DEBUG "DB PREPARING:\n$stmt\n";
+ return $dbh_tests->prepare($stmt);
+}
+
sub postfork () {
$mjobdb->jobdb_postfork();
}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
@ 2015-06-17 16:44 ` Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 03/25] cs-bisection-step: " Ian Jackson
` (23 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
sg-report-job-history | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/sg-report-job-history b/sg-report-job-history
index 409e3d5..e285caa 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -67,7 +67,7 @@ END
die "$flight ? @$branches ?" if @$branches!=1;
@branches= @$branches;
- my $selectq= $dbh_tests->prepare(<<END);
+ my $selectq= db_prepare(<<END);
SELECT job FROM jobs WHERE flight=? ORDER BY JOB
END
$selectq->execute($flight);
@@ -84,7 +84,7 @@ if (defined($flight)) {
push @jobs, $job;
}
-our $failstepq= $dbh_tests->prepare(<<END);
+our $failstepq= db_prepare(<<END);
SELECT * FROM steps
WHERE flight=? AND job=?
AND status!='pass'
@@ -120,7 +120,7 @@ sub run_getinfo ($) {
}
}
-our $revisionsq= $dbh_tests->prepare(<<END);
+our $revisionsq= db_prepare(<<END);
SELECT * FROM runvars
WHERE flight=? AND job=?
AND name LIKE E'built\\_revision\\_\%'
@@ -136,7 +136,7 @@ sub add_revisions ($$$$) {
}
}
-our $buildsq= $dbh_tests->prepare(<<END);
+our $buildsq= db_prepare(<<END);
SELECT * FROM runvars
WHERE flight=? AND job=?
AND name LIKE E'\%buildjob'
@@ -171,13 +171,13 @@ END
WHERE ($cond)
ORDER BY flight DESC
END
- my $flightsq= $dbh_tests->prepare(<<END);
+ my $flightsq= db_prepare(<<END);
SELECT * $fromstuff
LIMIT $limit
END
$flightsq->execute(@params);
- my $hostsq= $dbh_tests->prepare(<<END);
+ my $hostsq= db_prepare(<<END);
SELECT DISTINCT name
FROM runvars
JOIN flights USING (flight)
@@ -195,7 +195,7 @@ END
push @hostvarcols, $hostvar;
}
- my $hostq= $dbh_tests->prepare(<<END);
+ my $hostq= db_prepare(<<END);
SELECT val FROM runvars WHERE flight=? AND job=? AND name=?
END
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 03/25] cs-bisection-step: Use db_prepare
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare Ian Jackson
@ 2015-06-17 16:44 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 04/25] sg-report-flight: " Ian Jackson
` (22 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
And do away with one ad-hoc statement dump.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
cs-bisection-step | 21 ++++++++++-----------
1 file changed, 10 insertions(+), 11 deletions(-)
diff --git a/cs-bisection-step b/cs-bisection-step
index 67a92b7..bf41f81 100755
--- a/cs-bisection-step
+++ b/cs-bisection-step
@@ -203,7 +203,7 @@ END
AND flight = ?
END
- my $sth= $dbh_tests->prepare(<<END);
+ my $sth= db_prepare(<<END);
SELECT url.val AS uval,
rev.val AS rval,
@@ -303,7 +303,7 @@ sub findbasics () {
? "flight = $specflights{$fp}" : "TRUE";
}
- my $latestq= $dbh_tests->prepare(<<END);
+ my $latestq= db_prepare(<<END);
SELECT flights.flight, steps.status,
$flight_is_not_broken AS notbroken
FROM flights JOIN steps USING (flight)
@@ -316,7 +316,7 @@ sub findbasics () {
$maxflight_cond
ORDER BY notbroken DESC, flights.started DESC
END
- my $basisq= $dbh_tests->prepare(<<END);
+ my $basisq= db_prepare(<<END);
SELECT * FROM flights JOIN steps USING (flight)
WHERE job = ?
AND testid = ?
@@ -553,14 +553,14 @@ END
print STDERR "Searching for test results:\n";
- my $stepq= $dbh_tests->prepare("
+ my $stepq= db_prepare("
SELECT * FROM steps
WHERE flight = ?
AND job = ?
AND testid = ?
");
- my $jobq= $dbh_tests->prepare("
+ my $jobq= db_prepare("
SELECT * FROM flights JOIN jobs USING (flight)
WHERE job = ?
AND $blessingscond
@@ -1059,7 +1059,7 @@ sub preparejob ($$$) {
END
my (@trevisions) = split / /, $choose->{Rtuple};
- my $treeq= $dbh_tests->prepare(<<END);
+ my $treeq= db_prepare(<<END);
SELECT name FROM runvars
WHERE flight=? AND job=?
AND name = ?
@@ -1082,7 +1082,7 @@ END
# Check for subjobs:
- my $jobq= $dbh_tests->prepare(<<END);
+ my $jobq= db_prepare(<<END);
SELECT name, val FROM runvars
WHERE flight=? AND job=?
AND name LIKE '%job';
@@ -1156,7 +1156,7 @@ END
$dbh_tests->do("DROP TABLE bisection_runvars");
- my $jobsetq= $dbh_tests->prepare(<<END);
+ my $jobsetq= db_prepare(<<END);
UPDATE runvars SET val=?
WHERE flight=? AND job=? AND name=?
END
@@ -1185,7 +1185,7 @@ sub populateflight () {
}
my $removehosts=
- my $addhost= $dbh_tests->prepare(<<END);
+ my $addhost= db_prepare(<<END);
INSERT INTO runvars (flight,job,name,val,synth)
VALUES (?, ?, ?, ?, 'f')
END
@@ -1244,8 +1244,7 @@ END
LIMIT $maxequalflights + 1;
END
- print DEBUG "equalflightsq:\n$equalflightsqt";
- my $equalflightsq = $dbh_tests->prepare($equalflightsqt);
+ my $equalflightsq = db_prepare($equalflightsqt);
db_retry($popflight,'constructing', $dbh_tests,[qw(flights)], sub {
print STDERR "Checking for flail (since $repro_firstflight)...\n";
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 04/25] sg-report-flight: Use db_prepare
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (2 preceding siblings ...)
2015-06-17 16:44 ` [OSSTEST PATCH 03/25] cs-bisection-step: " Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement Ian Jackson
` (21 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
And do away with two commented-out ad-hoc statement dumps.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
sg-report-flight | 28 +++++++++++++---------------
1 file changed, 13 insertions(+), 15 deletions(-)
diff --git a/sg-report-flight b/sg-report-flight
index 80777af..bdaf1c8 100755
--- a/sg-report-flight
+++ b/sg-report-flight
@@ -163,7 +163,7 @@ sub findaflight ($$$$$) {
return undef;
}
- my $checkq= $dbh_tests->prepare(<<END);
+ my $checkq= db_prepare(<<END);
SELECT status FROM steps WHERE flight=? AND job=? AND testid=?
END
@@ -188,11 +188,10 @@ END
) AS sub
ORDER BY blessing ASC, flight DESC
END
- #print DEBUG "===\n$flightsq\n===\n";
- $flightsq= $dbh_tests->prepare($flightsq);
+ $flightsq= db_prepare($flightsq);
$flightsq->execute(@blessings);
- my $buildflightsq= $dbh_tests->prepare(<<END);
+ my $buildflightsq= db_prepare(<<END);
SELECT val FROM runvars
WHERE flight = ?
AND name LIKE '%buildjob'
@@ -204,10 +203,9 @@ END
AND name=?
GROUP BY flight, job, val
END
- #print DEBUG "===\n$mismatchq\n===\n";
- $revisionsq= $dbh_tests->prepare($revisionsq);
+ $revisionsq= db_prepare($revisionsq);
- my $revisionsosstestq= $dbh_tests->prepare(<<END);
+ my $revisionsosstestq= db_prepare(<<END);
SELECT harness AS val FROM flights_harness_touched
WHERE flight=?
END
@@ -319,7 +317,7 @@ END
};
$jobs= [ sort { $colmap->($a) cmp $colmap->($b) } @$jobs ];
- my $stepsq= $dbh_tests->prepare(<<END);
+ my $stepsq= db_prepare(<<END);
SELECT * FROM steps
WHERE flight=$flight AND job=?
ORDER BY stepno
@@ -441,7 +439,7 @@ END
});
}
- my $revh= $dbh_tests->prepare(<<END);
+ my $revh= db_prepare(<<END);
SELECT * FROM runvars
WHERE flight=$flight AND job='$j->{job}'
AND name like 'built_revision_%'
@@ -623,7 +621,7 @@ sub justifyfailures ($;$) {
AND $blessingscond
LIMIT 1
END
- $anypassq= $dbh_tests->prepare($anypassq);
+ $anypassq= db_prepare($anypassq);
my $duration_estimator= duration_estimator($branch, $blessings[0]);
foreach my $failv (@failures) {
@@ -802,7 +800,7 @@ sub htmloutjob ($$) {
my $title= "Info on flight $fi->{Flight} job $job";
my $branch= $fi->{FlightInfo}{branch};
- our $htmlout_jobq ||= $dbh_tests->prepare(<<END);
+ our $htmlout_jobq ||= db_prepare(<<END);
SELECT * FROM jobs WHERE flight = ? AND job = ?
END
$htmlout_jobq->execute($fi->{Flight}, $job);
@@ -826,7 +824,7 @@ END
END
my %issteplog;
- my $stepsq= $dbh_tests->prepare(<<END);
+ my $stepsq= db_prepare(<<END);
SELECT * FROM steps
WHERE flight=? AND job=?
ORDER BY stepno ASC
@@ -892,7 +890,7 @@ END
<h2>Test control variables</h2>
<table rules=all><tr><th>Name</th><th>Value</th><th>Source</th></tr>
END
- my $varsq= $dbh_tests->prepare(<<END);
+ my $varsq= db_prepare(<<END);
SELECT * FROM runvars
WHERE flight=? AND job=?
ORDER BY synth, name
@@ -1086,7 +1084,7 @@ END
}
my $started_str = show_abs_time $fi->{FlightInfo}{started};
- our $htmlout_finishedq ||= $dbh_tests->prepare(<<END);
+ our $htmlout_finishedq ||= db_prepare(<<END);
SELECT
(SELECT max(finished) FROM steps WHERE flight=?) max,
(SELECT count(*) FROM steps WHERE finished IS NULL AND flight=?) unfinished
@@ -1110,7 +1108,7 @@ END
<td>Test harness revision(s):</td><td>
END
- our $htmlout_touchedq ||= $dbh_tests->prepare(<<END);
+ our $htmlout_touchedq ||= db_prepare(<<END);
SELECT harness FROM flights_harness_touched WHERE flight=?
END
$htmlout_touchedq->execute($fi->{Flight});
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (3 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 04/25] sg-report-flight: " Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql Ian Jackson
` (20 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
sg-report-job-history | 2 ++
1 file changed, 2 insertions(+)
diff --git a/sg-report-job-history b/sg-report-job-history
index e285caa..f5f3496 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -150,6 +150,8 @@ sub altcolour ($) {
sub processjobbranch ($$) {
my ($j,$bra) = @_;
+ print DEBUG "processjobbranch('$j',", ($bra ? "'$bra'" : 'undef'), ")\n";
+
my %rev_grid_col;
my @rev_grid_cols;
my @test_rows;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (4 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short Ian Jackson
` (19 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
No functional change apart from slightly better debug output.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
sg-report-job-history | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/sg-report-job-history b/sg-report-job-history
index f5f3496..e4f35bf 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -160,9 +160,7 @@ sub processjobbranch ($$) {
my $cond = "job = ? AND $blessingscond";
my (@params) = ($j, @blessings);
if (defined $bra) {
- $cond .= <<END;
- AND branch = ?
-END
+ $cond .= " AND branch = ?";
push @params, $bra;
}
my $limit= 100;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (5 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 08/25] sg-report-job-history: Show start time Ian Jackson
` (18 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
If there have been less than 99 relevant flights, the inner SELECT (to
determine the minimum flight number) would return NULL. And anything
> NULL is NULL and NULL is treated as false. So the host runvar
identification would break.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
sg-report-job-history | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/sg-report-job-history b/sg-report-job-history
index e4f35bf..76cb146 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -182,10 +182,11 @@ END
FROM runvars
JOIN flights USING (flight)
WHERE ($cond)
- AND flight >= (
+ AND flight >= COALESCE(
+ (
SELECT flight $fromstuff
LIMIT 1 OFFSET $offset
- )
+ ), 0)
ORDER BY name;
END
$hostsq->execute(@params, @params); # sql text contains $cond twice
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 08/25] sg-report-job-history: Show start time
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (6 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm Ian Jackson
` (17 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
sg-report-job-history | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/sg-report-job-history b/sg-report-job-history
index 76cb146..bbb85fd 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -251,7 +251,8 @@ END
print H "<html><head><title>$title</title></head><body>\n";
print H "<h1>$title</h1>\n";
print H "<table rules=all>";
- print H "<tr><th>flight</th><th>branch</th><th>failure</th>\n";
+ print H "<tr><th>started</th><th>flight</th>",
+ "<th>branch</th><th>failure</th>\n";
print H "<th>", (join ", ", @hostvarcols), "</th>\n";
foreach my $c (@rev_grid_cols) {
print H "<th>".encode_entities($c)."</th>\n";
@@ -264,6 +265,10 @@ END
my $colour= "bgcolor=\"$r->{Colour}\"";
my $altcolour= altcolour($alternate);
print H "<tr $altcolour>";
+ my $started = $r->{Flight}{started};
+ print H "<td>";
+ print H show_abs_time $started if $started;
+ print H "</td>\n";
my $flt= $r->{Flight}{flight};
$url= "$c{ReportHtmlPubBaseUrl}/$flt";
print H "<td><a href=\"$url\">$flt</a></td>\n";
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (7 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 08/25] sg-report-job-history: Show start time Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue Ian Jackson
` (16 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
This is to support a forthcoming sg-report-host-history.
No functional change.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
Osstest/Executive.pm | 45 +++++++++++++++++++++++++++++++++++++++++++++
sg-report-job-history | 48 ++++--------------------------------------------
2 files changed, 49 insertions(+), 44 deletions(-)
diff --git a/Osstest/Executive.pm b/Osstest/Executive.pm
index 118a91a..90c615a 100644
--- a/Osstest/Executive.pm
+++ b/Osstest/Executive.pm
@@ -45,6 +45,7 @@ BEGIN {
@ISA = qw(Exporter);
@EXPORT = qw(get_harness_rev grabrepolock_reexec
findtask @all_lock_tables
+ report_run_getinfo report_altcolour
tcpconnect_queuedaemon plan_search
alloc_resources alloc_resources_rollback_begin_work
resource_check_allocated resource_shared_mark_ready
@@ -188,6 +189,50 @@ sub opendb ($) {
return $dbh;
}
+#---------- history reporting ----------
+
+our $green= '#008800';
+our $red= '#ff8888';
+our $yellow= '#ffff00';
+our $purple= '#ff00ff';
+
+sub report_run_getinfo ($) {
+ # $f is a joined flight/job row, must contain at least
+ # flight job status
+ my ($f) = @_;
+ my $status= $f->{status};
+ if ($status eq 'pass') {
+ return { Summary => "($status)", Colour => $green };
+ } elsif ($status eq 'fail') {
+ our $failstepq //= db_prepare(<<END);
+ SELECT * FROM steps
+ WHERE flight=? AND job=?
+ AND status!='pass'
+ ORDER BY stepno
+ LIMIT 1
+END
+ $failstepq->execute($f->{flight}, $f->{job});
+ my $fs= $failstepq->fetchrow_hashref();
+ if (!defined $fs) {
+ return { Summary => "(unknown)", Colour => $yellow };
+ } elsif ($fs->{status} eq 'fail') {
+ return { Summary => "$fs->{testid}", Colour => $red };
+ } else {
+ return { Summary => "$fs->{testid} $fs->{status}",
+ Colour => $red };
+ }
+ } elsif ($status eq 'blocked') {
+ return { Summary => "blocked", Colour => $purple },
+ } else {
+ return { Summary => "($f->{status})", Colour => $yellow };
+ }
+}
+
+sub report_altcolour ($) {
+ my ($bool) = @_;
+ return "bgcolor=\"#".(qw(d0d0d0 ffffff))[$bool]."\"";
+}
+
#---------- host (and other resource) allocation ----------
our $taskid;
diff --git a/sg-report-job-history b/sg-report-job-history
index bbb85fd..18821b3 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -25,6 +25,7 @@ use IO::Handle;
use HTML::Entities;
use Osstest::TestSupport;
+use Osstest::Executive;
our (@blessings,@branches);
our $limit= 500;
@@ -84,42 +85,6 @@ if (defined($flight)) {
push @jobs, $job;
}
-our $failstepq= db_prepare(<<END);
- SELECT * FROM steps
- WHERE flight=? AND job=?
- AND status!='pass'
- ORDER BY stepno
- LIMIT 1
-END
-
-our $green= '#008800';
-our $red= '#ff8888';
-our $yellow= '#ffff00';
-our $purple= '#ff00ff';
-
-sub run_getinfo ($) {
- my ($f) = @_;
- my $status= $f->{status};
- if ($status eq 'pass') {
- return { Summary => "($status)", Colour => $green };
- } elsif ($status eq 'fail') {
- $failstepq->execute($f->{flight}, $f->{job});
- my $fs= $failstepq->fetchrow_hashref();
- if (!defined $fs) {
- return { Summary => "(unknown)", Colour => $yellow };
- } elsif ($fs->{status} eq 'fail') {
- return { Summary => "$fs->{testid}", Colour => $red };
- } else {
- return { Summary => "$fs->{testid} $fs->{status}",
- Colour => $red };
- }
- } elsif ($status eq 'blocked') {
- return { Summary => "blocked", Colour => $purple },
- } else {
- return { Summary => "($f->{status})", Colour => $yellow };
- }
-}
-
our $revisionsq= db_prepare(<<END);
SELECT * FROM runvars
WHERE flight=? AND job=?
@@ -142,11 +107,6 @@ our $buildsq= db_prepare(<<END);
AND name LIKE E'\%buildjob'
END
-sub altcolour ($) {
- my ($bool) = @_;
- return "bgcolor=\"#".(qw(d0d0d0 ffffff))[$bool]."\"";
-}
-
sub processjobbranch ($$) {
my ($j,$bra) = @_;
@@ -201,7 +161,7 @@ END
END
while (my $f= $flightsq->fetchrow_hashref()) {
- my $ri= run_getinfo($f);
+ my $ri= report_run_getinfo($f);
$ri->{Hosts} = [ ];
foreach my $hostvar (@hostvarcols) {
@@ -263,7 +223,7 @@ END
my @alt_revs= ('0')x $#rev_grid_cols;
foreach my $r (@test_rows) {
my $colour= "bgcolor=\"$r->{Colour}\"";
- my $altcolour= altcolour($alternate);
+ my $altcolour= report_altcolour($alternate);
print H "<tr $altcolour>";
my $started = $r->{Flight}{started};
print H "<td>";
@@ -286,7 +246,7 @@ END
!defined($v) ? 0 :
$last_revs[$i] eq $v;
$alt_revs[$i] ^= !$same;
- print H "<td ".altcolour($alt_revs[$i]).">";
+ print H "<td ".report_altcolour($alt_revs[$i]).">";
if (defined $v) {
my $vp= $v;
if (defined $lastrev && $v eq $lastrev) {
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (8 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
` (15 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
Osstest/Executive.pm | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/Osstest/Executive.pm b/Osstest/Executive.pm
index 90c615a..6edbfee 100644
--- a/Osstest/Executive.pm
+++ b/Osstest/Executive.pm
@@ -52,9 +52,9 @@ BEGIN {
duration_estimator
db_pg_dsn opendb opendb_state
);
- %EXPORT_TAGS = ( );
+ %EXPORT_TAGS = ( colours => [qw($green $red $yellow $purple $blue)] );
- @EXPORT_OK = qw();
+ @EXPORT_OK = @{ $EXPORT_TAGS{colours} };
}
# DATABASE TABLE LOCK HIERARCHY
@@ -195,6 +195,7 @@ our $green= '#008800';
our $red= '#ff8888';
our $yellow= '#ffff00';
our $purple= '#ff00ff';
+our $blue= '#0000ff';
sub report_run_getinfo ($) {
# $f is a joined flight/job row, must contain at least
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (9 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-18 8:44 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
` (14 subsequent siblings)
25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
Osstest/Executive.pm | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/Osstest/Executive.pm b/Osstest/Executive.pm
index 6edbfee..6c16fdd 100644
--- a/Osstest/Executive.pm
+++ b/Osstest/Executive.pm
@@ -204,7 +204,8 @@ sub report_run_getinfo ($) {
my $status= $f->{status};
if ($status eq 'pass') {
return { Summary => "($status)", Colour => $green };
- } elsif ($status eq 'fail') {
+ } elsif ($status eq 'fail' or $status eq 'broken') {
+ my $failcolour = $status eq 'fail' ? $red : $yellow;
our $failstepq //= db_prepare(<<END);
SELECT * FROM steps
WHERE flight=? AND job=?
@@ -217,10 +218,12 @@ END
if (!defined $fs) {
return { Summary => "(unknown)", Colour => $yellow };
} elsif ($fs->{status} eq 'fail') {
- return { Summary => "$fs->{testid}", Colour => $red };
+ return { Summary => "$fs->{testid}", Colour => $failcolour };
+ } elsif ($fs->{status} eq 'broken') {
+ return { Summary => "$fs->{testid} broken", Colour => $yellow };
} else {
return { Summary => "$fs->{testid} $fs->{status}",
- Colour => $red };
+ Colour => $failcolour };
}
} elsif ($status eq 'blocked') {
return { Summary => "blocked", Colour => $purple },
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 12/25] reporting: Move job histories into history/
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (10 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-18 8:45 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
` (13 subsequent siblings)
25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Replace some `.'s in HTML filenames with `/'s, making the directory
listings easier to deal with.
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
sg-report-flight | 4 ++--
sg-report-job-history | 10 ++++++----
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/sg-report-flight b/sg-report-flight
index bdaf1c8..d3c0166 100755
--- a/sg-report-flight
+++ b/sg-report-flight
@@ -813,8 +813,8 @@ END
<h1>$title</h1>
<ul>
<li><a href="../">Flight $fi->{Flight} scoreboard</a>
-<li><a href="$c{ResultsHtmlPubBaseUrl}/history.$job">History for $job</a>
-<li><a href="$c{ResultsHtmlPubBaseUrl}/history.$job.$branch">History for $job in tests of $branch</a>
+<li><a href="$c{ResultsHtmlPubBaseUrl}/history/$job/ALL">History for $job</a>
+<li><a href="$c{ResultsHtmlPubBaseUrl}/history/$job/$branch">History for $job in tests of $branch</a>
<li><a href="./">Logfiles for $fi->{Flight} $job as webserver directory listing</a>
</ul>
<h2>Steps</h2>
diff --git a/sg-report-job-history b/sg-report-job-history
index 18821b3..900d998 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -198,12 +198,14 @@ END
if (defined $htmlout) {
my ($title,$html_file,$url);
+ ensuredir "$htmlout/history";
+ ensuredir "$htmlout/history/$j";
if (defined $bra) {
- $title= "$j $bra";
- $html_file= "history.$j.$bra.html";
+ $title= "$j (branch $bra)";
+ $html_file= "history/$j/$bra.html";
} else {
- $title= $j;
- $html_file= "history.$j.html";
+ $title= "$j (all branches)";
+ $html_file= "history/$j/ALL.html";
}
$html_file= "$htmlout/$html_file";
open H, "> $html_file.new" or die "$html_file $!";
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 13/25] reporting: Move bisection report outputs
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (11 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-18 8:45 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
` (12 subsequent siblings)
25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Replace some `.'s in filenames with `/'s, making the directory
listings easier to deal with.
(I haven't been able to conveniently do a proper test of this change,
but I have dry-run the critical parts.)
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
cri-bisect | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/cri-bisect b/cri-bisect
index af3f23f..03b2e29 100644
--- a/cri-bisect
+++ b/cri-bisect
@@ -20,8 +20,9 @@
compute_state_core () {
reportfile=tmp/$flight.bisection-report
summaryfile=tmp/$flight.bisection-summary
- bisleaf=bisect.$branch.$job.$laundered_testid
+ bisleaf=bisect/$branch/$job.$laundered_testid
bisfile=$OSSTEST_HTML_DIR/$bisleaf
+ mkdir -p "${bisfile%/*}"
echo "computing $branch $job $testid $*"
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (12 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-18 8:46 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script Ian Jackson
` (11 subsequent siblings)
25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
The function findaflight should not, when the caller specifies a job,
find a flight which does not contain that job at all.
One effect of allowing it to find such flights is that it might find a
bisection flight and try to use it as a basis for claiming a
regression, or as a justification for something not being a
regression, and then complain that all the missing steps in the
bisection flight are `blocked'.
This can be seen in the report for 58627:
test-amd64-i386-xl-qemuu-winxpsp3 6 xen-boot fail blocked in 56366-bisect
After this patch, a report generated for 58627 no longer mentions
56366 at all.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
sg-report-flight | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/sg-report-flight b/sg-report-flight
index d3c0166..6481521 100755
--- a/sg-report-flight
+++ b/sg-report-flight
@@ -178,10 +178,23 @@ END
return undef;
}
+ my @flightsq_params;
+ my $flightsq_jobcond='TRUE';
+ if (defined $job) {
+ push @flightsq_params, $job;
+ $flightsq_jobcond = <<END;
+ EXISTS (SELECT 1
+ FROM jobs
+ WHERE jobs.flight = flights.flight
+ AND jobs.job = ?)
+END
+ }
+
my $flightsq= <<END;
SELECT * FROM (
SELECT flight, blessing FROM flights
WHERE $branches_cond_q
+ AND $flightsq_jobcond
AND $blessingscond
ORDER BY flight DESC
LIMIT 1000
@@ -189,7 +202,7 @@ END
ORDER BY blessing ASC, flight DESC
END
$flightsq= db_prepare($flightsq);
- $flightsq->execute(@blessings);
+ $flightsq->execute(@flightsq_params, @blessings);
my $buildflightsq= db_prepare(<<END);
SELECT val FROM runvars
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (13 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange Ian Jackson
` (10 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
This script is still a work in progress. It does currently mostly
work and can generate a useful report. However, it needs:
* Some database locking (to avoid deadlock errors)
* A --flight= option to make it automatically report host usage
for all hosts touched in a flight, for the use of cr-*
* Hooking into cr-*
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
v2: Improved commit message.
Fix a debug bug.
Actually install the html; do not leave it as .new
---
sg-report-host-history | 246 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 246 insertions(+)
create mode 100755 sg-report-host-history
diff --git a/sg-report-host-history b/sg-report-host-history
new file mode 100755
index 0000000..14f8458
--- /dev/null
+++ b/sg-report-host-history
@@ -0,0 +1,246 @@
+#!/usr/bin/perl -w
+
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2013 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+use strict qw(vars);
+
+use DBI;
+use Osstest;
+use IO::Handle;
+use HTML::Entities;
+
+use Osstest::Executive qw(:DEFAULT :colours);
+
+our $limit= 200;
+our $flightlimit;
+our $htmlout = ".";
+our @blessings;
+
+open DEBUG, ">/dev/null";
+
+my $namecond= "(name = 'host' or name like '%_host')";
+csreadconfig();
+
+while (@ARGV && $ARGV[0] =~ m/^-/) {
+ $_= shift @ARGV;
+ last if m/^--?$/;
+ if (m/^--(limit)\=([1-9]\d*)$/) {
+ $$1= $2;
+ } elsif (m/^--flight-limit\=([1-9]\d*)$/) {
+ $flightlimit= $1;
+ } elsif (m/^--blessings?=(.*)$/) {
+ push @blessings, split ',', $1;
+ } elsif (m/^--html-dir=(.*)$/) {
+ $htmlout= $1;
+ } elsif (m/^--debug/) {
+ open DEBUG, ">&2" or die $!;
+ DEBUG->autoflush(1);
+ } else {
+ die "$_ ?";
+ }
+}
+@blessings= qw(real) if !@blessings;
+
+@ARGV or die $!;
+
+$dbh_tests->begin_work;
+
+if (!$flightlimit) {
+ my $flagscond =
+ '('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
+ my $nhostsq = db_prepare(<<END);
+ SELECT count(*)
+ FROM resources r
+ WHERE restype='host'
+ AND EXISTS (SELECT 1
+ FROM hostflags f
+ WHERE f.hostname=r.resname
+ AND $flagscond)
+END
+ $nhostsq->execute();
+ my ($nhosts) = $nhostsq->fetchrow_array();
+ print DEBUG "COUNTED $nhosts hosts\n";
+ $flightlimit = $nhosts * $limit * 2;
+}
+
+my $minflightsq = db_prepare(<<END);
+ SELECT flight
+ FROM (
+ SELECT flight
+ FROM flights
+ ORDER BY flight DESC
+ LIMIT $flightlimit
+ ) f
+ ORDER BY flight ASC
+ LIMIT 1
+END
+$minflightsq->execute();
+my ($minflight) = $minflightsq->fetchrow_array();
+$minflight //= 0;
+
+our $flightcond = "(flight > $minflight)";
+
+$dbh_tests->do("SET LOCAL enable_seqscan=false");
+# Otherwise the PostgreSQL query planner likes to do a complete scan
+# of the runvars table, rather than walking backwards through the
+# flights until it has what we've told it is enough.
+
+my $runvarq = db_prepare(<<END);
+ SELECT flight, job, name, val
+ FROM runvars
+ WHERE $namecond
+ AND val = ?
+ AND $flightcond
+ ORDER BY flight DESC
+ LIMIT $limit * 2 + 100
+END
+
+my $endedq = db_prepare(<<END);
+ SELECT finished, testid, status AS laststepstatus
+ FROM steps
+ WHERE flight=? AND job=? AND finished IS NOT NULL
+ ORDER BY finished DESC
+ LIMIT 1
+END
+
+my $infoq = db_prepare(<<END);
+ SELECT blessing, branch, intended, status
+ FROM flights
+ JOIN jobs USING (flight)
+ WHERE flight=? AND job=?
+END
+
+my $allocdq = db_prepare(<<END);
+ SELECT testid, finished, status
+ FROM steps
+ WHERE flight=? AND job=?
+ AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
+ ORDER BY finished ASC
+ LIMIT 1
+END
+
+sub jobquery ($$) {
+ my ($q, $jr) = @_;
+ $q->execute($jr->{flight}, $jr->{job});
+ return $q->fetchrow_hashref();
+}
+
+sub reporthost ($) {
+ my ($hostname) = @_;
+
+ die if $hostname =~ m/[^-_.+0-9a-z]/;
+
+ my $html_file= "$htmlout/host.$hostname.html";
+ open H, "> $html_file.new" or die "$html_file $!";
+
+ my $title= "host history $hostname\n";
+ $title= encode_entities($title);
+ print H "<html><head><title>$title</title></head><body>\n";
+ print H "<h1>$title</h1>\n";
+ print H "<table rules=all><tr>\n";
+
+ print H "<th>alloc testid</th><th>alloc completed</th>\n";
+ print H "<th>job finished</th>\n";
+ print H "<th>role</th>\n";
+
+ print H "<th>flight</th>\n";
+ print H "<th>branch</th><th>intended</th><th>blessing</th>\n";
+
+ print H "<th>job</th><th>failure</th>\n";
+
+ print H "</tr>\n";
+
+ my @rows;
+ $runvarq->execute($hostname);
+
+ print DEBUG "FIRST PASS\n";
+ while (my $jr= $runvarq->fetchrow_hashref()) {
+ print DEBUG "JOB $jr->{flight}.$jr->{job} ";
+
+ my $endedrow = jobquery($endedq, $jr);
+ if (!$endedrow) {
+ print DEBUG "no-finished\n";
+ next;
+ }
+ print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
+ print DEBUG ".\n";
+
+ push @rows, { %$jr, %$endedrow };
+ }
+
+ print DEBUG "FOUND ", (scalar @rows), " ROWS\n";
+
+ @rows = sort { $b->{finished} <=> $a->{finished} } @rows;
+ $#rows = $limit-1 if @rows > $limit;
+
+ my $alternate = 0;
+ foreach my $jr (@rows) {
+ print DEBUG "JOB $jr->{flight}.$jr->{job}\n";
+
+ my $ir = jobquery($infoq, $jr);
+ my $ar = jobquery($allocdq, $jr);
+
+ my $altcolour = report_altcolour($alternate);
+ print H "<tr $altcolour>";
+
+ if (!defined $ar->{testid}) {
+ print H "<td bgcolor=\"$red\"></td>";
+ print H "<td>?</td>";
+ } else {
+ if ($ar->{status} eq 'pass') {
+ print H "<td>$ar->{testid}</td>";
+ print H "<td>", (show_abs_time $ar->{finished}), "</td>";
+ } elsif ($ar->{status} eq 'running') {
+ print H "<td bgcolor=\"$blue\">$ar->{testid}</td>";
+ print H "<td>$ar->{status}</td>";
+ } else {
+ print H "<td bgcolor=\"$red\">$ar->{testid}</td>";
+ print H "<td>$ar->{status}</td>";
+ }
+ }
+ print H "\n";
+
+ print H "<td>", (show_abs_time $jr->{finished}), "</td>\n";
+ print H "<td>", $jr->{name}, "</td>\n";
+
+ my $url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}";
+ print H "<td><a href=\"$url\">$jr->{flight}</a></td>\n";
+ $url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}/".
+ encode_entities($jr->{job})."/";
+ print H "<td>$ir->{branch}</td>";
+ print H "<td>$ir->{intended}</td>";
+ print H "<td>";
+ print H $ir->{blessing} unless $ir->{blessing} eq 'running';
+ print H "</td>";
+
+ print H "<td><a href=\"$url\">$jr->{job}</td>\n";
+
+ my $ri = report_run_getinfo({ %$jr, %$ir });
+ print H "<td bgcolor=\"$ri->{Colour}\">$ri->{Summary}</td>\n";
+
+ print H "</tr>\n\n";
+ $alternate ^= 1;
+ }
+
+ print H "</table></body></html>\n";
+
+ close H or die $!;
+ rename "$html_file.new", "$html_file" or die "$html_file $!";
+}
+
+reporthost $_ foreach @ARGV;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (14 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery Ian Jackson
` (9 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
To do database locking coherently, it will be convenient to have this
in a function.
No functional change.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
sg-report-host-history | 65 +++++++++++++++++++++++++-----------------------
1 file changed, 34 insertions(+), 31 deletions(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index 14f8458..c56ed71 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -58,42 +58,44 @@ while (@ARGV && $ARGV[0] =~ m/^-/) {
@ARGV or die $!;
-$dbh_tests->begin_work;
-
-if (!$flightlimit) {
- my $flagscond =
- '('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
- my $nhostsq = db_prepare(<<END);
- SELECT count(*)
- FROM resources r
- WHERE restype='host'
- AND EXISTS (SELECT 1
- FROM hostflags f
- WHERE f.hostname=r.resname
- AND $flagscond)
+our $flightcond;
+
+sub computeflightsrange () {
+ if (!$flightlimit) {
+ my $flagscond =
+ '('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
+ my $nhostsq = db_prepare(<<END);
+ SELECT count(*)
+ FROM resources r
+ WHERE restype='host'
+ AND EXISTS (SELECT 1
+ FROM hostflags f
+ WHERE f.hostname=r.resname
+ AND $flagscond)
END
- $nhostsq->execute();
- my ($nhosts) = $nhostsq->fetchrow_array();
- print DEBUG "COUNTED $nhosts hosts\n";
- $flightlimit = $nhosts * $limit * 2;
-}
+ $nhostsq->execute();
+ my ($nhosts) = $nhostsq->fetchrow_array();
+ print DEBUG "COUNTED $nhosts hosts\n";
+ $flightlimit = $nhosts * $limit * 2;
+ }
-my $minflightsq = db_prepare(<<END);
- SELECT flight
- FROM (
+ my $minflightsq = db_prepare(<<END);
SELECT flight
- FROM flights
- ORDER BY flight DESC
- LIMIT $flightlimit
- ) f
- ORDER BY flight ASC
- LIMIT 1
+ FROM (
+ SELECT flight
+ FROM flights
+ ORDER BY flight DESC
+ LIMIT $flightlimit
+ ) f
+ ORDER BY flight ASC
+ LIMIT 1
END
-$minflightsq->execute();
-my ($minflight) = $minflightsq->fetchrow_array();
-$minflight //= 0;
+ $minflightsq->execute();
+ my ($minflight) = $minflightsq->fetchrow_array();
+ $minflight //= 0;
-our $flightcond = "(flight > $minflight)";
+ $flightcond = "(flight > $minflight)";
+}
$dbh_tests->do("SET LOCAL enable_seqscan=false");
# Otherwise the PostgreSQL query planner likes to do a complete scan
@@ -243,4 +245,5 @@ sub reporthost ($) {
rename "$html_file.new", "$html_file" or die "$html_file $!";
}
+computeflightsrange();
reporthost $_ foreach @ARGV;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (15 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations Ian Jackson
` (8 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
To get the scope of database locking, and of the SET LOCAL workaround,
right, it is convenient to move these query preparations into the
jobquery function.
No functional change.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
sg-report-host-history | 68 ++++++++++++++++++++++++------------------------
1 file changed, 34 insertions(+), 34 deletions(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index c56ed71..f7a773d 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -102,40 +102,6 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
# of the runvars table, rather than walking backwards through the
# flights until it has what we've told it is enough.
-my $runvarq = db_prepare(<<END);
- SELECT flight, job, name, val
- FROM runvars
- WHERE $namecond
- AND val = ?
- AND $flightcond
- ORDER BY flight DESC
- LIMIT $limit * 2 + 100
-END
-
-my $endedq = db_prepare(<<END);
- SELECT finished, testid, status AS laststepstatus
- FROM steps
- WHERE flight=? AND job=? AND finished IS NOT NULL
- ORDER BY finished DESC
- LIMIT 1
-END
-
-my $infoq = db_prepare(<<END);
- SELECT blessing, branch, intended, status
- FROM flights
- JOIN jobs USING (flight)
- WHERE flight=? AND job=?
-END
-
-my $allocdq = db_prepare(<<END);
- SELECT testid, finished, status
- FROM steps
- WHERE flight=? AND job=?
- AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
- ORDER BY finished ASC
- LIMIT 1
-END
-
sub jobquery ($$) {
my ($q, $jr) = @_;
$q->execute($jr->{flight}, $jr->{job});
@@ -167,6 +133,40 @@ sub reporthost ($) {
print H "</tr>\n";
+ our $runvarq //= db_prepare(<<END);
+ SELECT flight, job, name, val
+ FROM runvars
+ WHERE $namecond
+ AND val = ?
+ AND $flightcond
+ ORDER BY flight DESC
+ LIMIT $limit * 2 + 100
+END
+
+ our $endedq //= db_prepare(<<END);
+ SELECT finished, testid, status AS laststepstatus
+ FROM steps
+ WHERE flight=? AND job=? AND finished IS NOT NULL
+ ORDER BY finished DESC
+ LIMIT 1
+END
+
+ our $infoq //= db_prepare(<<END);
+ SELECT blessing, branch, intended, status
+ FROM flights
+ JOIN jobs USING (flight)
+ WHERE flight=? AND job=?
+END
+
+ our $allocdq //= db_prepare(<<END);
+ SELECT testid, finished, status
+ FROM steps
+ WHERE flight=? AND job=?
+ AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
+ ORDER BY finished ASC
+ LIMIT 1
+END
+
my @rows;
$runvarq->execute($hostname);
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (16 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts Ian Jackson
` (7 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Arrange for the SET LOCAL to have the right scope. Run
computeflightsrange, and reporthost, each within a db transaction,
with an appropriate lock.
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
sg-report-host-history | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index f7a773d..5a3f8be 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -97,11 +97,6 @@ END
$flightcond = "(flight > $minflight)";
}
-$dbh_tests->do("SET LOCAL enable_seqscan=false");
-# Otherwise the PostgreSQL query planner likes to do a complete scan
-# of the runvars table, rather than walking backwards through the
-# flights until it has what we've told it is enough.
-
sub jobquery ($$) {
my ($q, $jr) = @_;
$q->execute($jr->{flight}, $jr->{job});
@@ -245,5 +240,17 @@ END
rename "$html_file.new", "$html_file" or die "$html_file $!";
}
-computeflightsrange();
-reporthost $_ foreach @ARGV;
+db_retry($dbh_tests, [qw(flights resources)], sub {
+ computeflightsrange();
+});
+
+$dbh_tests->do("SET LOCAL enable_seqscan=false");
+# Otherwise the PostgreSQL query planner likes to do a complete scan
+# of the runvars table, rather than walking backwards through the
+# flights until it has what we've told it is enough.
+
+foreach my $host (@ARGV) {
+ db_retry($dbh_tests, [qw(flights)], sub {
+ reporthost $host;
+ });
+}
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (17 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT Ian Jackson
` (6 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
This will allow deduplication. No functional change other than a
change to the order of processing.
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
sg-report-host-history | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/sg-report-host-history b/sg-report-host-history
index 5a3f8be..8a4d40f 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -249,7 +249,13 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
# of the runvars table, rather than walking backwards through the
# flights until it has what we've told it is enough.
+our %hosts;
+
foreach my $host (@ARGV) {
+ $hosts{$host}++;
+}
+
+foreach my $host (sort keys %hosts) {
db_retry($dbh_tests, [qw(flights)], sub {
reporthost $host;
});
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (18 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts Ian Jackson
` (5 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
sg-report-host-history | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index 8a4d40f..b3ef63e 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -162,6 +162,8 @@ END
LIMIT 1
END
+ print DEBUG "QUERYING RUNVARS FOR $hostname\n";
+
my @rows;
$runvarq->execute($hostname);
@@ -252,7 +254,25 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
our %hosts;
foreach my $host (@ARGV) {
- $hosts{$host}++;
+ if ($host =~ m/^flight:/) {
+ my $flight=$'; #';
+ db_retry($dbh_tests, [qw(flights)], sub {
+ our $hostsinflightq //= db_prepare(<<END);
+ SELECT DISTINCT val
+ FROM runvars
+ WHERE flight=?
+ AND (name = 'host' or name like '%_host')
+END
+ $hostsinflightq->execute($flight);
+ while (my $row = $hostsinflightq->fetchrow_hashref()) {
+ $hosts{$row->{val}}++;
+ }
+ });
+ } elsif ($host !~ m/:/) {
+ $hosts{$host}++;
+ } else {
+ die "$host ?";
+ }
}
foreach my $host (sort keys %hosts) {
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (19 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction Ian Jackson
` (4 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
This is much faster. It might short-change unpopular hosts rather;
hence the change of the limit fudge factor from 2 to 3.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
sg-report-host-history | 96 +++++++++++++++++++++++++++---------------------
1 file changed, 55 insertions(+), 41 deletions(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index b3ef63e..bab636e 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -23,6 +23,7 @@ use DBI;
use Osstest;
use IO::Handle;
use HTML::Entities;
+use POSIX;
use Osstest::Executive qw(:DEFAULT :colours);
@@ -103,6 +104,50 @@ sub jobquery ($$) {
return $q->fetchrow_hashref();
}
+our %hosts;
+
+sub mainquery () {
+ our $valcond = join " OR ", map { "val = ?" } keys %hosts;
+ our @params = keys %hosts;
+
+ our $runvarq //= db_prepare(<<END);
+ SELECT flight, job, name, val
+ FROM runvars
+ WHERE $namecond
+ AND ($valcond)
+ AND $flightcond
+ ORDER BY flight DESC
+ LIMIT ($limit * 3 + 100) * ?
+END
+
+ push @params, scalar keys %hosts;
+
+ our $endedq //= db_prepare(<<END);
+ SELECT finished, testid, status AS laststepstatus
+ FROM steps
+ WHERE flight=? AND job=? AND finished IS NOT NULL
+ ORDER BY finished DESC
+ LIMIT 1
+END
+
+ $runvarq->execute(@params);
+
+ print DEBUG "FIRST PASS\n";
+ while (my $jr= $runvarq->fetchrow_hashref()) {
+ print DEBUG "JOB $jr->{flight}.$jr->{job} ";
+
+ my $endedrow = jobquery($endedq, $jr);
+ if (!$endedrow) {
+ print DEBUG "no-finished\n";
+ next;
+ }
+ print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
+ print DEBUG ".\n";
+
+ push @{ $hosts{$jr->{val}} }, { %$jr, %$endedrow };
+ }
+}
+
sub reporthost ($) {
my ($hostname) = @_;
@@ -128,24 +173,6 @@ sub reporthost ($) {
print H "</tr>\n";
- our $runvarq //= db_prepare(<<END);
- SELECT flight, job, name, val
- FROM runvars
- WHERE $namecond
- AND val = ?
- AND $flightcond
- ORDER BY flight DESC
- LIMIT $limit * 2 + 100
-END
-
- our $endedq //= db_prepare(<<END);
- SELECT finished, testid, status AS laststepstatus
- FROM steps
- WHERE flight=? AND job=? AND finished IS NOT NULL
- ORDER BY finished DESC
- LIMIT 1
-END
-
our $infoq //= db_prepare(<<END);
SELECT blessing, branch, intended, status
FROM flights
@@ -162,27 +189,10 @@ END
LIMIT 1
END
- print DEBUG "QUERYING RUNVARS FOR $hostname\n";
-
my @rows;
- $runvarq->execute($hostname);
-
- print DEBUG "FIRST PASS\n";
- while (my $jr= $runvarq->fetchrow_hashref()) {
- print DEBUG "JOB $jr->{flight}.$jr->{job} ";
-
- my $endedrow = jobquery($endedq, $jr);
- if (!$endedrow) {
- print DEBUG "no-finished\n";
- next;
- }
- print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
- print DEBUG ".\n";
+ @rows = @{ $hosts{$hostname} };
- push @rows, { %$jr, %$endedrow };
- }
-
- print DEBUG "FOUND ", (scalar @rows), " ROWS\n";
+ print DEBUG "FOUND ", (scalar @rows), " ROWS for $hostname\n";
@rows = sort { $b->{finished} <=> $a->{finished} } @rows;
$#rows = $limit-1 if @rows > $limit;
@@ -251,8 +261,6 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
# of the runvars table, rather than walking backwards through the
# flights until it has what we've told it is enough.
-our %hosts;
-
foreach my $host (@ARGV) {
if ($host =~ m/^flight:/) {
my $flight=$'; #';
@@ -265,16 +273,22 @@ foreach my $host (@ARGV) {
END
$hostsinflightq->execute($flight);
while (my $row = $hostsinflightq->fetchrow_hashref()) {
- $hosts{$row->{val}}++;
+ $hosts{$row->{val}} = [ ];
}
});
} elsif ($host !~ m/:/) {
- $hosts{$host}++;
+ $hosts{$host} = [ ];
} else {
die "$host ?";
}
}
+exit 0 unless %hosts;
+
+db_retry($dbh_tests, [qw(flights)], sub {
+ mainquery();
+});
+
foreach my $host (sort keys %hosts) {
db_retry($dbh_tests, [qw(flights)], sub {
reporthost $host;
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (20 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete" Ian Jackson
` (3 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
No substantial change, but reduces the work done in the main
transaction.
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
sg-report-host-history | 46 ++++++++++++++++++++++++----------------------
1 file changed, 24 insertions(+), 22 deletions(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index bab636e..cd9ec61 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -122,29 +122,12 @@ END
push @params, scalar keys %hosts;
- our $endedq //= db_prepare(<<END);
- SELECT finished, testid, status AS laststepstatus
- FROM steps
- WHERE flight=? AND job=? AND finished IS NOT NULL
- ORDER BY finished DESC
- LIMIT 1
-END
-
$runvarq->execute(@params);
print DEBUG "FIRST PASS\n";
while (my $jr= $runvarq->fetchrow_hashref()) {
print DEBUG "JOB $jr->{flight}.$jr->{job} ";
-
- my $endedrow = jobquery($endedq, $jr);
- if (!$endedrow) {
- print DEBUG "no-finished\n";
- next;
- }
- print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
- print DEBUG ".\n";
-
- push @{ $hosts{$jr->{val}} }, { %$jr, %$endedrow };
+ push @{ $hosts{$jr->{val}} }, $jr;
}
}
@@ -173,6 +156,14 @@ sub reporthost ($) {
print H "</tr>\n";
+ our $endedq //= db_prepare(<<END);
+ SELECT finished, testid, status AS laststepstatus
+ FROM steps
+ WHERE flight=? AND job=? AND finished IS NOT NULL
+ ORDER BY finished DESC
+ LIMIT 1
+END
+
our $infoq //= db_prepare(<<END);
SELECT blessing, branch, intended, status
FROM flights
@@ -189,18 +180,29 @@ END
LIMIT 1
END
+ my $inrows = $hosts{$hostname};
+ print DEBUG "FOUND ", (scalar @$inrows), " ROWS for $hostname\n";
+
my @rows;
- @rows = @{ $hosts{$hostname} };
+ foreach my $jr (@$inrows) {
+ print DEBUG "JOB $jr->{flight}.$jr->{job}\n";
- print DEBUG "FOUND ", (scalar @rows), " ROWS for $hostname\n";
+ my $endedrow = jobquery($endedq, $jr);
+ if (!$endedrow) {
+ print DEBUG "no-finished\n";
+ next;
+ }
+ print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
+ print DEBUG ".\n";
+
+ push @rows, { %$jr, %$endedrow };
+ }
@rows = sort { $b->{finished} <=> $a->{finished} } @rows;
$#rows = $limit-1 if @rows > $limit;
my $alternate = 0;
foreach my $jr (@rows) {
- print DEBUG "JOB $jr->{flight}.$jr->{job}\n";
-
my $ir = jobquery($infoq, $jr);
my $ar = jobquery($allocdq, $jr);
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete"
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (21 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/ Ian Jackson
` (2 subsequent siblings)
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Since they may in fact be abandoned.
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
sg-report-host-history | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index cd9ec61..ad9f248 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -218,7 +218,7 @@ END
print H "<td>", (show_abs_time $ar->{finished}), "</td>";
} elsif ($ar->{status} eq 'running') {
print H "<td bgcolor=\"$blue\">$ar->{testid}</td>";
- print H "<td>$ar->{status}</td>";
+ print H "<td>(incomplete)</td>";
} else {
print H "<td bgcolor=\"$red\">$ar->{testid}</td>";
print H "<td>$ar->{status}</td>";
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (22 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete" Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
2015-06-18 9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
sg-report-host-history | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sg-report-host-history b/sg-report-host-history
index ad9f248..f4e73e9 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -136,7 +136,7 @@ sub reporthost ($) {
die if $hostname =~ m/[^-_.+0-9a-z]/;
- my $html_file= "$htmlout/host.$hostname.html";
+ my $html_file= "$htmlout/$hostname.html";
open H, "> $html_file.new" or die "$html_file $!";
my $title= "host history $hostname\n";
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (23 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/ Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
2015-06-18 8:47 ` Ian Campbell
2015-06-18 9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
To: xen-devel; +Cc: Ian Jackson, Ian Campbell
Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
cri-args-hostlists | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/cri-args-hostlists b/cri-args-hostlists
index 08e733d..a4e57b3 100644
--- a/cri-args-hostlists
+++ b/cri-args-hostlists
@@ -108,6 +108,7 @@ start_email () {
local flight_html_dir=$OSSTEST_HTMLPUB_DIR/
local job_html_dir=$OSSTEST_HTML_DIR/
+ local host_html_dir=$OSSTEST_HTML_DIR/host/
globallockdir=`getconfig GlobalLockDir`
@@ -117,6 +118,10 @@ start_email () {
./sg-report-flight --html-dir=$flight_html_dir/$flight/ \
--allow=allow.all --allow=allow.$branch \
$sgr_args $flight
+
+ mkdir -p $host_html_dir
+ with-lock-ex -w $globallockdir/report-lock \
+ ./sg-report-host-history --html-dir=$host_html_dir flight:$flight
}
publish_send_email () {
--
1.7.10.4
^ permalink raw reply related [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs
2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
@ 2015-06-18 8:44 ` Ian Campbell
0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 8:44 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 12/25] reporting: Move job histories into history/
2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
@ 2015-06-18 8:45 ` Ian Campbell
0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 8:45 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Replace some `.'s in HTML filenames with `/'s, making the directory
> listings easier to deal with.
>
> Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 13/25] reporting: Move bisection report outputs
2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
@ 2015-06-18 8:45 ` Ian Campbell
0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 8:45 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Replace some `.'s in filenames with `/'s, making the directory
> listings easier to deal with.
>
> (I haven't been able to conveniently do a proper test of this change,
> but I have dry-run the critical parts.)
>
> Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs
2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
@ 2015-06-18 8:46 ` Ian Campbell
0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 8:46 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> The function findaflight should not, when the caller specifies a job,
> find a flight which does not contain that job at all.
>
> One effect of allowing it to find such flights is that it might find a
> bisection flight and try to use it as a basis for claiming a
> regression, or as a justification for something not being a
> regression, and then complain that all the missing steps in the
> bisection flight are `blocked'.
>
> This can be seen in the report for 58627:
> test-amd64-i386-xl-qemuu-winxpsp3 6 xen-boot fail blocked in 56366-bisect
>
> After this patch, a report generated for 58627 no longer mentions
> 56366 at all.
>
> Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history
2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
@ 2015-06-18 8:47 ` Ian Campbell
0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 8:47 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 00/25] Reporting improvements
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
` (24 preceding siblings ...)
2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
@ 2015-06-18 9:06 ` Ian Campbell
2015-06-18 11:21 ` Ian Jackson
25 siblings, 1 reply; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 9:06 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Wed, 2015-06-17 at 17:44 +0100, Ian Jackson wrote:
> Most of the early part of this series is straightforward (and acked)
> and could go in whenever.
Agreed. Do you happen to have a convenient directory with the new
output? (I'm happy to just wait for it to happen too).
> Patches 15-24 introduce sg-report-host-history. It's probably easiest
> to simply review the final version of the script, which can be found
> below. The history is preserved mostly for archeological purposes.
Sure.
>
> Ian.
>
> #!/usr/bin/perl -w
>
> # This is part of "osstest", an automated testing framework for Xen.
> # Copyright (C) 2009-2013 Citrix Inc.
Date ;-)
I couldn't spot anything else, FWIW. I think if you are happy with the
output there's no reason not to just run with it.
Ian.
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 00/25] Reporting improvements
2015-06-18 9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
@ 2015-06-18 11:21 ` Ian Jackson
2015-06-18 14:21 ` Ian Campbell
0 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-18 11:21 UTC (permalink / raw)
To: Ian Campbell; +Cc: xen-devel
Ian Campbell writes ("Re: [OSSTEST PATCH 00/25] Reporting improvements"):
> On Wed, 2015-06-17 at 17:44 +0100, Ian Jackson wrote:
> > Most of the early part of this series is straightforward (and acked)
> > and could go in whenever.
>
> Agreed. Do you happen to have a convenient directory with the new
> output? (I'm happy to just wait for it to happen too).
I have mostly run it on my workstation on the Cambridge instance.
Here is an example that I have lying about, copied to a public
webserver:
http://www.chiark.greenend.org.uk/~ijackson/quicksand/2015/grain-weevil.html
> > Patches 15-24 introduce sg-report-host-history. It's probably easiest
> > to simply review the final version of the script, which can be found
> > below. The history is preserved mostly for archeological purposes.
>
> Sure.
Thanks.
> > #!/usr/bin/perl -w
> >
> > # This is part of "osstest", an automated testing framework for Xen.
> > # Copyright (C) 2009-2013 Citrix Inc.
>
> Date ;-)
>
> I couldn't spot anything else, FWIW. I think if you are happy with the
> output there's no reason not to just run with it.
I'll give you a chance to look at the output URL above.
Ian.
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 00/25] Reporting improvements
2015-06-18 11:21 ` Ian Jackson
@ 2015-06-18 14:21 ` Ian Campbell
2015-06-18 14:36 ` Ian Jackson
0 siblings, 1 reply; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 14:21 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Thu, 2015-06-18 at 12:21 +0100, Ian Jackson wrote:
> Ian Campbell writes ("Re: [OSSTEST PATCH 00/25] Reporting improvements"):
> > On Wed, 2015-06-17 at 17:44 +0100, Ian Jackson wrote:
> > > Most of the early part of this series is straightforward (and acked)
> > > and could go in whenever.
> >
> > Agreed. Do you happen to have a convenient directory with the new
> > output? (I'm happy to just wait for it to happen too).
>
> I have mostly run it on my workstation on the Cambridge instance.
> Here is an example that I have lying about, copied to a public
> webserver:
> http://www.chiark.greenend.org.uk/~ijackson/quicksand/2015/grain-weevil.html
>
> > [...]
>
> I'll give you a chance to look at the output URL above.
Looks good, thanks.
OOI what is the expected usefulness of the "alloc testid" column?
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 00/25] Reporting improvements
2015-06-18 14:21 ` Ian Campbell
@ 2015-06-18 14:36 ` Ian Jackson
0 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-18 14:36 UTC (permalink / raw)
To: Ian Campbell; +Cc: xen-devel
Ian Campbell writes ("Re: [OSSTEST PATCH 00/25] Reporting improvements"):
> On Thu, 2015-06-18 at 12:21 +0100, Ian Jackson wrote:
> > I'll give you a chance to look at the output URL above.
>
> Looks good, thanks.
>
> OOI what is the expected usefulness of the "alloc testid" column?
With the system as it currently exists, it will always say
"hosts-allocate". But that's not a fixed assumption of the overall
resource allocation design.
Ian.
^ permalink raw reply [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging
2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
@ 2015-06-18 15:33 ` Ian Jackson
2015-06-18 15:46 ` Ian Campbell
0 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-18 15:33 UTC (permalink / raw)
To: xen-devel, Ian Campbell
Ian Jackson writes ("[OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging"):
> No callers, so no functional change, as yet.
> +sub db_prepare ($) {
> + # caller must ensure global filehandle DEBUG is open
> + my ($stmt) = @_;
> + print ::DEBUG "DB PREPARING:\n$stmt\n";
> + return $dbh_tests->prepare($stmt);
This generates:
Name "main::DEBUG" used only once: possible typo at Osstest.pm line
274.
Here is a v2 of this patch which fixes it.
Ian.
commit bd7cf4f0f1a8c45d840528bc28dbd89b34732d7a
Author: Ian Jackson <ian.jackson@eu.citrix.com>
Date: Fri May 29 13:19:39 2015 +0000
Osstest.pm: Provide new db_prepare helper with built-in debugging
No callers, so no functional change, as yet.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
v2: Suppress "used only once" warning
diff --git a/Osstest.pm b/Osstest.pm
index e8bd77b..8948666 100644
--- a/Osstest.pm
+++ b/Osstest.pm
@@ -35,7 +35,7 @@ BEGIN {
getmethod
postfork
$dbh_tests db_retry db_retry_retry db_retry_abort
- db_begin_work
+ db_begin_work db_prepare
ensuredir get_filecontents_core_quiet system_checked
nonempty visible_undef show_abs_time
);
@@ -49,6 +49,10 @@ our $mjobdb;
our $dbh_tests;
+scalar *main::DEBUG;
+# declaration prevents `Name "main::DEBUG" used only once'
+# scalar prevents `useless use of a variable in void context'
+
#---------- static default config settings ----------
our %c = qw(
@@ -268,6 +272,13 @@ sub db_retry ($$$;$$) {
return $r;
}
+sub db_prepare ($) {
+ # caller must ensure global filehandle DEBUG is open
+ my ($stmt) = @_;
+ print ::DEBUG "DB PREPARING:\n$stmt\n";
+ return $dbh_tests->prepare($stmt);
+}
+
sub postfork () {
$mjobdb->jobdb_postfork();
}
^ permalink raw reply related [flat|nested] 37+ messages in thread
* Re: [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging
2015-06-18 15:33 ` Ian Jackson
@ 2015-06-18 15:46 ` Ian Campbell
0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 15:46 UTC (permalink / raw)
To: Ian Jackson; +Cc: xen-devel
On Thu, 2015-06-18 at 16:33 +0100, Ian Jackson wrote:
> Ian Jackson writes ("[OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging"):
> > No callers, so no functional change, as yet.
>
> > +sub db_prepare ($) {
> > + # caller must ensure global filehandle DEBUG is open
> > + my ($stmt) = @_;
> > + print ::DEBUG "DB PREPARING:\n$stmt\n";
> > + return $dbh_tests->prepare($stmt);
>
> This generates:
>
> Name "main::DEBUG" used only once: possible typo at Osstest.pm line
> 274.
>
> Here is a v2 of this patch which fixes it.
>
> Ian.
>
> commit bd7cf4f0f1a8c45d840528bc28dbd89b34732d7a
> Author: Ian Jackson <ian.jackson@eu.citrix.com>
> Date: Fri May 29 13:19:39 2015 +0000
>
> Osstest.pm: Provide new db_prepare helper with built-in debugging
>
> No callers, so no functional change, as yet.
>
> Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
> Acked-by: Ian Campbell <ian.campbell@citrix.com>
> ---
> v2: Suppress "used only once" warning
>
> diff --git a/Osstest.pm b/Osstest.pm
> index e8bd77b..8948666 100644
> --- a/Osstest.pm
> +++ b/Osstest.pm
> @@ -35,7 +35,7 @@ BEGIN {
> getmethod
> postfork
> $dbh_tests db_retry db_retry_retry db_retry_abort
> - db_begin_work
> + db_begin_work db_prepare
> ensuredir get_filecontents_core_quiet system_checked
> nonempty visible_undef show_abs_time
> );
> @@ -49,6 +49,10 @@ our $mjobdb;
>
> our $dbh_tests;
>
> +scalar *main::DEBUG;
> +# declaration prevents `Name "main::DEBUG" used only once'
> +# scalar prevents `useless use of a variable in void context'
um, wow ;-)
I suppose you will push this?
^ permalink raw reply [flat|nested] 37+ messages in thread
end of thread, other threads:[~2015-06-18 16:24 UTC | newest]
Thread overview: 37+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
2015-06-18 15:33 ` Ian Jackson
2015-06-18 15:46 ` Ian Campbell
2015-06-17 16:44 ` [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 03/25] cs-bisection-step: " Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 04/25] sg-report-flight: " Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 08/25] sg-report-job-history: Show start time Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
2015-06-18 8:44 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
2015-06-18 8:45 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
2015-06-18 8:45 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
2015-06-18 8:46 ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete" Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/ Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
2015-06-18 8:47 ` Ian Campbell
2015-06-18 9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
2015-06-18 11:21 ` Ian Jackson
2015-06-18 14:21 ` Ian Campbell
2015-06-18 14:36 ` Ian Jackson
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.