All of lore.kernel.org
 help / color / mirror / Atom feed
* [OSSTEST PATCH 00/25] Reporting improvements
@ 2015-06-17 16:44 Ian Jackson
  2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
                   ` (25 more replies)
  0 siblings, 26 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Campbell

Most of the early part of this series is straightforward (and acked)
and could go in whenever.

Patches 15-24 introduce sg-report-host-history.  It's probably easiest
to simply review the final version of the script, which can be found
below.  The history is preserved mostly for archeological purposes.

Ian.

#!/usr/bin/perl -w

# This is part of "osstest", an automated testing framework for Xen.
# Copyright (C) 2009-2013 Citrix Inc.
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# 
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


use strict qw(vars);

use DBI;
use Osstest;
use IO::Handle;
use HTML::Entities;
use POSIX;

use Osstest::Executive qw(:DEFAULT :colours);

our $limit= 200;
our $flightlimit;
our $htmlout = ".";
our @blessings;

open DEBUG, ">/dev/null";

my $namecond= "(name = 'host' or name like '%_host')";
csreadconfig();

while (@ARGV && $ARGV[0] =~ m/^-/) {
    $_= shift @ARGV;
    last if m/^--?$/;
    if (m/^--(limit)\=([1-9]\d*)$/) {
        $$1= $2;
    } elsif (m/^--flight-limit\=([1-9]\d*)$/) {
	$flightlimit= $1;
    } elsif (m/^--blessings?=(.*)$/) {
        push @blessings, split ',', $1;
    } elsif (m/^--html-dir=(.*)$/) {
        $htmlout= $1;
    } elsif (m/^--debug/) {
        open DEBUG, ">&2" or die $!;
        DEBUG->autoflush(1);
    } else {
        die "$_ ?";
    }
}
@blessings= qw(real) if !@blessings;

@ARGV or die $!;

our $flightcond;

sub computeflightsrange () {
    if (!$flightlimit) {
	my $flagscond =
	    '('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
	my $nhostsq = db_prepare(<<END);
	    SELECT count(*)
	      FROM resources r
	     WHERE restype='host'
	       AND EXISTS (SELECT 1
			     FROM hostflags f
			    WHERE f.hostname=r.resname
			      AND $flagscond)
END
        $nhostsq->execute();
	my ($nhosts) = $nhostsq->fetchrow_array();
	print DEBUG "COUNTED $nhosts hosts\n";
	$flightlimit = $nhosts * $limit * 2;
    }

    my $minflightsq = db_prepare(<<END);
	SELECT flight
	  FROM (
	    SELECT flight
	      FROM flights
	     ORDER BY flight DESC
	     LIMIT $flightlimit
	  ) f
	  ORDER BY flight ASC
	  LIMIT 1
END
    $minflightsq->execute();
    my ($minflight) = $minflightsq->fetchrow_array();
    $minflight //= 0;

    $flightcond = "(flight > $minflight)";
}

sub jobquery ($$) {
    my ($q, $jr) = @_;
    $q->execute($jr->{flight}, $jr->{job});
    return $q->fetchrow_hashref();
}

our %hosts;

sub mainquery () {
    our $valcond = join " OR ", map { "val = ?" } keys %hosts;
    our @params = keys %hosts;

    our $runvarq //= db_prepare(<<END);
	SELECT flight, job, name, val
	  FROM runvars
	 WHERE $namecond
	   AND ($valcond)
	   AND $flightcond
	 ORDER BY flight DESC
	 LIMIT ($limit * 3 + 100) * ?
END

    push @params, scalar keys %hosts;

    $runvarq->execute(@params);

    print DEBUG "FIRST PASS\n";
    while (my $jr= $runvarq->fetchrow_hashref()) {
	print DEBUG "JOB $jr->{flight}.$jr->{job} ";
	push @{ $hosts{$jr->{val}} }, $jr;
    }
}

sub reporthost ($) {
    my ($hostname) = @_;

    die if $hostname =~ m/[^-_.+0-9a-z]/;

    my $html_file= "$htmlout/$hostname.html";
    open H, "> $html_file.new" or die "$html_file $!";

    my $title= "host history $hostname\n";
    $title= encode_entities($title);
    print H "<html><head><title>$title</title></head><body>\n";
    print H "<h1>$title</h1>\n";
    print H "<table rules=all><tr>\n";

    print H "<th>alloc testid</th><th>alloc completed</th>\n";
    print H "<th>job finished</th>\n";
    print H "<th>role</th>\n";

    print H "<th>flight</th>\n";
    print H "<th>branch</th><th>intended</th><th>blessing</th>\n";

    print H "<th>job</th><th>failure</th>\n";

    print H "</tr>\n";

    our $endedq //= db_prepare(<<END);
	SELECT finished, testid, status AS laststepstatus
	  FROM steps
	 WHERE flight=? AND job=? AND finished IS NOT NULL
	 ORDER BY finished DESC
	 LIMIT 1
END

    our $infoq //= db_prepare(<<END);
	SELECT blessing, branch, intended, status
	  FROM flights
	  JOIN jobs USING (flight)
	 WHERE flight=? AND job=?
END

    our $allocdq //= db_prepare(<<END);
	SELECT testid, finished, status
	  FROM steps
	 WHERE flight=? AND job=?
	   AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
	 ORDER BY finished ASC
	 LIMIT 1
END

    my $inrows = $hosts{$hostname};
    print DEBUG "FOUND ", (scalar @$inrows), " ROWS for $hostname\n";

    my @rows;
    foreach my $jr (@$inrows) {
	print DEBUG "JOB $jr->{flight}.$jr->{job}\n";

	my $endedrow = jobquery($endedq, $jr);
	if (!$endedrow) {
	    print DEBUG "no-finished\n";
	    next;
	}
	print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
	print DEBUG ".\n";

	push @rows, { %$jr, %$endedrow };
    }

    @rows = sort { $b->{finished} <=> $a->{finished} } @rows;
    $#rows = $limit-1 if @rows > $limit;

    my $alternate = 0;
    foreach my $jr (@rows) {
	my $ir = jobquery($infoq, $jr);
	my $ar = jobquery($allocdq, $jr);

	my $altcolour = report_altcolour($alternate);
	print H "<tr $altcolour>";

	if (!defined $ar->{testid}) {
	    print H "<td bgcolor=\"$red\"></td>";
	    print H "<td>?</td>";
	} else {
	    if ($ar->{status} eq 'pass') {
		print H "<td>$ar->{testid}</td>";
		print H "<td>", (show_abs_time $ar->{finished}), "</td>";
	    } elsif ($ar->{status} eq 'running') {
		print H "<td bgcolor=\"$blue\">$ar->{testid}</td>";
		print H "<td>(incomplete)</td>";
	    } else {
		print H "<td bgcolor=\"$red\">$ar->{testid}</td>";
		print H "<td>$ar->{status}</td>";
	    }
	}
	print H "\n";

	print H "<td>", (show_abs_time $jr->{finished}), "</td>\n";
	print H "<td>", $jr->{name}, "</td>\n";

	my $url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}";
	print H "<td><a href=\"$url\">$jr->{flight}</a></td>\n";
	$url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}/".
	    encode_entities($jr->{job})."/";
	print H "<td>$ir->{branch}</td>";
	print H "<td>$ir->{intended}</td>";
	print H "<td>";
	print H $ir->{blessing} unless $ir->{blessing} eq 'running';
	print H "</td>";

	print H "<td><a href=\"$url\">$jr->{job}</td>\n";

	my $ri = report_run_getinfo({ %$jr, %$ir });
	print H "<td bgcolor=\"$ri->{Colour}\">$ri->{Summary}</td>\n";

	print H "</tr>\n\n";
	$alternate ^= 1;
    }

    print H "</table></body></html>\n";

    close H or die $!;
    rename "$html_file.new", "$html_file" or die "$html_file $!";
}

db_retry($dbh_tests, [qw(flights resources)], sub {
    computeflightsrange();
});

$dbh_tests->do("SET LOCAL enable_seqscan=false");
# Otherwise the PostgreSQL query planner likes to do a complete scan
# of the runvars table, rather than walking backwards through the
# flights until it has what we've told it is enough.

foreach my $host (@ARGV) {
    if ($host =~ m/^flight:/) {
	my $flight=$'; #';
	db_retry($dbh_tests, [qw(flights)], sub {
	    our $hostsinflightq //= db_prepare(<<END);
	        SELECT DISTINCT val
		  FROM runvars
		 WHERE flight=?
		   AND (name = 'host' or name like '%_host')
END
            $hostsinflightq->execute($flight);
	    while (my $row = $hostsinflightq->fetchrow_hashref()) {
		$hosts{$row->{val}} = [ ];
	    }
	});
    } elsif ($host !~ m/:/) {
	$hosts{$host} = [ ];
    } else {
	die "$host ?";
    }
}

exit 0 unless %hosts;

db_retry($dbh_tests, [qw(flights)], sub {
    mainquery();
});

foreach my $host (sort keys %hosts) {
    db_retry($dbh_tests, [qw(flights)], sub {
	reporthost $host;
    });
}

^ permalink raw reply	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
@ 2015-06-17 16:44 ` Ian Jackson
  2015-06-18 15:33   ` Ian Jackson
  2015-06-17 16:44 ` [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare Ian Jackson
                   ` (24 subsequent siblings)
  25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

No callers, so no functional change, as yet.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 Osstest.pm |    9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Osstest.pm b/Osstest.pm
index e8bd77b..7d6949f 100644
--- a/Osstest.pm
+++ b/Osstest.pm
@@ -35,7 +35,7 @@ BEGIN {
                       getmethod
                       postfork
                       $dbh_tests db_retry db_retry_retry db_retry_abort
-                      db_begin_work
+                      db_begin_work db_prepare
                       ensuredir get_filecontents_core_quiet system_checked
                       nonempty visible_undef show_abs_time
                       );
@@ -268,6 +268,13 @@ sub db_retry ($$$;$$) {
     return $r;
 }
 
+sub db_prepare ($) {
+    # caller must ensure global filehandle DEBUG is open
+    my ($stmt) = @_;
+    print ::DEBUG "DB PREPARING:\n$stmt\n";
+    return $dbh_tests->prepare($stmt);
+}
+
 sub postfork () {
     $mjobdb->jobdb_postfork();
 }
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
  2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
@ 2015-06-17 16:44 ` Ian Jackson
  2015-06-17 16:44 ` [OSSTEST PATCH 03/25] cs-bisection-step: " Ian Jackson
                   ` (23 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 sg-report-job-history |   14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/sg-report-job-history b/sg-report-job-history
index 409e3d5..e285caa 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -67,7 +67,7 @@ END
     die "$flight ? @$branches ?" if @$branches!=1;
     @branches= @$branches;
 
-    my $selectq= $dbh_tests->prepare(<<END);
+    my $selectq= db_prepare(<<END);
         SELECT job FROM jobs WHERE flight=? ORDER BY JOB
 END
     $selectq->execute($flight);
@@ -84,7 +84,7 @@ if (defined($flight)) {
     push @jobs, $job;
 }
 
-our $failstepq= $dbh_tests->prepare(<<END);
+our $failstepq= db_prepare(<<END);
         SELECT * FROM steps
          WHERE flight=? AND job=?
            AND status!='pass'
@@ -120,7 +120,7 @@ sub run_getinfo ($) {
     }
 }
 
-our $revisionsq= $dbh_tests->prepare(<<END);
+our $revisionsq= db_prepare(<<END);
         SELECT * FROM runvars
          WHERE flight=? AND job=?
            AND name LIKE E'built\\_revision\\_\%'
@@ -136,7 +136,7 @@ sub add_revisions ($$$$) {
     }
 }
 
-our $buildsq= $dbh_tests->prepare(<<END);
+our $buildsq= db_prepare(<<END);
         SELECT * FROM runvars
          WHERE flight=? AND job=?
            AND name LIKE E'\%buildjob'
@@ -171,13 +171,13 @@ END
          WHERE ($cond)
       ORDER BY flight DESC
 END
-    my $flightsq= $dbh_tests->prepare(<<END);
+    my $flightsq= db_prepare(<<END);
         SELECT * $fromstuff
          LIMIT $limit
 END
     $flightsq->execute(@params);
 
-    my $hostsq= $dbh_tests->prepare(<<END);
+    my $hostsq= db_prepare(<<END);
         SELECT DISTINCT name
 	 FROM runvars
 	 JOIN flights USING (flight)
@@ -195,7 +195,7 @@ END
 	push @hostvarcols, $hostvar;
     }
 
-    my $hostq= $dbh_tests->prepare(<<END);
+    my $hostq= db_prepare(<<END);
         SELECT val FROM runvars WHERE flight=? AND job=? AND name=?
 END
 
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 03/25] cs-bisection-step: Use db_prepare
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
  2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
  2015-06-17 16:44 ` [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare Ian Jackson
@ 2015-06-17 16:44 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 04/25] sg-report-flight: " Ian Jackson
                   ` (22 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:44 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

And do away with one ad-hoc statement dump.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 cs-bisection-step |   21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/cs-bisection-step b/cs-bisection-step
index 67a92b7..bf41f81 100755
--- a/cs-bisection-step
+++ b/cs-bisection-step
@@ -203,7 +203,7 @@ END
 	       AND flight = ?
 END
 
-    my $sth= $dbh_tests->prepare(<<END);
+    my $sth= db_prepare(<<END);
 
         SELECT url.val AS uval,
 	       rev.val AS rval,
@@ -303,7 +303,7 @@ sub findbasics () {
             ? "flight = $specflights{$fp}" : "TRUE";
     }
 
-    my $latestq= $dbh_tests->prepare(<<END);
+    my $latestq= db_prepare(<<END);
             SELECT flights.flight, steps.status,
                    $flight_is_not_broken AS notbroken
                 FROM flights JOIN steps USING (flight)
@@ -316,7 +316,7 @@ sub findbasics () {
                      $maxflight_cond
                ORDER BY notbroken DESC, flights.started DESC
 END
-    my $basisq= $dbh_tests->prepare(<<END);
+    my $basisq= db_prepare(<<END);
         SELECT * FROM flights JOIN steps USING (flight)
 	   WHERE job = ?
 	     AND testid = ?
@@ -553,14 +553,14 @@ END
 
     print STDERR "Searching for test results:\n";
 
-    my $stepq= $dbh_tests->prepare("
+    my $stepq= db_prepare("
        SELECT * FROM steps
           WHERE flight = ?
             AND job = ?
             AND testid = ?
     ");
 
-    my $jobq= $dbh_tests->prepare("
+    my $jobq= db_prepare("
        SELECT * FROM flights JOIN jobs USING (flight)
           WHERE job = ?
             AND $blessingscond
@@ -1059,7 +1059,7 @@ sub preparejob ($$$) {
 END
     my (@trevisions) = split / /, $choose->{Rtuple};
 
-    my $treeq= $dbh_tests->prepare(<<END);
+    my $treeq= db_prepare(<<END);
         SELECT name FROM runvars
           WHERE  flight=? AND job=?
             AND  name = ?
@@ -1082,7 +1082,7 @@ END
 
     # Check for subjobs:
 
-    my $jobq= $dbh_tests->prepare(<<END);
+    my $jobq= db_prepare(<<END);
         SELECT name, val FROM runvars
             WHERE  flight=? AND job=?
               AND  name LIKE '%job';
@@ -1156,7 +1156,7 @@ END
 
     $dbh_tests->do("DROP TABLE bisection_runvars");
 
-    my $jobsetq= $dbh_tests->prepare(<<END);
+    my $jobsetq= db_prepare(<<END);
         UPDATE runvars SET val=?
             WHERE  flight=? AND job=? AND name=?
 END
@@ -1185,7 +1185,7 @@ sub populateflight () {
     }
 
     my $removehosts= 
-    my $addhost= $dbh_tests->prepare(<<END);
+    my $addhost= db_prepare(<<END);
         INSERT INTO runvars (flight,job,name,val,synth)
                      VALUES (?,     ?,  ?,   ?,  'f')
 END
@@ -1244,8 +1244,7 @@ END
               LIMIT $maxequalflights + 1;
 END
 
-    print DEBUG "equalflightsq:\n$equalflightsqt";
-    my $equalflightsq = $dbh_tests->prepare($equalflightsqt);
+    my $equalflightsq = db_prepare($equalflightsqt);
 
     db_retry($popflight,'constructing', $dbh_tests,[qw(flights)], sub {
         print STDERR "Checking for flail (since $repro_firstflight)...\n";
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 04/25] sg-report-flight: Use db_prepare
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (2 preceding siblings ...)
  2015-06-17 16:44 ` [OSSTEST PATCH 03/25] cs-bisection-step: " Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement Ian Jackson
                   ` (21 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

And do away with two commented-out ad-hoc statement dumps.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 sg-report-flight |   28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/sg-report-flight b/sg-report-flight
index 80777af..bdaf1c8 100755
--- a/sg-report-flight
+++ b/sg-report-flight
@@ -163,7 +163,7 @@ sub findaflight ($$$$$) {
         return undef;
     }
 
-    my $checkq= $dbh_tests->prepare(<<END);
+    my $checkq= db_prepare(<<END);
         SELECT status FROM steps WHERE flight=? AND job=? AND testid=?
 END
 
@@ -188,11 +188,10 @@ END
       ) AS sub
       ORDER BY blessing ASC, flight DESC
 END
-    #print DEBUG "===\n$flightsq\n===\n";
-    $flightsq= $dbh_tests->prepare($flightsq);
+    $flightsq= db_prepare($flightsq);
     $flightsq->execute(@blessings);
 
-    my $buildflightsq= $dbh_tests->prepare(<<END);
+    my $buildflightsq= db_prepare(<<END);
         SELECT val FROM runvars
 	 WHERE flight = ?
            AND name LIKE '%buildjob'
@@ -204,10 +203,9 @@ END
 		  AND name=?
 		GROUP BY flight, job, val
 END
-    #print DEBUG "===\n$mismatchq\n===\n";
-    $revisionsq= $dbh_tests->prepare($revisionsq);
+    $revisionsq= db_prepare($revisionsq);
 
-    my $revisionsosstestq= $dbh_tests->prepare(<<END);
+    my $revisionsosstestq= db_prepare(<<END);
             SELECT harness AS val FROM flights_harness_touched
                 WHERE flight=?
 END
@@ -319,7 +317,7 @@ END
     };
     $jobs= [ sort { $colmap->($a) cmp $colmap->($b) } @$jobs ];
 
-    my $stepsq= $dbh_tests->prepare(<<END);
+    my $stepsq= db_prepare(<<END);
         SELECT * FROM steps
             WHERE flight=$flight AND job=?
             ORDER BY stepno
@@ -441,7 +439,7 @@ END
             });
         }
 
-        my $revh= $dbh_tests->prepare(<<END);
+        my $revh= db_prepare(<<END);
             SELECT * FROM runvars
                 WHERE flight=$flight AND job='$j->{job}'
                   AND name like 'built_revision_%'
@@ -623,7 +621,7 @@ sub justifyfailures ($;$) {
               AND $blessingscond
             LIMIT 1
 END
-    $anypassq= $dbh_tests->prepare($anypassq);
+    $anypassq= db_prepare($anypassq);
 
     my $duration_estimator= duration_estimator($branch, $blessings[0]);
     foreach my $failv (@failures) {
@@ -802,7 +800,7 @@ sub htmloutjob ($$) {
     my $title= "Info on flight $fi->{Flight} job $job";
     my $branch= $fi->{FlightInfo}{branch};
 
-    our $htmlout_jobq ||= $dbh_tests->prepare(<<END);
+    our $htmlout_jobq ||= db_prepare(<<END);
         SELECT * FROM jobs WHERE flight = ? AND job = ?
 END
     $htmlout_jobq->execute($fi->{Flight}, $job);
@@ -826,7 +824,7 @@ END
 END
 
     my %issteplog;
-    my $stepsq= $dbh_tests->prepare(<<END);
+    my $stepsq= db_prepare(<<END);
         SELECT * FROM steps
                 WHERE flight=? AND job=?
              ORDER BY stepno ASC
@@ -892,7 +890,7 @@ END
 <h2>Test control variables</h2>
 <table rules=all><tr><th>Name</th><th>Value</th><th>Source</th></tr>
 END
-    my $varsq= $dbh_tests->prepare(<<END);
+    my $varsq= db_prepare(<<END);
         SELECT * FROM runvars
                 WHERE flight=? AND job=?
              ORDER BY synth, name
@@ -1086,7 +1084,7 @@ END
     }
     my $started_str = show_abs_time $fi->{FlightInfo}{started};
 
-    our $htmlout_finishedq ||= $dbh_tests->prepare(<<END);
+    our $htmlout_finishedq ||= db_prepare(<<END);
         SELECT
  (SELECT max(finished) FROM steps WHERE flight=?)                 max,
  (SELECT count(*) FROM steps WHERE finished IS NULL AND flight=?) unfinished
@@ -1110,7 +1108,7 @@ END
 <td>Test harness revision(s):</td><td>
 END
 
-    our $htmlout_touchedq ||= $dbh_tests->prepare(<<END);
+    our $htmlout_touchedq ||= db_prepare(<<END);
         SELECT harness FROM flights_harness_touched WHERE flight=?
 END
     $htmlout_touchedq->execute($fi->{Flight});
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (3 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 04/25] sg-report-flight: " Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql Ian Jackson
                   ` (20 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 sg-report-job-history |    2 ++
 1 file changed, 2 insertions(+)

diff --git a/sg-report-job-history b/sg-report-job-history
index e285caa..f5f3496 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -150,6 +150,8 @@ sub altcolour ($) {
 sub processjobbranch ($$) {
     my ($j,$bra) = @_;
 
+    print DEBUG "processjobbranch('$j',", ($bra ? "'$bra'" : 'undef'), ")\n";
+
     my %rev_grid_col;
     my @rev_grid_cols;
     my @test_rows;
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (4 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short Ian Jackson
                   ` (19 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

No functional change apart from slightly better debug output.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 sg-report-job-history |    4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/sg-report-job-history b/sg-report-job-history
index f5f3496..e4f35bf 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -160,9 +160,7 @@ sub processjobbranch ($$) {
     my $cond = "job = ? AND $blessingscond";
     my (@params) = ($j, @blessings);
     if (defined $bra) {
-        $cond .= <<END;
-           AND branch = ?
-END
+        $cond .= " AND branch = ?";
         push @params, $bra;
     }
     my $limit= 100;
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (5 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 08/25] sg-report-job-history: Show start time Ian Jackson
                   ` (18 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

If there have been less than 99 relevant flights, the inner SELECT (to
determine the minimum flight number) would return NULL.  And anything
> NULL is NULL and NULL is treated as false.  So the host runvar
identification would break.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 sg-report-job-history |    5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sg-report-job-history b/sg-report-job-history
index e4f35bf..76cb146 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -182,10 +182,11 @@ END
 	 FROM runvars
 	 JOIN flights USING (flight)
 	WHERE ($cond)
-	  AND flight >= (
+	  AND flight >= COALESCE(
+             (
 	      SELECT flight $fromstuff
 	      LIMIT 1 OFFSET $offset
-          )
+	     ), 0)
      ORDER BY name;
 END
     $hostsq->execute(@params, @params); # sql text contains $cond twice
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 08/25] sg-report-job-history: Show start time
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (6 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm Ian Jackson
                   ` (17 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 sg-report-job-history |    7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sg-report-job-history b/sg-report-job-history
index 76cb146..bbb85fd 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -251,7 +251,8 @@ END
         print H "<html><head><title>$title</title></head><body>\n";
         print H "<h1>$title</h1>\n";
         print H "<table rules=all>";
-        print H "<tr><th>flight</th><th>branch</th><th>failure</th>\n";
+        print H "<tr><th>started</th><th>flight</th>",
+	        "<th>branch</th><th>failure</th>\n";
         print H "<th>", (join ", ", @hostvarcols), "</th>\n";
         foreach my $c (@rev_grid_cols) {
             print H "<th>".encode_entities($c)."</th>\n";
@@ -264,6 +265,10 @@ END
             my $colour= "bgcolor=\"$r->{Colour}\"";
             my $altcolour= altcolour($alternate);
             print H "<tr $altcolour>";
+	    my $started = $r->{Flight}{started};
+	    print H "<td>";
+	    print H show_abs_time $started if $started;
+	    print H "</td>\n";
             my $flt= $r->{Flight}{flight};
             $url= "$c{ReportHtmlPubBaseUrl}/$flt";
             print H "<td><a href=\"$url\">$flt</a></td>\n";
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (7 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 08/25] sg-report-job-history: Show start time Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue Ian Jackson
                   ` (16 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

This is to support a forthcoming sg-report-host-history.

No functional change.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 Osstest/Executive.pm  |   45 +++++++++++++++++++++++++++++++++++++++++++++
 sg-report-job-history |   48 ++++--------------------------------------------
 2 files changed, 49 insertions(+), 44 deletions(-)

diff --git a/Osstest/Executive.pm b/Osstest/Executive.pm
index 118a91a..90c615a 100644
--- a/Osstest/Executive.pm
+++ b/Osstest/Executive.pm
@@ -45,6 +45,7 @@ BEGIN {
     @ISA         = qw(Exporter);
     @EXPORT      = qw(get_harness_rev grabrepolock_reexec
                       findtask @all_lock_tables
+                      report_run_getinfo report_altcolour
                       tcpconnect_queuedaemon plan_search
                       alloc_resources alloc_resources_rollback_begin_work
                       resource_check_allocated resource_shared_mark_ready
@@ -188,6 +189,50 @@ sub opendb ($) {
     return $dbh;
 }
 
+#---------- history reporting ----------
+
+our $green=  '#008800';
+our $red=    '#ff8888';
+our $yellow= '#ffff00';
+our $purple= '#ff00ff';
+
+sub report_run_getinfo ($) {
+    # $f is a joined flight/job row, must contain at least
+    #    flight job status
+    my ($f) = @_;
+    my $status= $f->{status};
+    if ($status eq 'pass') {
+        return { Summary => "($status)", Colour => $green };
+    } elsif ($status eq 'fail') {
+	our $failstepq //= db_prepare(<<END);
+	    SELECT * FROM steps
+	     WHERE flight=? AND job=?
+	       AND status!='pass'
+	  ORDER BY stepno
+	     LIMIT 1
+END
+        $failstepq->execute($f->{flight}, $f->{job});
+        my $fs= $failstepq->fetchrow_hashref();
+        if (!defined $fs) {
+            return { Summary => "(unknown)", Colour => $yellow };
+        } elsif ($fs->{status} eq 'fail') {
+            return { Summary => "$fs->{testid}", Colour => $red };
+        } else {
+            return { Summary => "$fs->{testid} $fs->{status}",
+                     Colour => $red };
+        }
+    } elsif ($status eq 'blocked') {
+        return { Summary => "blocked", Colour => $purple },
+    } else {
+        return { Summary => "($f->{status})", Colour => $yellow };
+    }
+}
+
+sub report_altcolour ($) {
+    my ($bool) = @_;
+    return "bgcolor=\"#".(qw(d0d0d0 ffffff))[$bool]."\"";
+}
+
 #---------- host (and other resource) allocation ----------
 
 our $taskid;
diff --git a/sg-report-job-history b/sg-report-job-history
index bbb85fd..18821b3 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -25,6 +25,7 @@ use IO::Handle;
 use HTML::Entities;
 
 use Osstest::TestSupport;
+use Osstest::Executive;
 
 our (@blessings,@branches);
 our $limit= 500;
@@ -84,42 +85,6 @@ if (defined($flight)) {
     push @jobs, $job;
 }
 
-our $failstepq= db_prepare(<<END);
-        SELECT * FROM steps
-         WHERE flight=? AND job=?
-           AND status!='pass'
-      ORDER BY stepno
-         LIMIT 1
-END
-
-our $green=  '#008800';
-our $red=    '#ff8888';
-our $yellow= '#ffff00';
-our $purple= '#ff00ff';
-
-sub run_getinfo ($) {
-    my ($f) = @_;
-    my $status= $f->{status};
-    if ($status eq 'pass') {
-        return { Summary => "($status)", Colour => $green };
-    } elsif ($status eq 'fail') {
-        $failstepq->execute($f->{flight}, $f->{job});
-        my $fs= $failstepq->fetchrow_hashref();
-        if (!defined $fs) {
-            return { Summary => "(unknown)", Colour => $yellow };
-        } elsif ($fs->{status} eq 'fail') {
-            return { Summary => "$fs->{testid}", Colour => $red };
-        } else {
-            return { Summary => "$fs->{testid} $fs->{status}",
-                     Colour => $red };
-        }
-    } elsif ($status eq 'blocked') {
-        return { Summary => "blocked", Colour => $purple },
-    } else {
-        return { Summary => "($f->{status})", Colour => $yellow };
-    }
-}
-
 our $revisionsq= db_prepare(<<END);
         SELECT * FROM runvars
          WHERE flight=? AND job=?
@@ -142,11 +107,6 @@ our $buildsq= db_prepare(<<END);
            AND name LIKE E'\%buildjob'
 END
 
-sub altcolour ($) {
-    my ($bool) = @_;
-    return "bgcolor=\"#".(qw(d0d0d0 ffffff))[$bool]."\"";
-}
-
 sub processjobbranch ($$) {
     my ($j,$bra) = @_;
 
@@ -201,7 +161,7 @@ END
 END
 
     while (my $f= $flightsq->fetchrow_hashref()) {
-        my $ri= run_getinfo($f);
+        my $ri= report_run_getinfo($f);
 
 	$ri->{Hosts} = [ ];
 	foreach my $hostvar (@hostvarcols) {
@@ -263,7 +223,7 @@ END
         my @alt_revs= ('0')x $#rev_grid_cols;
         foreach my $r (@test_rows) {
             my $colour= "bgcolor=\"$r->{Colour}\"";
-            my $altcolour= altcolour($alternate);
+            my $altcolour= report_altcolour($alternate);
             print H "<tr $altcolour>";
 	    my $started = $r->{Flight}{started};
 	    print H "<td>";
@@ -286,7 +246,7 @@ END
                     !defined($v) ? 0 :
                     $last_revs[$i] eq $v;
                 $alt_revs[$i] ^= !$same;
-                print H "<td ".altcolour($alt_revs[$i]).">";
+                print H "<td ".report_altcolour($alt_revs[$i]).">";
                 if (defined $v) {
                     my $vp= $v;
                     if (defined $lastrev && $v eq $lastrev) {
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (8 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
                   ` (15 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
---
 Osstest/Executive.pm |    5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Osstest/Executive.pm b/Osstest/Executive.pm
index 90c615a..6edbfee 100644
--- a/Osstest/Executive.pm
+++ b/Osstest/Executive.pm
@@ -52,9 +52,9 @@ BEGIN {
                       duration_estimator
                       db_pg_dsn opendb opendb_state
                       );
-    %EXPORT_TAGS = ( );
+    %EXPORT_TAGS = ( colours => [qw($green $red $yellow $purple $blue)] );
 
-    @EXPORT_OK   = qw();
+    @EXPORT_OK   = @{ $EXPORT_TAGS{colours} };
 }
 
 # DATABASE TABLE LOCK HIERARCHY
@@ -195,6 +195,7 @@ our $green=  '#008800';
 our $red=    '#ff8888';
 our $yellow= '#ffff00';
 our $purple= '#ff00ff';
+our $blue=   '#0000ff';
 
 sub report_run_getinfo ($) {
     # $f is a joined flight/job row, must contain at least
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (9 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-18  8:44   ` Ian Campbell
  2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
                   ` (14 subsequent siblings)
  25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 Osstest/Executive.pm |    9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Osstest/Executive.pm b/Osstest/Executive.pm
index 6edbfee..6c16fdd 100644
--- a/Osstest/Executive.pm
+++ b/Osstest/Executive.pm
@@ -204,7 +204,8 @@ sub report_run_getinfo ($) {
     my $status= $f->{status};
     if ($status eq 'pass') {
         return { Summary => "($status)", Colour => $green };
-    } elsif ($status eq 'fail') {
+    } elsif ($status eq 'fail' or $status eq 'broken') {
+	my $failcolour = $status eq 'fail' ? $red : $yellow;
 	our $failstepq //= db_prepare(<<END);
 	    SELECT * FROM steps
 	     WHERE flight=? AND job=?
@@ -217,10 +218,12 @@ END
         if (!defined $fs) {
             return { Summary => "(unknown)", Colour => $yellow };
         } elsif ($fs->{status} eq 'fail') {
-            return { Summary => "$fs->{testid}", Colour => $red };
+            return { Summary => "$fs->{testid}", Colour => $failcolour };
+        } elsif ($fs->{status} eq 'broken') {
+            return { Summary => "$fs->{testid} broken", Colour => $yellow };
         } else {
             return { Summary => "$fs->{testid} $fs->{status}",
-                     Colour => $red };
+                     Colour => $failcolour };
         }
     } elsif ($status eq 'blocked') {
         return { Summary => "blocked", Colour => $purple },
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 12/25] reporting: Move job histories into history/
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (10 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-18  8:45   ` Ian Campbell
  2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
                   ` (13 subsequent siblings)
  25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Replace some `.'s in HTML filenames with `/'s, making the directory
listings easier to deal with.

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 sg-report-flight      |    4 ++--
 sg-report-job-history |   10 ++++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/sg-report-flight b/sg-report-flight
index bdaf1c8..d3c0166 100755
--- a/sg-report-flight
+++ b/sg-report-flight
@@ -813,8 +813,8 @@ END
 <h1>$title</h1>
 <ul>
 <li><a href="../">Flight $fi->{Flight} scoreboard</a>
-<li><a href="$c{ResultsHtmlPubBaseUrl}/history.$job">History for $job</a>
-<li><a href="$c{ResultsHtmlPubBaseUrl}/history.$job.$branch">History for $job in tests of $branch</a>
+<li><a href="$c{ResultsHtmlPubBaseUrl}/history/$job/ALL">History for $job</a>
+<li><a href="$c{ResultsHtmlPubBaseUrl}/history/$job/$branch">History for $job in tests of $branch</a>
 <li><a href="./">Logfiles for $fi->{Flight} $job as webserver directory listing</a>
 </ul>
 <h2>Steps</h2>
diff --git a/sg-report-job-history b/sg-report-job-history
index 18821b3..900d998 100755
--- a/sg-report-job-history
+++ b/sg-report-job-history
@@ -198,12 +198,14 @@ END
 
     if (defined $htmlout) {
         my ($title,$html_file,$url);
+	ensuredir "$htmlout/history";
+	ensuredir "$htmlout/history/$j";
         if (defined $bra) {
-            $title= "$j $bra";
-            $html_file= "history.$j.$bra.html";
+            $title= "$j (branch $bra)";
+            $html_file= "history/$j/$bra.html";
         } else {
-            $title= $j;
-            $html_file= "history.$j.html";
+            $title= "$j (all branches)";
+            $html_file= "history/$j/ALL.html";
         }
         $html_file= "$htmlout/$html_file";
         open H, "> $html_file.new" or die "$html_file $!";
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 13/25] reporting: Move bisection report outputs
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (11 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-18  8:45   ` Ian Campbell
  2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
                   ` (12 subsequent siblings)
  25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Replace some `.'s in filenames with `/'s, making the directory
listings easier to deal with.

(I haven't been able to conveniently do a proper test of this change,
but I have dry-run the critical parts.)

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 cri-bisect |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cri-bisect b/cri-bisect
index af3f23f..03b2e29 100644
--- a/cri-bisect
+++ b/cri-bisect
@@ -20,8 +20,9 @@
 compute_state_core () {
         reportfile=tmp/$flight.bisection-report
         summaryfile=tmp/$flight.bisection-summary
-        bisleaf=bisect.$branch.$job.$laundered_testid
+        bisleaf=bisect/$branch/$job.$laundered_testid
         bisfile=$OSSTEST_HTML_DIR/$bisleaf
+	mkdir -p "${bisfile%/*}"
 
         echo "computing $branch $job $testid $*"
 
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (12 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-18  8:46   ` Ian Campbell
  2015-06-17 16:45 ` [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script Ian Jackson
                   ` (11 subsequent siblings)
  25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

The function findaflight should not, when the caller specifies a job,
find a flight which does not contain that job at all.

One effect of allowing it to find such flights is that it might find a
bisection flight and try to use it as a basis for claiming a
regression, or as a justification for something not being a
regression, and then complain that all the missing steps in the
bisection flight are `blocked'.

This can be seen in the report for 58627:
 test-amd64-i386-xl-qemuu-winxpsp3  6 xen-boot fail blocked in 56366-bisect

After this patch, a report generated for 58627 no longer mentions
56366 at all.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 sg-report-flight |   15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/sg-report-flight b/sg-report-flight
index d3c0166..6481521 100755
--- a/sg-report-flight
+++ b/sg-report-flight
@@ -178,10 +178,23 @@ END
         return undef;
     }
 
+    my @flightsq_params;
+    my $flightsq_jobcond='TRUE';
+    if (defined $job) {
+	push @flightsq_params, $job;
+	$flightsq_jobcond = <<END;
+                  EXISTS (SELECT 1
+			    FROM jobs
+			   WHERE jobs.flight = flights.flight
+			     AND jobs.job = ?)
+END
+    }
+
     my $flightsq= <<END;
       SELECT * FROM (
         SELECT flight, blessing FROM flights
             WHERE $branches_cond_q
+              AND $flightsq_jobcond
               AND $blessingscond
             ORDER BY flight DESC
             LIMIT 1000
@@ -189,7 +202,7 @@ END
       ORDER BY blessing ASC, flight DESC
 END
     $flightsq= db_prepare($flightsq);
-    $flightsq->execute(@blessings);
+    $flightsq->execute(@flightsq_params, @blessings);
 
     my $buildflightsq= db_prepare(<<END);
         SELECT val FROM runvars
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (13 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange Ian Jackson
                   ` (10 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

This script is still a work in progress.  It does currently mostly
work and can generate a useful report.  However, it needs:

 * Some database locking (to avoid deadlock errors)
 * A --flight= option to make it automatically report host usage
   for all hosts touched in a flight, for the use of cr-*
 * Hooking into cr-*

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
v2: Improved commit message.
    Fix a debug bug.
    Actually install the html; do not leave it as .new
---
 sg-report-host-history |  246 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 246 insertions(+)
 create mode 100755 sg-report-host-history

diff --git a/sg-report-host-history b/sg-report-host-history
new file mode 100755
index 0000000..14f8458
--- /dev/null
+++ b/sg-report-host-history
@@ -0,0 +1,246 @@
+#!/usr/bin/perl -w
+
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2013 Citrix Inc.
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use strict qw(vars);
+
+use DBI;
+use Osstest;
+use IO::Handle;
+use HTML::Entities;
+
+use Osstest::Executive qw(:DEFAULT :colours);
+
+our $limit= 200;
+our $flightlimit;
+our $htmlout = ".";
+our @blessings;
+
+open DEBUG, ">/dev/null";
+
+my $namecond= "(name = 'host' or name like '%_host')";
+csreadconfig();
+
+while (@ARGV && $ARGV[0] =~ m/^-/) {
+    $_= shift @ARGV;
+    last if m/^--?$/;
+    if (m/^--(limit)\=([1-9]\d*)$/) {
+        $$1= $2;
+    } elsif (m/^--flight-limit\=([1-9]\d*)$/) {
+	$flightlimit= $1;
+    } elsif (m/^--blessings?=(.*)$/) {
+        push @blessings, split ',', $1;
+    } elsif (m/^--html-dir=(.*)$/) {
+        $htmlout= $1;
+    } elsif (m/^--debug/) {
+        open DEBUG, ">&2" or die $!;
+        DEBUG->autoflush(1);
+    } else {
+        die "$_ ?";
+    }
+}
+@blessings= qw(real) if !@blessings;
+
+@ARGV or die $!;
+
+$dbh_tests->begin_work;
+
+if (!$flightlimit) {
+    my $flagscond =
+	'('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
+    my $nhostsq = db_prepare(<<END);
+        SELECT count(*)
+	  FROM resources r
+	 WHERE restype='host'
+	   AND EXISTS (SELECT 1
+		         FROM hostflags f
+		        WHERE f.hostname=r.resname
+		          AND $flagscond)
+END
+    $nhostsq->execute();
+    my ($nhosts) = $nhostsq->fetchrow_array();
+    print DEBUG "COUNTED $nhosts hosts\n";
+    $flightlimit = $nhosts * $limit * 2;
+}
+
+my $minflightsq = db_prepare(<<END);
+    SELECT flight
+      FROM (
+	SELECT flight
+	  FROM flights
+	 ORDER BY flight DESC
+	 LIMIT $flightlimit
+      ) f
+      ORDER BY flight ASC
+      LIMIT 1
+END
+$minflightsq->execute();
+my ($minflight) = $minflightsq->fetchrow_array();
+$minflight //= 0;
+
+our $flightcond = "(flight > $minflight)";
+
+$dbh_tests->do("SET LOCAL enable_seqscan=false");
+# Otherwise the PostgreSQL query planner likes to do a complete scan
+# of the runvars table, rather than walking backwards through the
+# flights until it has what we've told it is enough.
+
+my $runvarq = db_prepare(<<END);
+    SELECT flight, job, name, val
+      FROM runvars
+     WHERE $namecond
+       AND val = ?
+       AND $flightcond
+     ORDER BY flight DESC
+     LIMIT $limit * 2 + 100
+END
+
+my $endedq = db_prepare(<<END);
+    SELECT finished, testid, status AS laststepstatus
+      FROM steps
+     WHERE flight=? AND job=? AND finished IS NOT NULL
+     ORDER BY finished DESC
+     LIMIT 1
+END
+
+my $infoq = db_prepare(<<END);
+    SELECT blessing, branch, intended, status
+      FROM flights
+      JOIN jobs USING (flight)
+     WHERE flight=? AND job=?
+END
+
+my $allocdq = db_prepare(<<END);
+    SELECT testid, finished, status
+      FROM steps
+     WHERE flight=? AND job=?
+       AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
+     ORDER BY finished ASC
+     LIMIT 1
+END
+
+sub jobquery ($$) {
+    my ($q, $jr) = @_;
+    $q->execute($jr->{flight}, $jr->{job});
+    return $q->fetchrow_hashref();
+}
+
+sub reporthost ($) {
+    my ($hostname) = @_;
+
+    die if $hostname =~ m/[^-_.+0-9a-z]/;
+
+    my $html_file= "$htmlout/host.$hostname.html";
+    open H, "> $html_file.new" or die "$html_file $!";
+
+    my $title= "host history $hostname\n";
+    $title= encode_entities($title);
+    print H "<html><head><title>$title</title></head><body>\n";
+    print H "<h1>$title</h1>\n";
+    print H "<table rules=all><tr>\n";
+
+    print H "<th>alloc testid</th><th>alloc completed</th>\n";
+    print H "<th>job finished</th>\n";
+    print H "<th>role</th>\n";
+
+    print H "<th>flight</th>\n";
+    print H "<th>branch</th><th>intended</th><th>blessing</th>\n";
+
+    print H "<th>job</th><th>failure</th>\n";
+
+    print H "</tr>\n";
+
+    my @rows;
+    $runvarq->execute($hostname);
+
+    print DEBUG "FIRST PASS\n";
+    while (my $jr= $runvarq->fetchrow_hashref()) {
+	print DEBUG "JOB $jr->{flight}.$jr->{job} ";
+
+	my $endedrow = jobquery($endedq, $jr);
+	if (!$endedrow) {
+	    print DEBUG "no-finished\n";
+	    next;
+	}
+	print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
+	print DEBUG ".\n";
+
+	push @rows, { %$jr, %$endedrow };
+    }
+
+    print DEBUG "FOUND ", (scalar @rows), " ROWS\n";
+
+    @rows = sort { $b->{finished} <=> $a->{finished} } @rows;
+    $#rows = $limit-1 if @rows > $limit;
+
+    my $alternate = 0;
+    foreach my $jr (@rows) {
+	print DEBUG "JOB $jr->{flight}.$jr->{job}\n";
+
+	my $ir = jobquery($infoq, $jr);
+	my $ar = jobquery($allocdq, $jr);
+
+	my $altcolour = report_altcolour($alternate);
+	print H "<tr $altcolour>";
+
+	if (!defined $ar->{testid}) {
+	    print H "<td bgcolor=\"$red\"></td>";
+	    print H "<td>?</td>";
+	} else {
+	    if ($ar->{status} eq 'pass') {
+		print H "<td>$ar->{testid}</td>";
+		print H "<td>", (show_abs_time $ar->{finished}), "</td>";
+	    } elsif ($ar->{status} eq 'running') {
+		print H "<td bgcolor=\"$blue\">$ar->{testid}</td>";
+		print H "<td>$ar->{status}</td>";
+	    } else {
+		print H "<td bgcolor=\"$red\">$ar->{testid}</td>";
+		print H "<td>$ar->{status}</td>";
+	    }
+	}
+	print H "\n";
+
+	print H "<td>", (show_abs_time $jr->{finished}), "</td>\n";
+	print H "<td>", $jr->{name}, "</td>\n";
+
+	my $url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}";
+	print H "<td><a href=\"$url\">$jr->{flight}</a></td>\n";
+	$url= "$c{ReportHtmlPubBaseUrl}/$jr->{flight}/".
+	    encode_entities($jr->{job})."/";
+	print H "<td>$ir->{branch}</td>";
+	print H "<td>$ir->{intended}</td>";
+	print H "<td>";
+	print H $ir->{blessing} unless $ir->{blessing} eq 'running';
+	print H "</td>";
+
+	print H "<td><a href=\"$url\">$jr->{job}</td>\n";
+
+	my $ri = report_run_getinfo({ %$jr, %$ir });
+	print H "<td bgcolor=\"$ri->{Colour}\">$ri->{Summary}</td>\n";
+
+	print H "</tr>\n\n";
+	$alternate ^= 1;
+    }
+
+    print H "</table></body></html>\n";
+
+    close H or die $!;
+    rename "$html_file.new", "$html_file" or die "$html_file $!";
+}
+
+reporthost $_ foreach @ARGV;
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (14 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery Ian Jackson
                   ` (9 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

To do database locking coherently, it will be convenient to have this
in a function.

No functional change.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 sg-report-host-history |   65 +++++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index 14f8458..c56ed71 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -58,42 +58,44 @@ while (@ARGV && $ARGV[0] =~ m/^-/) {
 
 @ARGV or die $!;
 
-$dbh_tests->begin_work;
-
-if (!$flightlimit) {
-    my $flagscond =
-	'('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
-    my $nhostsq = db_prepare(<<END);
-        SELECT count(*)
-	  FROM resources r
-	 WHERE restype='host'
-	   AND EXISTS (SELECT 1
-		         FROM hostflags f
-		        WHERE f.hostname=r.resname
-		          AND $flagscond)
+our $flightcond;
+
+sub computeflightsrange () {
+    if (!$flightlimit) {
+	my $flagscond =
+	    '('.join(' OR ', map { "f.hostflag = 'blessed-$_'" } @blessings).')';
+	my $nhostsq = db_prepare(<<END);
+	    SELECT count(*)
+	      FROM resources r
+	     WHERE restype='host'
+	       AND EXISTS (SELECT 1
+			     FROM hostflags f
+			    WHERE f.hostname=r.resname
+			      AND $flagscond)
 END
-    $nhostsq->execute();
-    my ($nhosts) = $nhostsq->fetchrow_array();
-    print DEBUG "COUNTED $nhosts hosts\n";
-    $flightlimit = $nhosts * $limit * 2;
-}
+        $nhostsq->execute();
+	my ($nhosts) = $nhostsq->fetchrow_array();
+	print DEBUG "COUNTED $nhosts hosts\n";
+	$flightlimit = $nhosts * $limit * 2;
+    }
 
-my $minflightsq = db_prepare(<<END);
-    SELECT flight
-      FROM (
+    my $minflightsq = db_prepare(<<END);
 	SELECT flight
-	  FROM flights
-	 ORDER BY flight DESC
-	 LIMIT $flightlimit
-      ) f
-      ORDER BY flight ASC
-      LIMIT 1
+	  FROM (
+	    SELECT flight
+	      FROM flights
+	     ORDER BY flight DESC
+	     LIMIT $flightlimit
+	  ) f
+	  ORDER BY flight ASC
+	  LIMIT 1
 END
-$minflightsq->execute();
-my ($minflight) = $minflightsq->fetchrow_array();
-$minflight //= 0;
+    $minflightsq->execute();
+    my ($minflight) = $minflightsq->fetchrow_array();
+    $minflight //= 0;
 
-our $flightcond = "(flight > $minflight)";
+    $flightcond = "(flight > $minflight)";
+}
 
 $dbh_tests->do("SET LOCAL enable_seqscan=false");
 # Otherwise the PostgreSQL query planner likes to do a complete scan
@@ -243,4 +245,5 @@ sub reporthost ($) {
     rename "$html_file.new", "$html_file" or die "$html_file $!";
 }
 
+computeflightsrange();
 reporthost $_ foreach @ARGV;
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (15 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations Ian Jackson
                   ` (8 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

To get the scope of database locking, and of the SET LOCAL workaround,
right, it is convenient to move these query preparations into the
jobquery function.

No functional change.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 sg-report-host-history |   68 ++++++++++++++++++++++++------------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index c56ed71..f7a773d 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -102,40 +102,6 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
 # of the runvars table, rather than walking backwards through the
 # flights until it has what we've told it is enough.
 
-my $runvarq = db_prepare(<<END);
-    SELECT flight, job, name, val
-      FROM runvars
-     WHERE $namecond
-       AND val = ?
-       AND $flightcond
-     ORDER BY flight DESC
-     LIMIT $limit * 2 + 100
-END
-
-my $endedq = db_prepare(<<END);
-    SELECT finished, testid, status AS laststepstatus
-      FROM steps
-     WHERE flight=? AND job=? AND finished IS NOT NULL
-     ORDER BY finished DESC
-     LIMIT 1
-END
-
-my $infoq = db_prepare(<<END);
-    SELECT blessing, branch, intended, status
-      FROM flights
-      JOIN jobs USING (flight)
-     WHERE flight=? AND job=?
-END
-
-my $allocdq = db_prepare(<<END);
-    SELECT testid, finished, status
-      FROM steps
-     WHERE flight=? AND job=?
-       AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
-     ORDER BY finished ASC
-     LIMIT 1
-END
-
 sub jobquery ($$) {
     my ($q, $jr) = @_;
     $q->execute($jr->{flight}, $jr->{job});
@@ -167,6 +133,40 @@ sub reporthost ($) {
 
     print H "</tr>\n";
 
+    our $runvarq //= db_prepare(<<END);
+	SELECT flight, job, name, val
+	  FROM runvars
+	 WHERE $namecond
+	   AND val = ?
+	   AND $flightcond
+	 ORDER BY flight DESC
+	 LIMIT $limit * 2 + 100
+END
+
+    our $endedq //= db_prepare(<<END);
+	SELECT finished, testid, status AS laststepstatus
+	  FROM steps
+	 WHERE flight=? AND job=? AND finished IS NOT NULL
+	 ORDER BY finished DESC
+	 LIMIT 1
+END
+
+    our $infoq //= db_prepare(<<END);
+	SELECT blessing, branch, intended, status
+	  FROM flights
+	  JOIN jobs USING (flight)
+	 WHERE flight=? AND job=?
+END
+
+    our $allocdq //= db_prepare(<<END);
+	SELECT testid, finished, status
+	  FROM steps
+	 WHERE flight=? AND job=?
+	   AND (testid='hosts-allocate' OR step='ts-hosts-allocate')
+	 ORDER BY finished ASC
+	 LIMIT 1
+END
+
     my @rows;
     $runvarq->execute($hostname);
 
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (16 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts Ian Jackson
                   ` (7 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Arrange for the SET LOCAL to have the right scope.  Run
computeflightsrange, and reporthost, each within a db transaction,
with an appropriate lock.

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 sg-report-host-history |   21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index f7a773d..5a3f8be 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -97,11 +97,6 @@ END
     $flightcond = "(flight > $minflight)";
 }
 
-$dbh_tests->do("SET LOCAL enable_seqscan=false");
-# Otherwise the PostgreSQL query planner likes to do a complete scan
-# of the runvars table, rather than walking backwards through the
-# flights until it has what we've told it is enough.
-
 sub jobquery ($$) {
     my ($q, $jr) = @_;
     $q->execute($jr->{flight}, $jr->{job});
@@ -245,5 +240,17 @@ END
     rename "$html_file.new", "$html_file" or die "$html_file $!";
 }
 
-computeflightsrange();
-reporthost $_ foreach @ARGV;
+db_retry($dbh_tests, [qw(flights resources)], sub {
+    computeflightsrange();
+});
+
+$dbh_tests->do("SET LOCAL enable_seqscan=false");
+# Otherwise the PostgreSQL query planner likes to do a complete scan
+# of the runvars table, rather than walking backwards through the
+# flights until it has what we've told it is enough.
+
+foreach my $host (@ARGV) {
+    db_retry($dbh_tests, [qw(flights)], sub {
+	reporthost $host;
+    });
+}
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (17 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT Ian Jackson
                   ` (6 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

This will allow deduplication.  No functional change other than a
change to the order of processing.

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 sg-report-host-history |    6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sg-report-host-history b/sg-report-host-history
index 5a3f8be..8a4d40f 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -249,7 +249,13 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
 # of the runvars table, rather than walking backwards through the
 # flights until it has what we've told it is enough.
 
+our %hosts;
+
 foreach my $host (@ARGV) {
+    $hosts{$host}++;
+}
+
+foreach my $host (sort keys %hosts) {
     db_retry($dbh_tests, [qw(flights)], sub {
 	reporthost $host;
     });
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (18 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts Ian Jackson
                   ` (5 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 sg-report-host-history |   22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index 8a4d40f..b3ef63e 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -162,6 +162,8 @@ END
 	 LIMIT 1
 END
 
+    print DEBUG "QUERYING RUNVARS FOR $hostname\n";
+
     my @rows;
     $runvarq->execute($hostname);
 
@@ -252,7 +254,25 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
 our %hosts;
 
 foreach my $host (@ARGV) {
-    $hosts{$host}++;
+    if ($host =~ m/^flight:/) {
+	my $flight=$'; #';
+	db_retry($dbh_tests, [qw(flights)], sub {
+	    our $hostsinflightq //= db_prepare(<<END);
+	        SELECT DISTINCT val
+		  FROM runvars
+		 WHERE flight=?
+		   AND (name = 'host' or name like '%_host')
+END
+            $hostsinflightq->execute($flight);
+	    while (my $row = $hostsinflightq->fetchrow_hashref()) {
+		$hosts{$row->{val}}++;
+	    }
+	});
+    } elsif ($host !~ m/:/) {
+	$hosts{$host}++;
+    } else {
+	die "$host ?";
+    }
 }
 
 foreach my $host (sort keys %hosts) {
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (19 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction Ian Jackson
                   ` (4 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

This is much faster.  It might short-change unpopular hosts rather;
hence the change of the limit fudge factor from 2 to 3.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 sg-report-host-history |   96 +++++++++++++++++++++++++++---------------------
 1 file changed, 55 insertions(+), 41 deletions(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index b3ef63e..bab636e 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -23,6 +23,7 @@ use DBI;
 use Osstest;
 use IO::Handle;
 use HTML::Entities;
+use POSIX;
 
 use Osstest::Executive qw(:DEFAULT :colours);
 
@@ -103,6 +104,50 @@ sub jobquery ($$) {
     return $q->fetchrow_hashref();
 }
 
+our %hosts;
+
+sub mainquery () {
+    our $valcond = join " OR ", map { "val = ?" } keys %hosts;
+    our @params = keys %hosts;
+
+    our $runvarq //= db_prepare(<<END);
+	SELECT flight, job, name, val
+	  FROM runvars
+	 WHERE $namecond
+	   AND ($valcond)
+	   AND $flightcond
+	 ORDER BY flight DESC
+	 LIMIT ($limit * 3 + 100) * ?
+END
+
+    push @params, scalar keys %hosts;
+
+    our $endedq //= db_prepare(<<END);
+	SELECT finished, testid, status AS laststepstatus
+	  FROM steps
+	 WHERE flight=? AND job=? AND finished IS NOT NULL
+	 ORDER BY finished DESC
+	 LIMIT 1
+END
+
+    $runvarq->execute(@params);
+
+    print DEBUG "FIRST PASS\n";
+    while (my $jr= $runvarq->fetchrow_hashref()) {
+	print DEBUG "JOB $jr->{flight}.$jr->{job} ";
+
+	my $endedrow = jobquery($endedq, $jr);
+	if (!$endedrow) {
+	    print DEBUG "no-finished\n";
+	    next;
+	}
+	print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
+	print DEBUG ".\n";
+
+	push @{ $hosts{$jr->{val}} }, { %$jr, %$endedrow };
+    }
+}
+
 sub reporthost ($) {
     my ($hostname) = @_;
 
@@ -128,24 +173,6 @@ sub reporthost ($) {
 
     print H "</tr>\n";
 
-    our $runvarq //= db_prepare(<<END);
-	SELECT flight, job, name, val
-	  FROM runvars
-	 WHERE $namecond
-	   AND val = ?
-	   AND $flightcond
-	 ORDER BY flight DESC
-	 LIMIT $limit * 2 + 100
-END
-
-    our $endedq //= db_prepare(<<END);
-	SELECT finished, testid, status AS laststepstatus
-	  FROM steps
-	 WHERE flight=? AND job=? AND finished IS NOT NULL
-	 ORDER BY finished DESC
-	 LIMIT 1
-END
-
     our $infoq //= db_prepare(<<END);
 	SELECT blessing, branch, intended, status
 	  FROM flights
@@ -162,27 +189,10 @@ END
 	 LIMIT 1
 END
 
-    print DEBUG "QUERYING RUNVARS FOR $hostname\n";
-
     my @rows;
-    $runvarq->execute($hostname);
-
-    print DEBUG "FIRST PASS\n";
-    while (my $jr= $runvarq->fetchrow_hashref()) {
-	print DEBUG "JOB $jr->{flight}.$jr->{job} ";
-
-	my $endedrow = jobquery($endedq, $jr);
-	if (!$endedrow) {
-	    print DEBUG "no-finished\n";
-	    next;
-	}
-	print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
-	print DEBUG ".\n";
+    @rows = @{ $hosts{$hostname} };
 
-	push @rows, { %$jr, %$endedrow };
-    }
-
-    print DEBUG "FOUND ", (scalar @rows), " ROWS\n";
+    print DEBUG "FOUND ", (scalar @rows), " ROWS for $hostname\n";
 
     @rows = sort { $b->{finished} <=> $a->{finished} } @rows;
     $#rows = $limit-1 if @rows > $limit;
@@ -251,8 +261,6 @@ $dbh_tests->do("SET LOCAL enable_seqscan=false");
 # of the runvars table, rather than walking backwards through the
 # flights until it has what we've told it is enough.
 
-our %hosts;
-
 foreach my $host (@ARGV) {
     if ($host =~ m/^flight:/) {
 	my $flight=$'; #';
@@ -265,16 +273,22 @@ foreach my $host (@ARGV) {
 END
             $hostsinflightq->execute($flight);
 	    while (my $row = $hostsinflightq->fetchrow_hashref()) {
-		$hosts{$row->{val}}++;
+		$hosts{$row->{val}} = [ ];
 	    }
 	});
     } elsif ($host !~ m/:/) {
-	$hosts{$host}++;
+	$hosts{$host} = [ ];
     } else {
 	die "$host ?";
     }
 }
 
+exit 0 unless %hosts;
+
+db_retry($dbh_tests, [qw(flights)], sub {
+    mainquery();
+});
+
 foreach my $host (sort keys %hosts) {
     db_retry($dbh_tests, [qw(flights)], sub {
 	reporthost $host;
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (20 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete" Ian Jackson
                   ` (3 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

No substantial change, but reduces the work done in the main
transaction.

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 sg-report-host-history |   46 ++++++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index bab636e..cd9ec61 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -122,29 +122,12 @@ END
 
     push @params, scalar keys %hosts;
 
-    our $endedq //= db_prepare(<<END);
-	SELECT finished, testid, status AS laststepstatus
-	  FROM steps
-	 WHERE flight=? AND job=? AND finished IS NOT NULL
-	 ORDER BY finished DESC
-	 LIMIT 1
-END
-
     $runvarq->execute(@params);
 
     print DEBUG "FIRST PASS\n";
     while (my $jr= $runvarq->fetchrow_hashref()) {
 	print DEBUG "JOB $jr->{flight}.$jr->{job} ";
-
-	my $endedrow = jobquery($endedq, $jr);
-	if (!$endedrow) {
-	    print DEBUG "no-finished\n";
-	    next;
-	}
-	print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
-	print DEBUG ".\n";
-
-	push @{ $hosts{$jr->{val}} }, { %$jr, %$endedrow };
+	push @{ $hosts{$jr->{val}} }, $jr;
     }
 }
 
@@ -173,6 +156,14 @@ sub reporthost ($) {
 
     print H "</tr>\n";
 
+    our $endedq //= db_prepare(<<END);
+	SELECT finished, testid, status AS laststepstatus
+	  FROM steps
+	 WHERE flight=? AND job=? AND finished IS NOT NULL
+	 ORDER BY finished DESC
+	 LIMIT 1
+END
+
     our $infoq //= db_prepare(<<END);
 	SELECT blessing, branch, intended, status
 	  FROM flights
@@ -189,18 +180,29 @@ END
 	 LIMIT 1
 END
 
+    my $inrows = $hosts{$hostname};
+    print DEBUG "FOUND ", (scalar @$inrows), " ROWS for $hostname\n";
+
     my @rows;
-    @rows = @{ $hosts{$hostname} };
+    foreach my $jr (@$inrows) {
+	print DEBUG "JOB $jr->{flight}.$jr->{job}\n";
 
-    print DEBUG "FOUND ", (scalar @rows), " ROWS for $hostname\n";
+	my $endedrow = jobquery($endedq, $jr);
+	if (!$endedrow) {
+	    print DEBUG "no-finished\n";
+	    next;
+	}
+	print DEBUG join " ", map { $endedrow->{$_} } sort keys %$endedrow;
+	print DEBUG ".\n";
+
+	push @rows, { %$jr, %$endedrow };
+    }
 
     @rows = sort { $b->{finished} <=> $a->{finished} } @rows;
     $#rows = $limit-1 if @rows > $limit;
 
     my $alternate = 0;
     foreach my $jr (@rows) {
-	print DEBUG "JOB $jr->{flight}.$jr->{job}\n";
-
 	my $ir = jobquery($infoq, $jr);
 	my $ar = jobquery($allocdq, $jr);
 
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete"
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (21 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/ Ian Jackson
                   ` (2 subsequent siblings)
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Since they may in fact be abandoned.

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 sg-report-host-history |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index cd9ec61..ad9f248 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -218,7 +218,7 @@ END
 		print H "<td>", (show_abs_time $ar->{finished}), "</td>";
 	    } elsif ($ar->{status} eq 'running') {
 		print H "<td bgcolor=\"$blue\">$ar->{testid}</td>";
-		print H "<td>$ar->{status}</td>";
+		print H "<td>(incomplete)</td>";
 	    } else {
 		print H "<td bgcolor=\"$red\">$ar->{testid}</td>";
 		print H "<td>$ar->{status}</td>";
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (22 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete" Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
  2015-06-18  9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
  25 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 sg-report-host-history |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sg-report-host-history b/sg-report-host-history
index ad9f248..f4e73e9 100755
--- a/sg-report-host-history
+++ b/sg-report-host-history
@@ -136,7 +136,7 @@ sub reporthost ($) {
 
     die if $hostname =~ m/[^-_.+0-9a-z]/;
 
-    my $html_file= "$htmlout/host.$hostname.html";
+    my $html_file= "$htmlout/$hostname.html";
     open H, "> $html_file.new" or die "$html_file $!";
 
     my $title= "host history $hostname\n";
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (23 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/ Ian Jackson
@ 2015-06-17 16:45 ` Ian Jackson
  2015-06-18  8:47   ` Ian Campbell
  2015-06-18  9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
  25 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-17 16:45 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, Ian Campbell

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>
---
 cri-args-hostlists |    5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cri-args-hostlists b/cri-args-hostlists
index 08e733d..a4e57b3 100644
--- a/cri-args-hostlists
+++ b/cri-args-hostlists
@@ -108,6 +108,7 @@ start_email () {
 
 	local flight_html_dir=$OSSTEST_HTMLPUB_DIR/
 	local job_html_dir=$OSSTEST_HTML_DIR/
+	local host_html_dir=$OSSTEST_HTML_DIR/host/
 
 	globallockdir=`getconfig GlobalLockDir`
 
@@ -117,6 +118,10 @@ start_email () {
 	./sg-report-flight --html-dir=$flight_html_dir/$flight/ \
 		--allow=allow.all --allow=allow.$branch \
 		$sgr_args $flight
+
+	mkdir -p $host_html_dir
+	with-lock-ex -w $globallockdir/report-lock \
+	  ./sg-report-host-history --html-dir=$host_html_dir flight:$flight
 }
 
 publish_send_email () {
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs
  2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
@ 2015-06-18  8:44   ` Ian Campbell
  0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18  8:44 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>

Acked-by: Ian Campbell <ian.campbell@citrix.com>

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 12/25] reporting: Move job histories into history/
  2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
@ 2015-06-18  8:45   ` Ian Campbell
  0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18  8:45 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Replace some `.'s in HTML filenames with `/'s, making the directory
> listings easier to deal with.
> 
> Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>

Acked-by: Ian Campbell <ian.campbell@citrix.com>

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 13/25] reporting: Move bisection report outputs
  2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
@ 2015-06-18  8:45   ` Ian Campbell
  0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18  8:45 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Replace some `.'s in filenames with `/'s, making the directory
> listings easier to deal with.
> 
> (I haven't been able to conveniently do a proper test of this change,
> but I have dry-run the critical parts.)
> 
> Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>

Acked-by: Ian Campbell <ian.campbell@citrix.com>

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs
  2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
@ 2015-06-18  8:46   ` Ian Campbell
  0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18  8:46 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> The function findaflight should not, when the caller specifies a job,
> find a flight which does not contain that job at all.
> 
> One effect of allowing it to find such flights is that it might find a
> bisection flight and try to use it as a basis for claiming a
> regression, or as a justification for something not being a
> regression, and then complain that all the missing steps in the
> bisection flight are `blocked'.
> 
> This can be seen in the report for 58627:
>  test-amd64-i386-xl-qemuu-winxpsp3  6 xen-boot fail blocked in 56366-bisect
> 
> After this patch, a report generated for 58627 no longer mentions
> 56366 at all.
> 
> Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>

Acked-by: Ian Campbell <ian.campbell@citrix.com>

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history
  2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
@ 2015-06-18  8:47   ` Ian Campbell
  0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18  8:47 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Wed, 2015-06-17 at 17:45 +0100, Ian Jackson wrote:
> Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>

Acked-by: Ian Campbell <ian.campbell@citrix.com>

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 00/25] Reporting improvements
  2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
                   ` (24 preceding siblings ...)
  2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
@ 2015-06-18  9:06 ` Ian Campbell
  2015-06-18 11:21   ` Ian Jackson
  25 siblings, 1 reply; 37+ messages in thread
From: Ian Campbell @ 2015-06-18  9:06 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Wed, 2015-06-17 at 17:44 +0100, Ian Jackson wrote:
> Most of the early part of this series is straightforward (and acked)
> and could go in whenever.

Agreed. Do you happen to have a convenient directory with the new
output? (I'm happy to just wait for it to happen too).

> Patches 15-24 introduce sg-report-host-history.  It's probably easiest
> to simply review the final version of the script, which can be found
> below.  The history is preserved mostly for archeological purposes.

Sure.

> 
> Ian.
> 
> #!/usr/bin/perl -w
> 
> # This is part of "osstest", an automated testing framework for Xen.
> # Copyright (C) 2009-2013 Citrix Inc.

Date ;-)

I couldn't spot anything else, FWIW. I think if you are happy with the
output there's no reason not to just run with it.

Ian.

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 00/25] Reporting improvements
  2015-06-18  9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
@ 2015-06-18 11:21   ` Ian Jackson
  2015-06-18 14:21     ` Ian Campbell
  0 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-18 11:21 UTC (permalink / raw)
  To: Ian Campbell; +Cc: xen-devel

Ian Campbell writes ("Re: [OSSTEST PATCH 00/25] Reporting improvements"):
> On Wed, 2015-06-17 at 17:44 +0100, Ian Jackson wrote:
> > Most of the early part of this series is straightforward (and acked)
> > and could go in whenever.
> 
> Agreed. Do you happen to have a convenient directory with the new
> output? (I'm happy to just wait for it to happen too).

I have mostly run it on my workstation on the Cambridge instance.
Here is an example that I have lying about, copied to a public
webserver:
  http://www.chiark.greenend.org.uk/~ijackson/quicksand/2015/grain-weevil.html

> > Patches 15-24 introduce sg-report-host-history.  It's probably easiest
> > to simply review the final version of the script, which can be found
> > below.  The history is preserved mostly for archeological purposes.
> 
> Sure.

Thanks.

> > #!/usr/bin/perl -w
> > 
> > # This is part of "osstest", an automated testing framework for Xen.
> > # Copyright (C) 2009-2013 Citrix Inc.
> 
> Date ;-)
> 
> I couldn't spot anything else, FWIW. I think if you are happy with the
> output there's no reason not to just run with it.

I'll give you a chance to look at the output URL above.

Ian.

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 00/25] Reporting improvements
  2015-06-18 11:21   ` Ian Jackson
@ 2015-06-18 14:21     ` Ian Campbell
  2015-06-18 14:36       ` Ian Jackson
  0 siblings, 1 reply; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 14:21 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Thu, 2015-06-18 at 12:21 +0100, Ian Jackson wrote:
> Ian Campbell writes ("Re: [OSSTEST PATCH 00/25] Reporting improvements"):
> > On Wed, 2015-06-17 at 17:44 +0100, Ian Jackson wrote:
> > > Most of the early part of this series is straightforward (and acked)
> > > and could go in whenever.
> > 
> > Agreed. Do you happen to have a convenient directory with the new
> > output? (I'm happy to just wait for it to happen too).
> 
> I have mostly run it on my workstation on the Cambridge instance.
> Here is an example that I have lying about, copied to a public
> webserver:
>   http://www.chiark.greenend.org.uk/~ijackson/quicksand/2015/grain-weevil.html
> 
> > [...]
> 
> I'll give you a chance to look at the output URL above.

Looks good, thanks.

OOI what is the expected usefulness of the "alloc testid" column?

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 00/25] Reporting improvements
  2015-06-18 14:21     ` Ian Campbell
@ 2015-06-18 14:36       ` Ian Jackson
  0 siblings, 0 replies; 37+ messages in thread
From: Ian Jackson @ 2015-06-18 14:36 UTC (permalink / raw)
  To: Ian Campbell; +Cc: xen-devel

Ian Campbell writes ("Re: [OSSTEST PATCH 00/25] Reporting improvements"):
> On Thu, 2015-06-18 at 12:21 +0100, Ian Jackson wrote:
> > I'll give you a chance to look at the output URL above.
> 
> Looks good, thanks.
> 
> OOI what is the expected usefulness of the "alloc testid" column?

With the system as it currently exists, it will always say
"hosts-allocate".  But that's not a fixed assumption of the overall
resource allocation design.

Ian.

^ permalink raw reply	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging
  2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
@ 2015-06-18 15:33   ` Ian Jackson
  2015-06-18 15:46     ` Ian Campbell
  0 siblings, 1 reply; 37+ messages in thread
From: Ian Jackson @ 2015-06-18 15:33 UTC (permalink / raw)
  To: xen-devel, Ian Campbell

Ian Jackson writes ("[OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging"):
> No callers, so no functional change, as yet.

> +sub db_prepare ($) {
> +    # caller must ensure global filehandle DEBUG is open
> +    my ($stmt) = @_;
> +    print ::DEBUG "DB PREPARING:\n$stmt\n";
> +    return $dbh_tests->prepare($stmt);

This generates:

Name "main::DEBUG" used only once: possible typo at Osstest.pm line
274.

Here is a v2 of this patch which fixes it.

Ian.

commit bd7cf4f0f1a8c45d840528bc28dbd89b34732d7a
Author: Ian Jackson <ian.jackson@eu.citrix.com>
Date:   Fri May 29 13:19:39 2015 +0000

    Osstest.pm: Provide new db_prepare helper with built-in debugging
    
    No callers, so no functional change, as yet.
    
    Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
    Acked-by: Ian Campbell <ian.campbell@citrix.com>
    ---
    v2: Suppress "used only once" warning

diff --git a/Osstest.pm b/Osstest.pm
index e8bd77b..8948666 100644
--- a/Osstest.pm
+++ b/Osstest.pm
@@ -35,7 +35,7 @@ BEGIN {
                       getmethod
                       postfork
                       $dbh_tests db_retry db_retry_retry db_retry_abort
-                      db_begin_work
+                      db_begin_work db_prepare
                       ensuredir get_filecontents_core_quiet system_checked
                       nonempty visible_undef show_abs_time
                       );
@@ -49,6 +49,10 @@ our $mjobdb;
 
 our $dbh_tests;
 
+scalar *main::DEBUG;
+# declaration prevents `Name "main::DEBUG" used only once'
+# scalar prevents `useless use of a variable in void context'
+
 #---------- static default config settings ----------
 
 our %c = qw(
@@ -268,6 +272,13 @@ sub db_retry ($$$;$$) {
     return $r;
 }
 
+sub db_prepare ($) {
+    # caller must ensure global filehandle DEBUG is open
+    my ($stmt) = @_;
+    print ::DEBUG "DB PREPARING:\n$stmt\n";
+    return $dbh_tests->prepare($stmt);
+}
+
 sub postfork () {
     $mjobdb->jobdb_postfork();
 }

^ permalink raw reply related	[flat|nested] 37+ messages in thread

* Re: [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging
  2015-06-18 15:33   ` Ian Jackson
@ 2015-06-18 15:46     ` Ian Campbell
  0 siblings, 0 replies; 37+ messages in thread
From: Ian Campbell @ 2015-06-18 15:46 UTC (permalink / raw)
  To: Ian Jackson; +Cc: xen-devel

On Thu, 2015-06-18 at 16:33 +0100, Ian Jackson wrote:
> Ian Jackson writes ("[OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging"):
> > No callers, so no functional change, as yet.
> 
> > +sub db_prepare ($) {
> > +    # caller must ensure global filehandle DEBUG is open
> > +    my ($stmt) = @_;
> > +    print ::DEBUG "DB PREPARING:\n$stmt\n";
> > +    return $dbh_tests->prepare($stmt);
> 
> This generates:
> 
> Name "main::DEBUG" used only once: possible typo at Osstest.pm line
> 274.
> 
> Here is a v2 of this patch which fixes it.
> 
> Ian.
> 
> commit bd7cf4f0f1a8c45d840528bc28dbd89b34732d7a
> Author: Ian Jackson <ian.jackson@eu.citrix.com>
> Date:   Fri May 29 13:19:39 2015 +0000
> 
>     Osstest.pm: Provide new db_prepare helper with built-in debugging
>     
>     No callers, so no functional change, as yet.
>     
>     Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
>     Acked-by: Ian Campbell <ian.campbell@citrix.com>
>     ---
>     v2: Suppress "used only once" warning
> 
> diff --git a/Osstest.pm b/Osstest.pm
> index e8bd77b..8948666 100644
> --- a/Osstest.pm
> +++ b/Osstest.pm
> @@ -35,7 +35,7 @@ BEGIN {
>                        getmethod
>                        postfork
>                        $dbh_tests db_retry db_retry_retry db_retry_abort
> -                      db_begin_work
> +                      db_begin_work db_prepare
>                        ensuredir get_filecontents_core_quiet system_checked
>                        nonempty visible_undef show_abs_time
>                        );
> @@ -49,6 +49,10 @@ our $mjobdb;
>  
>  our $dbh_tests;
>  
> +scalar *main::DEBUG;
> +# declaration prevents `Name "main::DEBUG" used only once'
> +# scalar prevents `useless use of a variable in void context'

um, wow ;-)

I suppose you will push this?

^ permalink raw reply	[flat|nested] 37+ messages in thread

end of thread, other threads:[~2015-06-18 16:24 UTC | newest]

Thread overview: 37+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2015-06-17 16:44 [OSSTEST PATCH 00/25] Reporting improvements Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 01/25] Osstest.pm: Provide new db_prepare helper with built-in debugging Ian Jackson
2015-06-18 15:33   ` Ian Jackson
2015-06-18 15:46     ` Ian Campbell
2015-06-17 16:44 ` [OSSTEST PATCH 02/25] sg-report-job-history: Use db_prepare Ian Jackson
2015-06-17 16:44 ` [OSSTEST PATCH 03/25] cs-bisection-step: " Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 04/25] sg-report-flight: " Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 05/25] sg-report-job-history: Add a debugging statement Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 06/25] sg-report-job-history: Slightly prettify sql Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 07/25] sg-report-job-history: Cope if history too short Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 08/25] sg-report-job-history: Show start time Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 09/25] reporting: Move report_run_getinfo and some colours into Executive.pm Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 10/25] reporting: Add colours as optional export tag, and provide $blue Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 11/25] reporting: Show slightly better info for broken jobs Ian Jackson
2015-06-18  8:44   ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 12/25] reporting: Move job histories into history/ Ian Jackson
2015-06-18  8:45   ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 13/25] reporting: Move bisection report outputs Ian Jackson
2015-06-18  8:45   ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 14/25] reporting: sg-report-flight should ignore missing jobs Ian Jackson
2015-06-18  8:46   ` Ian Campbell
2015-06-17 16:45 ` [OSSTEST PATCH 15/25] sg-report-host-history: Introduce new script Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 16/25] sg-report-host-history: Break out computeflightsrange Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 17/25] sg-report-host-history: Move query preparation into jobquery Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 18/25] sg-report-host-history: Move database manipulations Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 19/25] sg-report-host-history: Use a hash for hosts Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 20/25] sg-report-host-history: Support flight:FLIGHT Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 21/25] sg-report-host-history: Aggregate runvars query for all hosts Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 22/25] sg-report-host-history: Move per-row endedq query into per-host transaction Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 23/25] sg-report-host-history: Show "running" jobs as "incomplete" Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 24/25] sg-report-host-history: Make --html-dir have to be host/ Ian Jackson
2015-06-17 16:45 ` [OSSTEST PATCH 25/25] cri-args-hostlists: Run sg-report-host-history Ian Jackson
2015-06-18  8:47   ` Ian Campbell
2015-06-18  9:06 ` [OSSTEST PATCH 00/25] Reporting improvements Ian Campbell
2015-06-18 11:21   ` Ian Jackson
2015-06-18 14:21     ` Ian Campbell
2015-06-18 14:36       ` Ian Jackson

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.