[PATCH RFC 1/2] scripts: Add script to do the repetitive bits of the release process

* [PATCH RFC 1/2] scripts: Add script to do the repetitive bits of the release process
@ 2019-04-05 17:13 ` George Dunlap
  0 siblings, 0 replies; 8+ messages in thread
From: George Dunlap @ 2019-04-05 17:13 UTC (permalink / raw)
  To: xen-devel; +Cc: Ian Jackson, George Dunlap

With this script, once the main checks are out of the way, doing a
release (either an RC or the final release) should mostly be a matter
of executing a sequence of 4 commands given by the `help` function in
this script.

Signed-off-by: George Dunlap <george.dunlap@citrix.com>
---
There's one hard-coded "default" path in here that refers to my own
directory structure.  If Ian finds these scripts useful, we should
probably move that to a copy on mail.xenproject.org somewhere instead.

There are also lots of opportunities for this script to be improved,
by (for instance) implementing programmatic checks for the various
checks listed as 'manual' at the moment.

I plan to implement containerize-able tests for the first three steps
(tag, make tarball, push tag), using "dummy" paths and gpg keys.  I've
made revisions to tarball-cvs-checkin-and-post which I haven't had the
opportunity to test yet; ideas for how to keep this "fresh" are
welcome.

CC: Ian Jackson <ian.jackson@citrix.com>
---
 scripts/release | 450 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 450 insertions(+)
 create mode 100755 scripts/release

diff --git a/scripts/release b/scripts/release
new file mode 100755
index 0000000000..0442cd4ef9
--- /dev/null
+++ b/scripts/release
@@ -0,0 +1,450 @@
+#!/bin/bash
+
+###
+# George's bash library core
+###
+
+# arg-parse debug
+_apd=false
+
+arg_parse_cmd=\
+"local -a args;
+local _a;
+local _vn;
+local _m;
+local CLVL=\$((\$CLVL+1))
+
+_m=true;
+
+for _a in \"\$@\" ; do
+    $_apd && echo \"Evaluating \${_a} [[ \"\${_a/=}\" = \"\${_a}\" ]]\";
+    if \$_m && [[ \"\${_a/=}\" != \"\${_a}\" ]] ; then
+        $_apd && echo Parameter;
+        _vn=\${_a%%=*};
+        eval \"local \$_vn\";
+        eval \"\$_a\";
+    elif \$_m && [[ \"\${_a}\" == \"--\" ]] ; then
+        $_apd && echo Separator;
+        _m=false;
+    else
+        $_apd && echo Argument;
+        _m=false;
+        args+=(\"\$_a\");
+    fi;
+done"
+
+arg_parse="eval $arg_parse_cmd"
+
+# Pass in either the current function name, or the name of the script
+requireargs="eval _func=\"\$FUNCNAME\" ; eval [[ -n \\\"\$_func\\\" ]] || _func=\$0 ; eval _require-args \$_func"
+
+function _require-args()
+{
+    local _arg
+    local _args
+
+    _args=($@)
+
+    for _arg in ${_args[@]:1} ; do
+	eval "[[ -n \"\${$_arg}\" ]] || fail \"${_args[0]}: Missing $_arg\""
+    done
+}
+
+function default()
+{
+    # l0: eval i="5"
+    # l1: default_post="eval $1=\"$2\""
+    # l3: eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval \$1=\\\"$2\\\"\" ; fi"
+    eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval local \$1=\\\"$2\\\"\" ; else unset default_post ; fi"
+}
+
+function fail()
+{
+   echo FATAL $@
+   [[ -n "$fail_cleanup" ]] && $fail_cleanup
+   exit 1
+}
+
+function info()
+{
+   echo INFO $CLVL $@ 1>&2
+}
+
+function error()
+{
+   echo ERROR $@ 1>&2
+}
+
+function status()
+{
+   echo STATUS $CLVL $@ 1>&2
+   return 0
+}
+
+function report-result()
+{
+    if [[ -n "$var" ]] ; then
+	eval "${var}=\"$1\""
+    else
+	if [[ -n "$1" ]] ; then
+	    echo "$1"
+	else
+	    echo "(empty)"
+	fi
+    fi
+}
+
+function cmdline()
+{
+    local cmd;
+
+    if [[ "$#" -eq "0" ]] ; then
+	help
+	exit 1
+    fi
+
+    $arg_parse
+    info Running "${args[0]}"
+    "${args[0]}" "${args[@]:1}" || exit 1
+
+    if ! [[ -z "$RET" ]] ; then
+	echo $RET
+    fi
+}
+
+###
+# release-specific code
+###
+
+# Global / meta variables:
+#
+# tdir: "root" directory to do tarball work.
+# rdir: Directory where tarball & sig will be put (==$tdir/$v)
+# rtgz: Base filename for tarball ($rdir/xen-$v.tar.gz)
+#
+# v: Full release version (e.g., 4.12.0-rc5, 4.10.3)
+# x: Major+minor xen release version (e.g., 4.12, 4.10)
+# p: point release (e.g., 0 in 4.12.0; 3 in 4.10.3)
+# #r: Numbers-only release (e.g., 4.12.0, 4.10.3) # PROBABLY NOT NEEDED
+# rc: -rcN
+#
+# s: branch name (e.g., master, stable-4.12, stable-4.10)
+# t: Tag from a given release (e.g,. 4.12.0-rc5, RELEASE-4.10.3)
+# isrc: Boolean indicating whether the version is an rc (e.g., true for 4.12.0-rc5, false for 4.10.3)
+
+
+
+function xen-make-prefix-config() {
+    $arg_parse
+
+    # TODO: Ping drall.uk.xensource.com to see if we can reach it?
+    
+    default cache_prefix "git://drall.uk.xensource.com:9419/" ; $default_post
+
+    perl -ne "if(/^([A-Z_]+_URL) \?= (git.*)/) { print \"\$1 ?= ${cache_prefix}\$2\n\"; }" Config.mk >> .config || fail "Generating .config"
+    cat .config
+}
+
+function set-tdir() {
+    if [[ -z "$tdir" || ! -e "$tdir" ]] ; then
+	info "$tdir doesn't exist, using /tmp"
+	tdir="/tmp"
+    fi
+}
+
+# Take `v` and generate the appropriate metavariables variables.
+function parse-version() {
+    $arg_parse
+
+    $requireargs v
+
+    if [[ -n "$x" && -n "$p" ]] ; then
+	echo "Version already parsed"
+	return
+    fi
+
+    if [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9])(-rc[0-9]) ]] ; then
+	x=${BASH_REMATCH[1]}
+	p=${BASH_REMATCH[2]}
+	rc=${BASH_REMATCH[3]}
+	isrc=true
+    elif [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9]) ]] ; then
+	x=${BASH_REMATCH[1]}
+	p=${BASH_REMATCH[2]}
+	isrc=false
+    else
+	fail "Bad version"
+    fi
+
+    if $isrc ; then
+	t=$v
+    else
+	t=RELEASE-$v
+    fi
+}
+
+function check() {
+    # TODO: Automate some of these
+    info "Please perform manually: All XSAs  have been applied"
+    info "Please perform manually: Check http://logs.test-lab.xenproject.org/osstest/results/all-branch-statuses.txt"
+    info "Please perform manually: Check version in README"
+    info "Please perform manually: Check version in SUPPORT.md"
+    info "Please perform manually: Tags for appropriate *_REVISION's in Config.mk"
+    info "Please perform manually: xen/Makefile:XEN_EXTRAVERSION set to 0"
+    info "Please perform manually: tools/Rules.mk: debug ?= n"
+    info "Please perform manually: xen/Kconfig.debug:config DEBUG should default to `n`"
+}
+
+# Usage:
+#   tag v=[version you want to release] [c=commithash]
+# eg.
+#   tag v=4.12.0-rc6
+# Other arguments:
+#  key:  Name of key to sign the commit with
+#  tdir: Name of top-level tarball directory
+function tag() {
+    $arg_parse
+
+    default key "23E3222C145F4475FA8060A783FE14C957E82BD9"; $default_post
+
+    $requireargs v
+
+    set-tdir
+
+    $requireargs tdir
+
+    parse-version
+
+    $requireargs t
+
+    git fetch origin
+
+    if [[ -n "$c" ]] ; then
+	info "Checking out commit $c"
+	git checkout $c || fail
+    else
+	local q
+	git checkout stable-$x || fail "Couldn't check out stable branch"
+	git merge || fail "Merge"
+	git log -n 10
+	read -p "Enter to continue, anything else to quit: " q
+	[[ -z "$q" ]] || return
+    fi
+
+    # FIXME: Add checks:
+    # - Make sure Config.mk has tags, not hashes
+    # - sonames?
+    # - Appropriate version numbers in SUPPORT.md, xen/Makefile, &c
+
+    echo git tag -u "$key" -s -m "Xen $v" $t ; sleep 1
+    git tag -u "$key" -s -m "Xen $v" $t || fail "Creating signed tag"
+
+    info "Release tagged.  Now run release make-tarball v=$v"
+}
+
+function push-tag() {
+    $arg_parse
+
+    $requireargs v
+
+    parse-version
+
+    git push origin $t || fail "Pushing tag"
+    # FIXME: This is in the release checklist, but I'm not sure why
+    # git push origin staging-$x || fail "Pushing tag commit"
+
+    info "Tag pushed.  Now run release tarball-cvs-checkin-and-post v=$v"
+}
+
+function make-tarball-only()
+{
+    $arg_parse
+
+    $requireargs v tdir
+
+    parse-version
+
+    git fetch || fail "git fetch"
+    
+    git checkout $t || fail "Checking out tag $t"
+
+    git clean -ffdx
+
+    xen-make-prefix-config
+
+    ./configure || fail "Configuring"
+    
+    if $isrc ; then
+	make src-tarball || fail "Making src-tarball"
+    else
+	make src-tarball-release || fail "Making src-tarball"
+    fi
+
+    rm -rf $tdir/$v
+
+    mkdir -p $tdir/$v || fail "Couldn't make target directory"
+
+    cp dist/xen-$v.tar.gz $tdir/$v || fail "Couldn't copy tarball"
+}
+
+function buildtest-tarball() {
+    $arg_parse
+
+    default bdir "/tmp" ; $default_post
+
+    $requireargs tdir v
+    
+    cd $bdir || fail "cd $bdir"
+
+    rm -rf build-$v
+    mkdir build-$v || fail "mkdir"
+
+    cd build-$v
+
+    tar xfz $tdir/$v/xen-$v.tar.gz || fail "Untar"
+
+    cd xen-$v || fail "cd"
+
+    xen-make-prefix-config
+    info "Testing build (tail -f $bdir/build-$v/log.$v)..."
+    (./configure && make -j4 && touch $tdir/$v/build-tested && echo OK) 2>&1 > ../log.$v
+
+    [[ -e $tdir/$v/build-tested ]] || fail "Build failed; log at $bdir/build-$v/log.$v"
+}
+
+function sign-tarball() {
+    $arg_parse
+
+    $requireargs v
+
+    if [[ -z "$rtgz" ]] ; then
+	set-tdir
+	rtgz=$tdir/$v/xen-$v.tar.gz
+    fi
+
+    default key "23E3222C145F4475FA8060A783FE14C957E82BD9" ; $default_post
+
+    if ! gpg --list-secret-keys | grep $key ; then
+	info "Signature required; please run the following command with the public key available"
+	info " gpg --detach-sign -u 'xen tree' $rtgz"
+	exit 0
+    fi
+
+    gpg --detach-sign -u $key $rtgz || fail "Signing $rtgz"
+}
+
+function tarball-checksig() {
+    gpg --verify $rtgz.sig || fail "Signature failed"
+}
+
+function make-tarball() {
+    local rdir
+    local rtgz
+    
+    $arg_parse
+
+    $requireargs v
+
+    set-tdir
+
+    $requireargs tdir
+
+    parse-version
+    
+    info "Using tag $t"
+
+    rdir=$tdir/$v
+
+    rtgz=$rdir/xen-$v.tar.gz
+
+    if [[ ! -e $rtgz ]] ; then
+	info "$rtgz not present, generating"
+	make-tarball-only
+    fi
+
+    info "Tarball created"
+
+    if [[ ! -e $rdir/build-tested ]] ; then
+	buildtest-tarball
+    fi
+
+    info "Build tested"
+
+    if [[ ! -e $rtgz.sig ]] ; then
+	sign-tarball
+    else
+	tarball-checksig
+    fi
+
+    info "Tarball made, signed, and build-tested.  Now run release push-tag v=$v"
+}
+
+function tarball-cvs-checkin-and-post() {
+    $arg_parse
+
+    $requireargs v
+
+    # TODO: This tree probably wants to be put somewhere on
+    # mail.xenproject.org
+    
+    default cvsdir "/build/hg/push/xen.org/" ; $default_post
+
+    if [[ ! -e $cvsdir ]] ; then
+	fail "$cvsdir does not exist"
+    fi
+
+    if [[ -z "$rtgz" ]] ; then
+	set-tdir
+	rtgz=$tdir/$v/xen-$v.tar.gz
+    fi
+
+    cd $cvsdir || fail "cd"
+
+    mkdir -p oss-xen/release/$v || fail "Creating directory in CVS"
+
+    cvs add -kb oss-xen/release/$v/ || fail "cvs add release directory"
+
+    cd oss-xen/release/$v || fail "cd"
+
+    cp $tdir/$v/xen-$v.tar.gz . || fail "Copying tarball"
+    cp $tdir/$v/xen-$v.tar.gz.sig . || fail "Copying sig"
+    
+    cvs add -kb xen-$v.tar.gz || fail "cvs add tarball"
+    cvs add -kb xen-$v.tar.gz.sig || fail "cvs add sig"
+    
+    cd ../../..
+
+    cvs ci -m $v || fail "cvs checkin"
+
+    ssh mail.xenproject.org "cd /data/downloads.xenproject.org/xen.org && cvs -q up -d" || fail "Deploying tarball"
+
+    info "Tarball Uploaded.  Xen version $v released."
+}
+
+function help() {
+    cat <<EOF
+General workflow:
+
+* Do a number of pre-release sanity checks
+  release check v=4.12.0-rc5
+
+* Tag and sign a Xen commit
+  release tag v=4.12.0-rc5
+   or
+  release tag v=4.12.0-rc5 c=07c181c
+
+* Create, test, and sign a release tarball
+  release make-tarball v=4.12.0-rc5
+
+* Push tags
+  release push-tag v=4.12.0-rc5
+
+* Publish tarball
+  release tarball-cvs-checkin-and-post v=4.12.0-rc5
+EOF
+}
+
+###
+# The actual command-line
+###
+cmdline "$@"
-- 
2.20.1


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

^ permalink raw reply related	[flat|nested] 8+ messages in thread