Added numbered symlinks to build steps for ease of use

Jenkinsfile

@@ -59,56 +59,21 @@ def revFor(String repo) {
   return (rev && rev.trim()) ? normalizeRev(rev.trim()) : defaultRev(repo)
 }
 
-// Returns the refspec to fetch a repo with. Branch heads are always fetched.
-// Pull requests live outside refs/heads. Building one means fetching its ref
-// too, or its commit is not in the clone.
-def refspecFor(String rev) {
-  def heads = '+refs/heads/*:refs/remotes/origin/*'
-  if (!(rev ==~ /^(?:refs\/)?pull\/\d+\/(merge|head)$/)) { return heads }
-  // pull/1234/merge -> [pull, 1234, merge]
-  def parts = rev.replaceAll(/^refs\//, '').split('/')
-  return "${heads} +refs/pull/${parts[1]}/${parts[2]}:refs/remotes/origin/pr/${parts[1]}"
-}
-
-// Resolves rev to the commit to build.
-//
-// url is the repo to ask.
-// rev is what revFor returned: a branch, tag, pull/<n>/merge ref,
-// refs/... ref, or commit id.
-//
-// Returns '' when rev names no ref, as a commit id does.
-//
-// A bare name is an ls-remote pattern matched against the tail of every ref,
-// not a ref name: 'master' also matched core's CFE-159/master, which sorts
-// first and so won. Ask for the full ref, refs/heads before refs/tags before
-// refs/, and take the first that exists.
-def resolveRev(String url, String rev) {
-  def candidates = rev.startsWith('refs/') ? [rev]
-                 : ["refs/heads/${rev}", "refs/tags/${rev}", "refs/${rev}"]
-  for (ref in candidates) {
-    // An annotated tag's own ref names the tag object, its ^{} the commit.
-    def peeled = "${ref}^{}"
-    def out = sh(returnStdout: true,
-                 script: "git ls-remote '${url}' '${ref}' '${peeled}'").trim()
-    def sha = ''
-    for (line in out.readLines()) {
-      def parts = line.split()
-      if (parts[1] == peeled) { return parts[0] }
-      if (parts[1] == ref) { sha = parts[0] }
-    }
-    if (sha) { return sha }
-  }
-  return ''
+// The refspec the other jobs fetch with. Branch heads are always fetched. Pull
+// requests live outside refs/heads, so building one means fetching its ref too,
+// or its commit is not in the clone.
+def refspec() {
+  return '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pull/*'
 }
 
 // Runs one build in the workspace of the node the caller allocated.
 //
-// Cleans up after the previous build. Checks out each repo at its commit from
-// shas. Builds, then archives the packages.
+// Cleans up after the previous build. Checks out each repo at its revision from
+// revs. Builds, then archives the packages.
 //
 // opts holds the build-in-container.py flags that vary per build. The flags
 // every build shares are added below.
-def containerBuild(String opts, List repos, Map shas, Map revs) {
+def containerBuild(String opts, List repos, Map revs) {
   // The container hands the directories it writes back to us as it exits, so
   // this only covers a build that never got to exit (e.g. killed).
   sh 'sudo chown -R "$(id -u):$(id -g)" "$WORKSPACE" 2>/dev/null || true'
@@ -117,10 +82,10 @@ def containerBuild(String opts, List repos, Map shas, Map revs) {
   repos.each { repo ->
     dir("src/${repo}") {
       checkout([$class: 'GitSCM',
-                branches: [[name: shas[repo]]],
+                branches: [[name: revs[repo]]],
                 userRemoteConfigs: [[url: "git@github.com:cfengine/${repo}.git",
                                      credentialsId: 'jenkins-github',
-                                     refspec: refspecFor(revs[repo])]],
+                                     refspec: refspec()]],
                 // Full history on purpose: the build reads SOURCE_DATE_EPOCH and
                 // every dependency's revision out of git log, so a shallow clone
                 // would change the timestamps it pins.
@@ -151,15 +116,10 @@ def containerBuild(String opts, List repos, Map shas, Map revs) {
 }
 
 // All filled in by Resolve refs and read by the build stages, which run on other
-// nodes. labels holds the build labels asked for. revs and shas hold what was
-// asked for, and what it resolved to:
+// nodes. labels holds the build labels asked for. revs holds the revision each
+// repo builds at:
 //
 //   revs['core'] = pull/1234/merge
-//   shas['core'] = 5dca070a98f9be...
-//
-// Nodes check out the sha, so a push mid-run cannot change what is built. The
-// rev is kept too: a sha does not say whether a pull ref has to be fetched.
-def shas = [:]
 def labels = []
 def revs = [:]
 
@@ -218,27 +178,12 @@ pipeline {
           def repos = reposFor(params.PROJECT)
           echo "Building ${labels.size()} labels:\n  ${labels.join('\n  ')}"
 
-          repos.each { repo -> revs[repo] = revFor(repo) }
-
-          // Each platform checks out on its own node, so a push while the job
-          // runs would otherwise leave them building different sources. Resolve
-          // to commits once, here, and hand those to every build.
-          sshagent(['jenkins-github']) {
-            repos.each { repo ->
-              def rev = revs[repo]
-              def sha = resolveRev("git@github.com:cfengine/${repo}.git", rev)
-              if (!sha) {
-                // A commit id matches no ref, which is the one case where an
-                // empty answer is fine.
-                if (!(rev ==~ /[0-9a-f]{7,40}/)) { error "${repo}: cannot resolve '${rev}'" }
-                sha = rev
-              }
-              shas[repo] = sha
-              echo "${repo}: ${rev} -> ${sha}"
-            }
+          repos.each { repo ->
+            revs[repo] = revFor(repo)
+            echo "${repo}: ${revs[repo]}"
           }
 
-          currentBuild.description = "${params.PROJECT} @ ${shas['core'].take(7)}: ${labels.size()} labels"
+          currentBuild.description = "${params.PROJECT} @ ${revs['core']}: ${labels.size()} labels"
         }
       }
     }
@@ -251,7 +196,7 @@ pipeline {
           // --tarballs builds core and masterfiles alone, in an image of its own,
           // and forces project and platform itself. Only the build type is ours
           // to pass: it decides the version string.
-          containerBuild('--tarballs', ['buildscripts', 'core', 'masterfiles'], shas, revs)
+          containerBuild('--tarballs', ['buildscripts', 'core', 'masterfiles'], revs)
         }
       }
     }
@@ -267,7 +212,7 @@ pipeline {
                 // The label decides the platform, the role and the container
                 // architecture, so --arch would only contradict it.
                 containerBuild("--label '${label}' --project '${params.PROJECT}'",
-                               repos, shas, revs)
+                               repos, revs)
               }
             }]
           }

README-build-in-container.md

@@ -0,0 +1,8 @@
+Build with ./build-in-container.py
+
+Using ./build-in-container.py --shell you can run the build interactively and debug issues.
+Start in the container by running /srv/source/buildscripts/build-in-container-inner.sh
+
+This will copy repository sources that are needed from the read-only /srv location to read-write work area in /home/builder/build.
+
+Continue debugging by running steps in /home/builder/buildscripts/build-scripts/0*.sh

build-in-container-inner.sh

@@ -4,15 +4,20 @@ set -e
 # Configuration via environment variables:
 #   PROJECT, BUILD_TYPE, EXPLICIT_ROLE, BUILD_NUMBER, EXPLICIT_VERSION
 
+# let setup-cfengine-build-host.sh know we are in a container
+sudo touch /etc/cfengine-in-container.flag
+
 BASEDIR=/home/builder/build
 export BASEDIR
 export AUTOBUILD_PATH="$BASEDIR/buildscripts"
+OUTPUT=/output
+export OUTPUT
 
 mkdir -p "$BASEDIR"
 
 # Bind-mounted directories may be owned by the host user's UID.
 # Fix ownership so builder can write to them.
-sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" /output
+sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" "$OUTPUT"
 
 # And hand ownership back to the host user on the way out.
 if [ -n "$HOST_UID" ] && [ -n "$HOST_GID" ]; then
@@ -85,105 +90,22 @@ fi
 export SOURCE_DATE_EPOCH
 echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
 
-install_mission_portal_deps() (
-    set -e
-
-    if [ -f "$BASEDIR/mission-portal/public/scripts/package.json" ]; then
-        echo "Installing npm dependencies..."
-        npm ci --prefix "$BASEDIR/mission-portal/public/scripts/"
-        echo "Building react components..."
-        npm run build --prefix "$BASEDIR/mission-portal/public/scripts/"
-        rm -rf "$BASEDIR/mission-portal/public/scripts/node_modules"
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/composer.json" ]; then
-        echo "Installing Mission Portal PHP dependencies..."
-        (cd "$BASEDIR/mission-portal" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    if [ -f "$BASEDIR/nova/api/http/composer.json" ]; then
-        echo "Installing Nova API PHP dependencies..."
-        (cd "$BASEDIR/nova/api/http" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/public/themes/default/bootstrap/cfengine_theme.less" ]; then
-        echo "Compiling Mission Portal styles..."
-        mkdir -p "$BASEDIR/mission-portal/public/themes/default/bootstrap/compiled/css"
-        (cd "$BASEDIR/mission-portal/public/themes/default/bootstrap" &&
-            lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css)
-    fi
-
-    if [ -f "$BASEDIR/mission-portal/ldap/composer.json" ]; then
-        echo "Installing LDAP API PHP dependencies..."
-        (cd "$BASEDIR/mission-portal/ldap" && composer install --no-dev --ignore-platform-reqs --prefer-dist)
-    fi
-
-    # Composer falls back to git clone when GitHub's anonymous zipball
-    # rate limit is hit, leaving non-reproducible .git directories in the
-    # vendor tree. Strip them.
-    find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} +
-)
-
 # Lets whoever consumes the output check that it arrived intact. Sorted in the C
 # locale so that the list itself comes out the same every time.
 write_sha256sums() (
-    cd /output
+    cd "$OUTPUT"
     # shellcheck disable=SC2094
     # > Make sure not to read and write the same file in the same pipeline.
     # find leaves it out by name, so the list never covers itself.
     find . -maxdepth 1 -type f ! -name sha256sums.txt -printf '%P\n' \
         | LC_ALL=C sort | xargs -r sha256sum > sha256sums.txt
 )
 
-# Build the source tarballs. They are the same whichever platform builds them,
-# so only this image builds them, and nothing else here does. /output is
-# <output-dir>/tarballs on the host, as the packages' /output is per label.
-#
-# Each tarball's timestamps follow its own repository: Makefile.am in core and in
-# masterfiles clamps every mtime in the tarball to SOURCE_DATE_EPOCH, so taking
-# it from the last commit keeps a tarball identical until its own sources change.
-build_tarballs() (
-    set -e
-
-    (
-        cd "$BASEDIR/core"
-        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
-        export SOURCE_DATE_EPOCH
-        echo "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
-
-        rm -f cfengine-3.*.tar.gz
-        # Configure so the dist target exists, undone again below.
-        ./configure -C
-        make dist
-        mv cfengine-3.*.tar.gz /output/
-        make distclean
-    )
-
-    (
-        cd "$BASEDIR/masterfiles"
-        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
-        export SOURCE_DATE_EPOCH
-        echo "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
-
-        rm -f cfengine-masterfiles*.tar.gz
-        ./configure
-        make dist        # source tarball:  cfengine-masterfiles-<version>.tar.gz
-        make tar-package # package tarball: cfengine-masterfiles-<version>.pkg.tar.gz
-        mv cfengine-masterfiles*.tar.gz /output/
-        make distclean
-    )
-
-    write_sha256sums
-)
-
-# === Step runner with failure reporting ===
-# Disable set -e so we can capture exit codes and report which step failed.
-set +e
 run_step() {
     local name="$1"
     shift
     echo "=== Running $name ==="
-    "$@"
+    "$BASEDIR/buildscripts/build-scripts/$name" "$@"
     local rc=$?
     if [ $rc -ne 0 ]; then
         echo ""
@@ -193,36 +115,26 @@ run_step() {
 }
 
 # === Build steps ===
-run_step "01-autogen" "$BASEDIR/buildscripts/build-scripts/autogen"
 
 if [ "$TARBALLS" = yes ]; then
-    run_step "02-tarballs" build_tarballs
+    run_step autogen
+    run_step build-tarballs
     echo ""
     echo "=== Build complete ==="
-    ls -lh /output/
+    ls -lh "$OUTPUT"
     exit 0
 fi
 
-run_step "02-install-dependencies" "$BASEDIR/buildscripts/build-scripts/install-dependencies"
-# Mission Portal is an Enterprise/nova-only component; its sources are only
-# synced when PROJECT=nova. Skip this step for community hubs.
-if [ "$PROJECT" = "nova" ] && [ "$EXPLICIT_ROLE" = "hub" ]; then
-    run_step "03-mission-portal-deps" install_mission_portal_deps
-fi
-run_step "04-configure" "$BASEDIR/buildscripts/build-scripts/configure"
-run_step "05-compile" "$BASEDIR/buildscripts/build-scripts/compile"
-run_step "06-package" "$BASEDIR/buildscripts/build-scripts/package"
-
-# === Copy output packages ===
-# Packages are created under $BASEDIR/<project>/ by dpkg-buildpackage / rpmbuild.
-# Exclude deps-packaging to avoid copying dependency packages.
-find "$BASEDIR" -maxdepth 4 \
-    -path "$BASEDIR/buildscripts/deps-packaging" -prune -o \
-    \( -name '*.deb' -o -name '*.rpm' -o -name '*.msi' -o -name '*.pkg.tar.gz' \) -print \
-    -exec cp {} /output/ \;
+NO_TESTS=true
+export NO_TESTS
+
+for script in "$BASEDIR/buildscripts/build-scripts"/0*; do
+  name="$(basename "$script")"
+  run_step "$name"
+done
 
 write_sha256sums
 
 echo ""
 echo "=== Build complete ==="
-ls -lh /output/
+ls -lh "$OUTPUT"/

build-scripts/0000-system-dependencies

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -ex
+thisdir="$(dirname "$0")"
+sudo bash "$thisdir"/../ci/setup-cfengine-build-host.sh

build-scripts/0005-repositories

@@ -0,0 +1 @@
+repositories

build-scripts/0010-autogen

@@ -0,0 +1 @@
+autogen

build-scripts/0020-clean-buildmachine

@@ -0,0 +1 @@
+clean-buildmachine

build-scripts/0030-bootstrap-mission-portal

@@ -0,0 +1 @@
+bootstrap-mission-portal

build-scripts/0035-generate-pull-request-file

@@ -0,0 +1 @@
+generate-pull-request-file

build-scripts/0040-build-tarballs

@@ -0,0 +1 @@
+build-tarballs

build-scripts/0050-unpack-tarballs

@@ -0,0 +1 @@
+unpack-tarballs

build-scripts/0060-install-dependencies

@@ -0,0 +1 @@
+install-dependencies

build-scripts/0070-configure

@@ -0,0 +1 @@
+configure

build-scripts/0080-generate-source-tarballs

@@ -0,0 +1 @@
+generate-source-tarballs

build-scripts/0090-compile

@@ -0,0 +1 @@
+compile

build-scripts/0100-produce-debug-symbols

@@ -0,0 +1 @@
+produce-debug-symbols

build-scripts/0110-package

@@ -0,0 +1 @@
+package

build-scripts/0120-prepare-results

@@ -0,0 +1 @@
+prepare-results

build-scripts/0130-test

@@ -0,0 +1 @@
+test

build-scripts/0140-prepare-results

@@ -0,0 +1 @@
+prepare-results