Working prototype

Can run CI jobs on the cluster using a fixed amount of resources (4 tasks). The commands to be used in .gitlab-ci.yml are limited to mpiexec or single core.

run.sh

@@ -2,33 +2,72 @@
 
 set -e
 
+module load git
+
 #
-# ./this_script [my_args...] tmp_script stage_name
-#                            \________/
-#                        This has to be run!
+# ./this_script [my_args...] script stage_name
+#                            \____/
+#                       This has to be run!
 #
 
-BLUE='\033[0;34m'
-NC='\033[0m' # No Color
-
-script="${@: -2:1}"
-runStage="${@: -1}"
-echo "run stage: $runStage"
+script=${@: -2:1}
+runStage=${@: -1:1}
+# this is needed since the runner's working directory is /tmp which is not shared between nodes
+workDir="/home/pape58/runner-wd"
 
 module load git
 
+1>&2 echo "Run stage: $runStage"
+
 if [[ "$runStage" == "build_script" ]]; then
+    # This will be the name of the batch script that is constructed from the script passed to this
+    # program. Later we move this to the old script name.
+    newScript="$script.new"
+
+    {
+        # write shebang and SBATCH options to new script
+        echo "#!/usr/bin/env bash";
+        echo "#SBATCH -o $workDir/slurm-%j.out";
+        echo "#SBATCH -e $workDir/slurm-%j.err";
+        echo "#SBATCH -n 4";
+        echo "";
+
+        # write settings to new script
+        # TODO: Better read out these settings than assume they'll always stay like this
+        echo "set -eo pipefail";
+        echo "set +o noclobber";
+        echo "";
+
+        # add the rest of the old script
+        tail -n +5 "$script";
+    } >> "$newScript"
+
+    # move the script
+    mv "$newScript" "$script"
+
+    # pass the script to sbatch, catching its job ID
+    chmod +x "$script"
+    jobID=$(sbatch "$script" | awk '{ print $4 }')
+
+    # wait for job to finish
+    # don't use -o pipefail or this might crash
+    until [[ $(squeue -j "$jobID" | wc -l) -le "1" ]]; do sleep 5; done
+
+    outFile="$workDir/slurm-$jobID.out"
+    errFile="$workDir/slurm-$jobID.err"
+
+    # print output and error
+    cat "$outFile"
+    1>&2 cat "$errFile"
 
-    scriptPath="$(dirname $script)"
-    scriptPathInTmp==$(echo "$scriptPath" | cut -c 5-)  # cuts off /tmp in the front
-    scriptName="$(basename $script)"
-    newDir="/home/pape58/runner-wd$scriptPathInTmp"
+    # cleanup
+    rm "$outFile" "$errFile"
 
-    mkdir -p "$newDir"
-    cp "$script" "$newDir"
+    # get the jobs exit code
+    exitCode=$(scontrol show job="$jobID" | grep "ExitCode" | awk '{ print $5 }' |
+               cut -d "=" -f 2 | cut -d ":" -f 1)
+    exit "$exitCode"
 
-    echo -e "${BLUE}##### SRUN #####${NC}"
-    srun "$newDir/$scriptName"
 else
     "$script"
 fi

run.sh.srun

+#!/usr/bin/env bash
+
+set -e
+
+module load git
+
+#
+# ./this_script [my_args...] script stage_name
+#                            \____/
+#                       This has to be run!
+#
+
+script=${( "${@: -2:1}" )[*]}
+runStage=${( "${@: -1:1}" )[*]}
+echo "run stage: $runStage"
+workDir="/home/pape58/runner-wd"
+
+module load git
+
+if [[ "$runStage" == "build_script" ]]; then
+
+    scriptPath=$(dirname "$script")
+    scriptPathInTmp==$(echo "$scriptPath" | cut -c 6-)  # cut off /tmp/ in the front
+    scriptName=$(basename "$script")
+    newDir="$workDir/$scriptPathInTmp"
+
+    mkdir -p "$newDir"
+    cp "$script" "$newDir"
+
+    srun -n 4 "$newDir/$scriptName"
+else
+    "$script"
+fi
+
+exit 0

setup.sh

 #!/usr/bin/env bash
 
-set -e
-
 RUNNER_CONFIG_DIR="/home/$USER/.gitlab-runner"
 LOCAL_BIN_DIR="/home/$USER/.local/bin"
 REPO_DIR="$(pwd)"
 
-if [ ! $(which gitlab-runner) ]; then
-    echo "gitlab-runner not in path" 1>&2
-    exit 1
-fi
+which "gitlab-runner" > /dev/null
 
 mkdir -p "$RUNNER_CONFIG_DIR"
 cd "$RUNNER_CONFIG_DIR"
-ln -frs "$REPO_DIR/runner-config.toml" "config.toml"
+ln -s "$REPO_DIR/runner-config.toml" "config.toml"
 
 mkdir -p "$LOCAL_BIN_DIR"
 cd "$LOCAL_BIN_DIR"
-ln -frs "$REPO_DIR/run.sh" "run.sh"
+ln -s "$REPO_DIR/run.sh" "run.sh"
 
 exit 0