Commit 1eea22ca authored by Pape, David (FWCC) - 139658's avatar Pape, David (FWCC) - 139658
Browse files

Working prototype

Can run CI jobs on the cluster using a fixed amount of resources (4 tasks). The commands to be used in .gitlab-ci.yml are limited to mpiexec or single core.
parent 11f194ba
......@@ -2,33 +2,72 @@
set -e
module load git
#
# ./this_script [my_args...] tmp_script stage_name
# \________/
# This has to be run!
# ./this_script [my_args...] script stage_name
# \____/
# This has to be run!
#
BLUE='\033[0;34m'
NC='\033[0m' # No Color
script="${@: -2:1}"
runStage="${@: -1}"
echo "run stage: $runStage"
script=${@: -2:1}
runStage=${@: -1:1}
# this is needed since the runner's working directory is /tmp which is not shared between nodes
workDir="/home/pape58/runner-wd"
module load git
1>&2 echo "Run stage: $runStage"
if [[ "$runStage" == "build_script" ]]; then
# This will be the name of the batch script that is constructed from the script passed to this
# program. Later we move this to the old script name.
newScript="$script.new"
{
# write shebang and SBATCH options to new script
echo "#!/usr/bin/env bash";
echo "#SBATCH -o $workDir/slurm-%j.out";
echo "#SBATCH -e $workDir/slurm-%j.err";
echo "#SBATCH -n 4";
echo "";
# write settings to new script
# TODO: Better read out these settings than assume they'll always stay like this
echo "set -eo pipefail";
echo "set +o noclobber";
echo "";
# add the rest of the old script
tail -n +5 "$script";
} >> "$newScript"
# move the script
mv "$newScript" "$script"
# pass the script to sbatch, catching its job ID
chmod +x "$script"
jobID=$(sbatch "$script" | awk '{ print $4 }')
# wait for job to finish
# don't use -o pipefail or this might crash
until [[ $(squeue -j "$jobID" | wc -l) -le "1" ]]; do sleep 5; done
outFile="$workDir/slurm-$jobID.out"
errFile="$workDir/slurm-$jobID.err"
# print output and error
cat "$outFile"
1>&2 cat "$errFile"
scriptPath="$(dirname $script)"
scriptPathInTmp==$(echo "$scriptPath" | cut -c 5-) # cuts off /tmp in the front
scriptName="$(basename $script)"
newDir="/home/pape58/runner-wd$scriptPathInTmp"
# cleanup
rm "$outFile" "$errFile"
mkdir -p "$newDir"
cp "$script" "$newDir"
# get the jobs exit code
exitCode=$(scontrol show job="$jobID" | grep "ExitCode" | awk '{ print $5 }' |
cut -d "=" -f 2 | cut -d ":" -f 1)
exit "$exitCode"
echo -e "${BLUE}##### SRUN #####${NC}"
srun "$newDir/$scriptName"
else
"$script"
fi
......
#!/usr/bin/env bash
set -e
module load git
#
# ./this_script [my_args...] script stage_name
# \____/
# This has to be run!
#
script=${( "${@: -2:1}" )[*]}
runStage=${( "${@: -1:1}" )[*]}
echo "run stage: $runStage"
workDir="/home/pape58/runner-wd"
module load git
if [[ "$runStage" == "build_script" ]]; then
scriptPath=$(dirname "$script")
scriptPathInTmp==$(echo "$scriptPath" | cut -c 6-) # cut off /tmp/ in the front
scriptName=$(basename "$script")
newDir="$workDir/$scriptPathInTmp"
mkdir -p "$newDir"
cp "$script" "$newDir"
srun -n 4 "$newDir/$scriptName"
else
"$script"
fi
exit 0
#!/usr/bin/env bash
set -e
RUNNER_CONFIG_DIR="/home/$USER/.gitlab-runner"
LOCAL_BIN_DIR="/home/$USER/.local/bin"
REPO_DIR="$(pwd)"
if [ ! $(which gitlab-runner) ]; then
echo "gitlab-runner not in path" 1>&2
exit 1
fi
which "gitlab-runner" > /dev/null
mkdir -p "$RUNNER_CONFIG_DIR"
cd "$RUNNER_CONFIG_DIR"
ln -frs "$REPO_DIR/runner-config.toml" "config.toml"
ln -s "$REPO_DIR/runner-config.toml" "config.toml"
mkdir -p "$LOCAL_BIN_DIR"
cd "$LOCAL_BIN_DIR"
ln -frs "$REPO_DIR/run.sh" "run.sh"
ln -s "$REPO_DIR/run.sh" "run.sh"
exit 0
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment