Commit 6a5147d5 authored by Matthias Bernt's avatar Matthias Bernt
Browse files

initial commit

parents
image: singularityware/singularity:gitlab-3.1-slim
build:
script:
- /bin/bash .gitlabci/build.sh Singularity
# step 1. build the container!
# You can add any other sregistry push commands here, and specify a client
# (and make sure your define the encrypted environment credentials in gitlab
# to push to your storage locations of choice
- mkdir -p build && cp *.sif build
- mkdir -p build && cp Singularity* build
# Step 2. Take a look at "artifacts" below and add the paths you want added
# You can also add the entire build folder. You can also upload to storage
# clients defined by sregistry, here are some examples
# https://singularityhub.github.io/sregistry-cli/clients
# Environment variables must be defined in CI encrypted secrets/settings
# https://code.stanford.edu/help/ci/variables/README#variables).
#- /bin/bash build.sh --uri collection/container --cli google-storage Singularity
#- /bin/bash build.sh --uri collection/container --cli google-drive Singularity
#- /bin/bash build.sh --uri collection/container --cli globus Singularity
#- /bin/bash build.sh --uri collection/container --cli registry Singularity
# This is where you can save job artifacts
# https://docs.gitlab.com/ee/user/project/pipelines/job_artifacts.html
# You can specify the path to containers or the build folder to save.
# Don't forget to save your recipes too!
artifacts:
paths:
- build/Singularity.sif
- build/Singularity
#!/bin/bash
# build.sh will build a Singularity container. It's not overly complicated.
#
# USAGE: build.sh --uri collection-name/container-name --cli registry Singularity
# build.sh --uri collection-name/container-name --cli registry
# build.sh Singularity
# Copyright (C) 2017-2020 Vanessa Sochat.
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
set -o errexit
set -o nounset
function usage() {
echo "USAGE: build [recipe] [options]"
echo ""
echo "OPTIONS:
Image Format
--uri -u if uploading, a uri to give to sregistry
--cli -c the sregistry client to use (if uploading)
--help -h show this help and exit
"
}
# --- Option processing --------------------------------------------------------
uri=""
cli=""
tag=""
while true; do
case ${1:-} in
-h|--help|help)
usage
exit 0
;;
-u|--uri)
shift
uri="${1:-}"
shift
;;
-t|--tag)
shift
tag="${1:-}"
shift
;;
-c|--cli)
shift
cli="${1:-}"
shift
;;
\?) printf "illegal option: -%s\n" "${1:-}" >&2
usage
exit 1
;;
-*)
printf "illegal option: -%s\n" "${1:-}" >&2
usage
exit 1
;;
*)
break;
;;
esac
done
################################################################################
### Recipe File ################################################################
################################################################################
if [ $# == 0 ] ; then
recipe="Singularity"
else
recipe=$1
fi
echo ""
echo "Image Recipe: ${recipe}"
################################################################################
### Storage Client #############################################################
################################################################################
is_valid_client () {
local e match="$1"
shift
for e; do [[ "$e" == "$match" ]] && return 0; done
return 1
}
# Test if client is valid
clients=("google-storage" "registry" "globus" "dropbox" "google-drive")
if [ "${cli}" != "" ]; then
is_valid_client "${cli}" "${clients[@]}"
if [ $? -ne 0 ]; then
echo "${cli} is not a valid choice! Choose from ${clients[@]}";
exit 1
fi
echo "Storage Client: ${cli}"
else
echo "Storage Client: none"
fi
################################################################################
### Build! #####################################################################
################################################################################
# Continue if the image recipe is found
if [ -f "$recipe" ]; then
imagefile="${recipe}.sif"
echo "Creating $imagefile using $recipe..."
singularity build $imagefile $recipe
# If the image is successfully built, test it and upload (examples)
if [ -f "${imagefile}" ]; then
# Example testing using run (you could also use test command)
echo "Testing the image... Marco!"
singularity exec $imagefile echo "Polo!"
# Example sregistry commands to push to endpoints
if [ "${cli}" != "" ]; then
# If the uri isn't provided, he gets a robot name
if [ "${uri}" == "" ]; then
uri=$(python -c "from sregistry.logger.namer import RobotNamer; bot=RobotNamer(); print(bot.generate())")
fi
# If a tag is provided, add to uri
if [ "${tag}" != "" ]; then
uri="${uri}:${tag}"
fi
echo "Pushing ${uri} to ${cli}://"
echo "SREGISTRY_CLIENT=${cli} sregistry push --name ${uri} ${imagefile}"
SREGISTRY_CLIENT="${cli}" sregistry push --name "${uri}" "${imagefile}"
else
echo "Skipping upload. Image $imagefile is finished!"
fi
fi
else
echo "Singularity recipe ${recipe} not found!"
exit 1
fi
#!/bin/bash
apt-get update && apt-get install -y wget git \
build-essential \
squashfs-tools \
libtool \
autotools-dev \
libarchive-dev \
automake \
autoconf \
uuid-dev \
libssl-dev
sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers
# Check Python
echo "Python Version:"
python --version
pip install sregistry[all]
sregistry version
echo "sregistry Version:"
# Install Singularity
cd /tmp && \
git clone -b vault/release-2.5 https://www.github.com/sylabs/singularity.git
cd singularity && \
./autogen.sh && \
./configure --prefix=/usr/local && \
make && make install
BSD 3-Clause License
Copyright (c) 2018, Vanessa Sochat
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Singularity Builder GitLab
[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3834833.svg)](https://doi.org/10.5281/zenodo.3834833)
![.gitlabci/sregistry-gitlab.png](.gitlabci/sregistry-gitlab.png)
This is a simple example of how you can achieve:
- version control of your recipes
- versioning to include image hash *and* commit id
- build of associated container and
- push to a storage endpoint (or GitLab artifact)
for a reproducible build workflow.
**Why should this be managed via GitLab?**
GitLab, by way of easy integration with continuous integration, is an easy way
to have a workflow set up where multiple people can collaborate on a container recipe,
the recipe can be tested (with whatever testing you need), discussed in pull requests,
and then finally pushed to be a GitLab artifact, to your storage of choice
or to Singularity Registry.
**Why should I use this instead of a service?**
You could use a remote builder, but if you do the build in a continuous integration
service you get complete control over it. This means everything from the version of
Singularity to use, to the tests that you run for your container. You have a lot more
freedom in the rate of building, and organization of your repository, because it's you
that writes the configuration. Although the default would work for most, you can
edit the build, setup, and circle configuration file in the
[.gitlabci](.gitlabci) folder to fit your needs.
## Quick Start
Add your Singularity recipes to this repository, and edit the build commands in
the [build.sh](.gitlabci/build.sh) file. This is where you can specify endpoints
(Singularity Registry, Dropbox, Google Storage, AWS) along with container names
(the uri) and tag. You can build as many recipes as you like, just add another line!
```yaml
# recipe relative to repository base
- /bin/bash .gitlabci/build.sh Singularity
- /bin/bash .gitlabci/build.sh --uri collection/container --tag tacos --cli google-storage Singularity
- /bin/bash .gitlabci/build.sh --uri collection/container --cli google-drive Singularity
- /bin/bash .gitlabci/build.sh --uri collection/container --cli globus Singularity
- /bin/bash .gitlabci/build.sh --uri collection/container --cli registry Singularity
```
For each client that you use, required environment variables (e.g., credentials to push,
or interact with the API) must be defined in the (encrypted) Travis environment. To
know what variables to define, along with usage for the various clients, see
the [client specific pages](https://singularityhub.github.io/sregistry-cli/clients)
## Detailed Started
### 0. Fork this repository
You can clone and tweak, but it's easiest likely to get started with our example
files and edit them as you need.
### 1. Get to Know GitLab
We will be working with [GitLab](https://www.gitlab.com) repositories, which have
a built-in Continuous Integration service. Note that if you aren't using GitLab.com
or need to configure or enable runners, you can do this by going to your project page,
and clicking on the Settings --> CI/CD and selecting your "Runners." Typically a runner
is some external service, or the GitLab shared runners. I used GitLab.com so the
shared runners were already active for me.
### 2. Add your Recipe(s)
For the example here, we have a single recipe named "Singularity" that is provided
as an input argument to the [build script](.gitlabci/build.sh). You could add another
recipe, and then of course call the build to happen more than once.
The build script will name the image based on the recipe, and you of course
can change this. Just write the path to it (relative to the repository base) in
your [.gitlab-ci.yml](.gitlab-ci.yml).
### 3. Configure Singularity
We previously used [setup](.gitlabci/setup.sh) to setup the build, but now use a base image instead.
The previous instructions are provided for posterity.
- Install Singularity, we use the release 2.6 branch as it was the last to not be written in GoLang. You could of course change the lines in [setup.sh](.gitlabci/setup.sh) to use a specific tagged release, an older version, or development version.
- Install the sregistry client, if needed. The [sregistry client](https://singularityhub.github.io/sregistry-cli) allows you to issue a command like "sregistry push ..." to upload a finished image to one of your cloud / storage endpoints. By default, the push won't happen, and you will just build an image using the CI.
### 4. Configure the Build
The basic steps for the [build](.gitlabci/build.sh) are the following:
- Running build.sh with no inputs will default to a recipe called "Singularity" in the base of the repository. You can provide an argument to point to a different recipe path, always relative to the base of your repository.
- If you want to define a particular unique resource identifier for a finished container (to be uploaded to your storage endpoint) you can do that with `--uri collection/container`. If you don't define one, a robot name will be generated.
- You can add `--uri` to specify a custom name, and this can include the tag, OR you can specify `--tag` to go along with a name without one. It depends on which is easier for you.
- If you add `--cli` then this is telling the build script that you have defined the [needed environment variables](https://circleci.com/docs/2.0/env-vars/) for your [client of choice](https://singularityhub.github.io/sregistry-cli/clients) and you want successful builds to be pushed to your storage endpoint. See [here](https://singularityhub.github.io/sregistry-cli/clients) for a list of current client endpoints, or roll your own!
See the [.gitlab-ci.yml](.gitlab-ci.yml) for examples of this build.sh command (commented out). If there is some cloud service that you'd like that is not provided, please [open an issue](https://www.github.com/singularityhub/sregistry-cli/issues).
### 5. Pull your Container
If you want to pull from the artifacts folder in GitLab, check out the
[Singularity Registry Client](https://www.github.com/singularityhub/sregistry-cli), and
specifically the [GitLab client](https://singularityhub.github.io/client-gitlab). The client
will help you to pull a container stored as an artifact given that you know the job identifier,
and if you change the job name (default is build) that will need to be provided too!
See the [GitLab client](https://singularityhub.github.io/client-gitlab) pages for more information
on how to do this. If you don't want to use GitLab artifacts for deployment, this is okay too!
Just remember that if you are using Singularity Registry Client (or GitLab for that matter)
to deploy a container elsewhere, you likely need to configure environment variables to help authenticate you. Instructions can be found [here](https://code.stanford.edu/help/ci/variables/README#variables).
Bootstrap: docker
From: ubuntu:16.04
%runscript
exec echo "Polo $@!"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment