Commit e56174ef authored by Pape, David (FWCC) - 139658's avatar Pape, David (FWCC) - 139658
Browse files

Refactoring.

parent d456d946
......@@ -5,10 +5,15 @@ stages:
image: fedora:latest
before_script:
- "dnf install -y \
cmake make g++ pkgconf lcov \
gtest-devel libconfig-devel spdlog-devel \
doxygen graphviz python"
- dnf install -y git cmake make g++ pkgconf lcov libconfig-devel spdlog-devel doxygen graphviz python
# TODO: Install gtest-devel and gmock-devel from Fedora once version 1.10 is available.
- git clone https://github.com/google/googletest.git
- mkdir googletest/build
- pushd googletest/build
- cmake ..
- make -j $(nproc)
- make install
- popd
build_program:
stage: build
......
......@@ -19,14 +19,21 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
find_package (Threads REQUIRED)
find_package (spdlog REQUIRED)
find_package (GTest REQUIRED)
find_package (PkgConfig REQUIRED)
pkg_check_modules(CONFIG++ REQUIRED IMPORTED_TARGET libconfig++)
find_package (GTest)
find_package (GMock)
find_package (Doxygen)
find_package (Sphinx)
include_directories(src)
add_subdirectory(src)
add_subdirectory(tests)
add_subdirectory(docs)
if("${GTEST_FOUND}" AND "${GMOCK_FOUND}")
add_subdirectory(tests)
endif()
if("${DOXYGEN_FOUND}" AND "${SPHINX_FOUND}")
add_subdirectory(docs)
endif()
......@@ -19,7 +19,7 @@ Required programs and libraries:
- A compiler that supports C++17
- GNU Make and CMake
- [Google Test](https://github.com/google/googletest)
- [Google Test](https://github.com/google/googletest) and Google Mock
- [spdlog](https://github.com/gabime/spdlog)
- [libconfig](https://github.com/hyperrealm/libconfig)
- lcov (for test coverage)
......
This diff is collapsed.
......@@ -10,12 +10,10 @@ set(DOXYGEN_EXTRACT_LOCAL_METHODS YES)
set(DOXYGEN_WARN_IF_UNDOCUMENTED YES)
set(DOXYGEN_WARN_NO_PARAMDOC YES)
if("${DOXYGEN_FOUND}")
doxygen_add_docs(xml
../src
COMMENT "Generating Doxygen XML output"
)
endif()
doxygen_add_docs(xml
../src
COMMENT "Generating Doxygen XML output"
)
# based on:
# https://devblogs.microsoft.com/cppblog/clear-functional-c-documentation-with-sphinx-breathe-doxygen-cmake/
......@@ -26,23 +24,20 @@ set(SPHINX_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
# TODO: Build out of source when Exhale allows this
set(EXHALE_CONTAINMENT_FOLDER source_code_doc)
if("${DOXYGEN_FOUND}" AND "${SPHINX_FOUND}")
add_custom_target(docs
COMMAND "${SPHINX_EXECUTABLE}"
-b html
add_custom_target(docs
COMMAND "${SPHINX_EXECUTABLE}"
-b html
# sphinx parameters
-Drelease="${PROJECT_VERSION}"
# sphinx parameters
-Drelease="${PROJECT_VERSION}"
# breathe parameters
-Dbreathe_projects.runner="${SPHINX_BUILD_DIR}/xml"
# breathe parameters
-Dbreathe_projects.runner="${SPHINX_BUILD_DIR}/xml"
# exhale parameters
-Dexhale_args.containmentFolder="${EXHALE_CONTAINMENT_FOLDER}" # must be in source dir :-/
"${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}"
DEPENDS xml
COMMENT "Generating Sphinx documentation"
)
endif()
# exhale parameters
-Dexhale_args.containmentFolder="${EXHALE_CONTAINMENT_FOLDER}" # must be in source dir :-/
"${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}"
DEPENDS xml
COMMENT "Generating Sphinx documentation"
)
#include "ci_cleanup_job.h"
#include "CICleanupJob.h"
#include <cstdlib>
#include <filesystem>
#include <string>
#include <system_error>
#include "ci_environment.h"
#include "spdlog/spdlog.h"
ci_cleanup_job::ci_cleanup_job() {
CICleanupJob::CICleanupJob(Environment *env)
: CIJob(env) {
SPDLOG_INFO("New cleanup job");
}
int ci_cleanup_job::run() {
int CICleanupJob::Run() {
std::error_code ec;
std::filesystem::remove_all(ci_environment::instance()->builds_dir(), ec);
std::filesystem::remove_all(CIJob::GetEnv()->GetBuildsDir(), ec);
if (ec) {
SPDLOG_ERROR("Error during deletion of build directory {}: {}",
ci_environment::instance()->builds_dir().string(), ec.message());
return EXIT_SYSTEM_FAILURE;
CIJob::GetEnv()->GetBuildsDir().string(), ec.message());
return CIJob::GetEnv()->GetExitSystemFailure();
}
return 0;
}
#ifndef CI_CLEANUP_JOB_H
#define CI_CLEANUP_JOB_H
#include "ci_job.h"
#include "CIJob.h"
/**
* @brief Class to represent a job of the cleanup stage.
*/
class ci_cleanup_job : public ci_job {
class CICleanupJob : public CIJob {
public:
/**
* @brief Simple constructor.
*/
ci_cleanup_job();
CICleanupJob(Environment *env);
/**
* @brief Method used to execute the job.
* @return The exit code of the job.
*/
int run() override;
int Run() override;
};
#endif // CI_CLEANUP_JOB_H
#include "ci_config_job.h"
#include "CIConfigJob.h"
#include <filesystem>
#include <iostream>
#include "ci_environment.h"
#include "spdlog/spdlog.h"
ci_config_job::ci_config_job() {
CIConfigJob::CIConfigJob(Environment *env)
: CIJob(env) {
SPDLOG_INFO("New config job");
}
int ci_config_job::run() {
int CIConfigJob::Run() {
std::cout << "{\n"
" \"builds_dir\": \"" + ci_environment::instance()->builds_dir().string() + "\",\n"
" \"cache_dir\": \"" + ci_environment::instance()->cache_dir().string() + "\"\n"
" \"builds_dir\": \"" + CIJob::GetEnv()->GetBuildsDir().string() + "\",\n"
" \"cache_dir\": \"" + CIJob::GetEnv()->GetCacheDir().string() + "\"\n"
"}" << std::endl;
return 0;
......
#ifndef CI_CONFIG_JOB_H
#define CI_CONFIG_JOB_H
#include "ci_job.h"
#include "CIJob.h"
/**
* @brief Class to represent a job of the config stage.
*/
class ci_config_job : public ci_job {
class CIConfigJob : public CIJob {
public:
/**
* @brief Simple constructor.
*/
ci_config_job();
CIConfigJob(Environment *env);
/**
* @brief Simple destructor.
*/
~ci_config_job() override = default;
~CIConfigJob() override = default;
/**
* @brief Method used to execute the job.
* @return The exit code of the job.
*/
int run() override;
int Run() override;
};
#endif // CI_CONFIG_JOB_H
#include "ci_environment.h"
#include "CIEnvironment.h"
#include <cstdio>
#include <cstdlib>
......@@ -7,29 +7,29 @@
#include "spdlog/spdlog.h"
ci_environment::ci_environment() {
CIEnvironment::CIEnvironment() {
const char *esf = std::getenv("SYSTEM_FAILURE_EXIT_CODE");
const char *ebf = std::getenv("BUILD_FAILURE_EXIT_CODE");
if (esf && ebf) {
std::sscanf(esf, "%d", &exit_system_failure_);
std::sscanf(ebf, "%d", &exit_build_failure_);
std::sscanf(esf, "%d", &ExitSystemFailure);
std::sscanf(ebf, "%d", &ExitBuildFailure);
SPDLOG_DEBUG("SYSTEM_FAILURE_EXIT_CODE={}, BUILD_FAILURE_EXIT_CODE={}",
exit_system_failure_, exit_build_failure_);
ExitSystemFailure, ExitBuildFailure);
} else {
exit_system_failure_ = 2;
exit_build_failure_ = 1;
ExitSystemFailure = 2;
ExitBuildFailure = 1;
SPDLOG_WARN("gitlab-runner did not provide SYSTEM_FAILURE_EXIT_CODE and/or "
"BUILD_FAILURE_EXIT_CODE environment variables. Using return values {} and {} "
"instead.", exit_system_failure_, exit_build_failure_);
"instead.", ExitSystemFailure, ExitBuildFailure);
}
const char *home = std::getenv("HOME");
if (home)
home_ = std::filesystem::path(home);
Home = std::filesystem::path(home);
else {
SPDLOG_ERROR("Could not get home directory");
std::exit(exit_system_failure_);
std::exit(ExitSystemFailure);
}
std::vector<std::string> keys({"CUSTOM_ENV_ARTIFACT_DOWNLOAD_ATTEMPTS",
......@@ -140,7 +140,7 @@ ci_environment::ci_environment() {
try {
std::string val = std::getenv(key.c_str());
SPDLOG_DEBUG("Environment variable ({}, {})", key, val);
variables.insert({key, val});
Variables.insert({key, val});
} catch (...) {
// TODO: This spams the UI since not all vars seem to be set in all steps of the job
//std::cerr << "Couldn't get value of environment variable " << k << std::endl;
......@@ -148,44 +148,22 @@ ci_environment::ci_environment() {
}
std::filesystem::path job_path_relative = variables["CUSTOM_ENV_CI_RUNNER_ID"] + "-"
+ variables["CUSTOM_ENV_CI_PROJECT_ID"] + "-"
+ variables["CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID"] + "-"
+ variables["CUSTOM_ENV_CI_JOB_ID"];
std::filesystem::path jobPathRelative = Variables["CUSTOM_ENV_CI_RUNNER_ID"] + "-"
+ Variables["CUSTOM_ENV_CI_PROJECT_ID"] + "-"
+ Variables["CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID"] + "-"
+ Variables["CUSTOM_ENV_CI_JOB_ID"];
if (job_path_relative.string().front() == '-' || job_path_relative.string().back() == '-'
|| job_path_relative.string().find("--") != std::string::npos) {
if (jobPathRelative.string().front() == '-' || jobPathRelative.string().back() == '-'
|| jobPathRelative.string().find("--") != std::string::npos) {
SPDLOG_ERROR("At least one of the following environment variables was not set: "
"CUSTOM_ENV_CI_RUNNER_ID, CUSTOM_ENV_CI_PROJECT_ID, "
"CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID, CUSTOM_ENV_CI_JOB_ID. This resulted in "
"an invalid build path.");
std::exit(exit_system_failure_);
std::exit(ExitSystemFailure);
}
builds_dir_ = home_ / "builds" / job_path_relative;
cache_dir_ = home_ / "cache" / job_path_relative;
SPDLOG_INFO("builds_dir: {}", builds_dir_.string());
SPDLOG_INFO("cache_dir: {}", cache_dir_.string());
}
ci_environment *ci_environment::instance() {
static mem_guard g;
if (!instance_)
instance_ = new ci_environment();
return instance_;
}
int ci_environment::exit_system_failure() { return exit_system_failure_; }
int ci_environment::exit_build_failure() { return exit_build_failure_; }
std::filesystem::path ci_environment::builds_dir() { return builds_dir_; }
std::filesystem::path ci_environment::cache_dir() { return cache_dir_; }
std::string ci_environment::get_variable(std::string key) { return variables[key]; }
ci_environment *ci_environment::instance_ = nullptr;
ci_environment::mem_guard::~mem_guard() {
if(ci_environment::instance_ != nullptr) {
delete ci_environment::instance_;
ci_environment::instance_ = nullptr;
}
BuildsDir = Home / "builds" / jobPathRelative;
CacheDir = Home / "cache" / jobPathRelative;
SPDLOG_INFO("builds_dir: {}", BuildsDir.string());
SPDLOG_INFO("cache_dir: {}", CacheDir.string());
}
#ifndef CI_ENVIRONMENT_H
#define CI_ENVIRONMENT_H
#include <filesystem>
#include <map>
#include <string>
#include "Environment.h"
/**
* @brief Easy global access to ci_environment::exit_system_failure_.
*/
#define EXIT_SYSTEM_FAILURE ci_environment::instance()->exit_system_failure()
/**
* @brief Easy global access to ci_environment::exit_build_failure_.
*/
#define EXIT_BUILD_FAILURE ci_environment::instance()->exit_build_failure()
#include <map>
/**
* @brief Class to represent the CI environment.
*
* This class represents the environment the GitLab Runner service provides when calling the HPC
* runner. It is implemented as a singleton to provide easy global access to all environment
* variables.
* runner.
*/
class ci_environment {
class CIEnvironment : public Environment {
public:
/**
* @brief Simple constructor.
*/
CIEnvironment();
/**
* @brief Simple getter.
* @return #exit_system_failure_
* @return #ExitSystemFailure
*/
int exit_system_failure();
int GetExitSystemFailure() override { return ExitSystemFailure; }
/**
* @brief Simple getter.
* @return #exit_build_failure_
* @return #ExitBuildFailure
*/
int exit_build_failure();
int GetExitBuildFailure() override { return ExitBuildFailure; }
/**
* @brief Simple getter.
* @return #home_
* @return #Home
*/
std::filesystem::path home();
std::filesystem::path GetHome() override { return Home; }
/**
* @brief Simple getter.
* @return #builds_dir_
* @return #BuildsDir
*/
std::filesystem::path builds_dir();
std::filesystem::path GetBuildsDir() override { return BuildsDir; }
/**
* @brief Simple getter.
* @return #cache_dir_
* @return #CacheDir
*/
std::filesystem::path cache_dir();
std::filesystem::path GetCacheDir() override { return CacheDir; }
/**
* @brief Getter for environment variables.
* @param[in] key The name of the environment variable.
* @return If it exists, the value of the environment variable \p key, otherwise an empty
* string.
*/
std::string get_variable(std::string key);
/**
* @brief Getter for the singleton instance of this class.
* @return #instance_
*/
static ci_environment *instance();
std::string GetVariable(std::string key) override { return Variables[key]; }
private:
/**
* @brief Simple constructor. Set private to keep #instance_ singleton.
*/
ci_environment();
/**
* @brief Simple copy constructor. Set private to keep #instance_ singleton.
*/
ci_environment(const ci_environment&); // copy constructor
int exit_system_failure_; /**< The exit code to use when the system fails. */
int exit_build_failure_; /**< The exit code to use when the build fails. */
std::filesystem::path home_; /**< The path to the home directory of the current user. */
std::filesystem::path builds_dir_; /**< The path to the directory where repos are cloned and
int ExitSystemFailure; /**< The exit code to use when the system fails. */
int ExitBuildFailure; /**< The exit code to use when the build fails. */
std::filesystem::path Home; /**< The path to the home directory of the current user. */
std::filesystem::path BuildsDir; /**< The path to the directory where repos are cloned and
builds are executed. */
std::filesystem::path cache_dir_; /**< The path to the cache directory. */
std::map<std::string, std::string> variables; /**< A map containing the environment variables
std::filesystem::path CacheDir; /**< The path to the cache directory. */
std::map<std::string, std::string> Variables; /**< A map containing the environment variables
set by the GitLab runner service. */
static ci_environment *instance_; /**< A pointer to the singleton instance of this class. */
/**
* @brief Guard used to delete #instance_.
*/
class mem_guard {
public:
~mem_guard();
};
};
#endif // CI_ENVIRONMENT_H
#ifndef CI_JOB_H
#define CI_JOB_H
#include "Environment.h"
/**
* @brief Class to represent a job for the CI system.
*/
class ci_job {
class CIJob {
public:
/**
* @brief Simple constructor.
* @param env Sets #Env
*/
CIJob(Environment *env) : Env(env) {}
/**
* @brief Simple destructor.
*/
virtual ~ci_job() = 0;
virtual ~CIJob() = default;
/**
* @brief Method that is called to run the job. Children of this class need to implement this.
*/
virtual int run() = 0;
virtual int Run() = 0;
Environment *GetEnv() { return Env; }
private:
Environment *Env;
};
#endif // CI_JOB_H
#include "ci_job_factory.h"
#include "CIJobFactory.h"
#include <algorithm>
#include <cstdlib>
#include "ci_environment.h"
#include "ci_job.h"
#include "ci_cleanup_job.h"
#include "ci_config_job.h"
#include "ci_run_job_head.h"
#include "ci_run_job_cluster.h"
#include "CIJob.h"
#include "CICleanupJob.h"
#include "CIConfigJob.h"
#include "CIRunJobHead.h"
#include "CIRunJobCluster.h"
#include "spdlog/spdlog.h"
ci_job *ci_job_factory::create_ci_job(const std::string &job_stage,
const std::filesystem::path &script,
const std::string &substage, bool unified_output) {
CIJobFactory::CIJobFactory(Environment *env) {
if (env == nullptr)
throw std::invalid_argument("env must not be nullptr!");
Env = env;
}
CIJob *CIJobFactory::CreateCIJob(const std::string &jobStage, const std::filesystem::path &script,
const std::string &subStage, bool unifiedOutput) {
if (job_stage == "config")
return new ci_config_job();
if (jobStage == "config")
return new CIConfigJob(Env);
else if (job_stage == "run") {
else if (jobStage == "run") {
if (script.empty()) {
SPDLOG_ERROR("No script provided for run job.");
return nullptr;
} else {
std::error_code ec;
bool is_file = std::filesystem::is_regular_file(script, ec);
if (!is_file) {
if (!std::filesystem::is_regular_file(script, ec)) {
if (!ec) {
SPDLOG_ERROR("Provided script {} is not a regular file.", script.string());
} else {
......@@ -36,35 +40,35 @@ ci_job *ci_job_factory::create_ci_job(const std::string &job_stage,
}
}
if (substage.empty()) {
if (subStage.empty()) {
SPDLOG_ERROR("No substage provided for run job.");
return nullptr;
}
if (substage == "build_script")
return new ci_run_job_cluster(script, substage, unified_output);
if (subStage == "build_script")
return new CIRunJobCluster(Env, script, subStage, unifiedOutput);
else {
std::array<std::string, 8> other_substages = { "prepare_script",
"get_sources",
"restore_cache",
"download_artifacts",
"after_script",
"archive_cache",
"upload_artifacts_on_success",
"upload_artifacts_on_failure" };
if (std::any_of(other_substages.begin(), other_substages.end(),
[substage](std::string s) { return s == substage; }))
return new ci_run_job_head(script, substage);
std::array<std::string, 8> otherSubStages = { "prepare_script",
"get_sources",
"restore_cache",
"download_artifacts",
"after_script",
"archive_cache",
"upload_artifacts_on_success",
"upload_artifacts_on_failure" };
if (std::any_of(otherSubStages.begin(), otherSubStages.end(),
[subStage](std::string s) { return s == subStage; }))
return new CIRunJobHead(Env, script, subStage);
else {
SPDLOG_ERROR("Unknown substage {} for run job.", substage);
SPDLOG_ERROR("Unknown substage {} for run job.", subStage);
return nullptr;
}
}
}
else if (job_stage == "cleanup")
return new ci_cleanup_job();
else if (jobStage == "cleanup")
return new CICleanupJob(Env);
SPDLOG_ERROR("Unknown job stage {}.", job_stage);
SPDLOG_ERROR("Unknown job stage {}.", jobStage);
return nullptr;
}
......@@ -4,28 +4,33 @@
#include <filesystem>