Commit 7a8378fe authored by Pape, David (FWCC) - 139658's avatar Pape, David (FWCC) - 139658
Browse files

Merge branch 'state-update' into 'master'

State update

See merge request !8
parents fcfa36ea 7311e965
Pipeline #77762 failed with stages
in 52 seconds
CMakeLists.txt.user
*.kdev4
build/
docs/source_code_doc/
stages:
- build
- test
- docs
- deploy
image: registry.hzdr.de/fwcc/slurm-in-docker:latest
......@@ -26,8 +27,22 @@ build_program:
paths:
- build
test_program:
stage: test
dependencies:
- build_program
script:
- cd build
- make coverage
artifacts:
paths:
- build/coverage
build_docs:
stage: build
stage: docs
dependencies: []
before_script:
- ":"
script:
- pip install -r docs/requirements.txt
- mkdir build && cd build
......@@ -42,17 +57,6 @@ build_docs:
paths:
- public
test_program:
stage: test
dependencies:
- build_program
script:
- cd build
- make coverage
artifacts:
paths:
- build/coverage
pages:
stage: deploy
before_script:
......
......@@ -56,8 +56,9 @@ CIJob *CIJobFactory::CreateCIJob(const std::string &jobStage, const std::filesys
"archive_cache",
"upload_artifacts_on_success",
"upload_artifacts_on_failure" };
if (std::any_of(otherSubStages.begin(), otherSubStages.end(),
[subStage](std::string s) { return s == subStage; }))
if (std::find(otherSubStages.cbegin(), otherSubStages.cend(), subStage)
!= otherSubStages.cend())
return new CIRunJobHead(Env, script, subStage);
else {
SPDLOG_ERROR("Unknown substage {} for run job.", subStage);
......
......@@ -57,27 +57,30 @@ int SlurmJob::Run() {
UpdateState();
while (std::any_of(InProgressStates.begin(), InProgressStates.end(),
[this](std::string s) { return s == State; })) {
std::cerr << "Slurm job (" << ID << ") state: " << State << std::endl;
while (std::find(InProgressStates.cbegin(), InProgressStates.cend(), State)
!= InProgressStates.cend()) {
{
std::lock_guard guard(OutputLock);
std::cerr << "\rSlurm job (" << ID << ") state: " << State << std::flush;
}
std::this_thread::sleep_for(std::chrono::seconds(5));
UpdateState();
}
killSigP.set_value();
std::cerr << "Slurm job (" << ID << ") state: " << State
<< ", exit code: " << ExitCode << std::endl;
{
std::lock_guard guard(OutputLock);
std::cerr << "\rSlurm job (" << ID << ") state: " << State
<< ", exit code: " << ExitCode << std::endl;
}
// TODO: Differentiate between job states and correctly exit with system failure or build
// failure (when users fault)
//if (State != "COMPLETED") {
if (std::none_of(SuccessStates.begin(), SuccessStates.end(),
[this](std::string s) { return s == State; })) {
if (std::find(SuccessStates.cbegin(), SuccessStates.cend(), State) == SuccessStates.cend()) {
SPDLOG_INFO("Slurm job with ID {} has not completed. State {}, exit code {}", ID, State,
ExitCode);
if (std::any_of(FailStatesJob.begin(), FailStatesJob.end(),
[this](std::string s) { return s == State; }))
if (std::find(FailStatesJob.cbegin(), FailStatesJob.cend(), State) != FailStatesJob.cend())
return HPCJob::GetEnv()->GetExitBuildFailure();
else
return HPCJob::GetEnv()->GetExitSystemFailure();
......@@ -232,8 +235,10 @@ void SlurmJob::OutputFile(std::ostream &out, const std::string &fileName,
// (Slurm buffers its output anyway, so ...)
while (killSig.wait_for(std::chrono::milliseconds(100)) == std::future_status::timeout) {
std::string line;
while (std::getline(file, line))
out << line << std::endl;
while (std::getline(file, line)) {
std::lock_guard guard(OutputLock);
out << '\r' << line << std::endl;
}
file.clear();
}
SPDLOG_DEBUG("Output job for {} received kill signal. Finalizing ...", fileName);
......
......@@ -8,6 +8,7 @@
#include <fstream>
#include <future>
#include <map>
#include <mutex>
#include <string>
#include <utility>
......@@ -65,7 +66,7 @@ private:
* @param[in] fileFame The path to the file to be outputted.
* @param[in] killSig Signals when to stop the output.
*/
void OutputFile(std::ostream& out, const std::string &fileName,
void OutputFile(std::ostream &out, const std::string &fileName,
std::shared_future<void> killSig);
/**
......@@ -87,6 +88,8 @@ private:
static std::string YmlToBash(std::string s);
static inline std::mutex OutputLock;
static inline std::array<std::string, 1> SuccessStates = {
"COMPLETED"
};
......@@ -119,7 +122,6 @@ private:
"STOPPED",
"SUSPENDED"
};
};
#endif // SLURM_JOB_H
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment