Commit 3602f2fd authored by Erxleben, Fredo's avatar Erxleben, Fredo
Browse files

Resolve "Split metadata YAML file up into one file per question collection"

parent 5f76fdbf
Pipeline #94181 passed with stage
in 1 minute and 7 seconds
......@@ -25,6 +25,8 @@ build:package:
- pip install poetry
- poetry config virtualenvs.in-project true
- poetry install
# Create required folders that the surveyval tool looks for on startup
- mkdir metadata
script:
- make package
rules:
......
......@@ -24,6 +24,8 @@ test:
- pip install poetry
- poetry config virtualenvs.in-project true
- poetry install
# Create required folders that the surveyval tool looks for on startup
- mkdir metadata
stage: test
script:
- make test
......
......@@ -175,13 +175,21 @@ def analyze(survey_data: pathlib.Path) -> None:
# Load the metadata
logging.info(f"Attempt to load metadata from {settings.METADATA}")
with settings.METADATA.open(mode="r",
encoding="utf-8") as metadata_io_stream:
metadata_yaml = yaml.safe_load(metadata_io_stream)
raw_data.load_metadata(metadata_yaml)
yaml_files = [file for file in settings.METADATA.iterdir()]
# Filter out those files that do not have a YAML file extension
yaml_files = list(filter(
lambda file: file.suffix.lower() in [".yml", ".yaml"],
yaml_files
))
for file in yaml_files:
logging.debug(f"Loading Metadata from {file}")
with file.open(mode="r", encoding="utf-8") as io_stream:
metadata_yaml = yaml.safe_load(io_stream)
raw_data.load_metadata(metadata_yaml)
# Load the actual survey data
logging.info(f"Attempt to load survey data from {survey_data}")
with survey_data.open(mode="r", encoding="utf-8") as data_io_stream:
csv_reader = reader(data_io_stream)
raw_data.load_survey_data(csv_data=list(csv_reader))
......
......@@ -126,23 +126,52 @@ class FileSettings(BaseSettings):
PREPROCESSING_FILENAME: Path = Path("preprocess.py")
@validator("PREPROCESSING_FILENAME")
def validate_preprocessing_script(cls, to_validate: str) -> Path:
def validate_preprocessing_script(cls, to_validate: Path) -> Path:
"""
Assure, that preprocessing script is a Python file.
Ensure that preprocessing script is a Python file.
Args:
to_validate (str):
Preprocessing script path as string to be validated.
to_validate:
Preprocessing script path to be validated.
Returns:
Path: Path to the preprocessing script.
Path to the preprocessing script.
Raises:
ValueError:
If the given script did not end with ".py" and therefore
probably is not a python script.
"""
if not str(to_validate).endswith(".py"):
if to_validate.suffix != ".py":
raise ValueError("Preprocessing Script must be a python script")
return Path(to_validate)
return to_validate
# Path to metadata folder
METADATA: Path = Path("metadata/")
@validator("METADATA")
def validate_metadata_folder(cls, to_validate: Path) -> Path:
"""
Ensure the metadata folder is a folder and exists.
Args:
to_validate:
The path to the metadata folder, which is to be validated.
Returns:
The path to the metadata folder if it is valid.
# Path to metadata
METADATA: Path = Path("metadata/meta.yml")
Raises:
ValueError:
If either the given path was not a folder or did not exist.
"""
if not to_validate.exists():
raise ValueError(f"Metadata folder {to_validate.absolute()} does "
f"not exist")
if not to_validate.is_dir():
raise ValueError(f"Metadata folder {to_validate.absolute()} is "
f"not a folder")
return to_validate
# Path in which modules to be executed are located which defaults
# to "scripts" folder.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment