Commit 9e7be3b9 authored by mdolling-gfz's avatar mdolling-gfz
Browse files

moved config to external config file

parent 0c584f0c
Pipeline #70242 passed with stages
in 7 minutes and 25 seconds
......@@ -31,3 +31,10 @@ manual dependency config:
- import name: 'hifis_surveyval#noqa'
pkg name: 'hifis_surveyval'
ignore: True
- import name: 'pkg_resources'
pkg name: 'pkg_resources'
ignore: True
- import name: 'python-dotenv'
pkg name: 'python-dotenv'
licenses:
- 'BSD-3-Clause'
......@@ -33,8 +33,7 @@ import os
import sys
from unittest.mock import MagicMock
# Now we can import local modules.
import hifis_surveyval # noqa
import pkg_resources
# Determine the absolute path to the directory containing the python modules.
_pysrc = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
......@@ -111,14 +110,12 @@ author = "HIFIS Software"
# built documents.
#
# The short X.Y version.
# version = hifis_surveyval.__version__
version = pkg_resources.require("hifis_surveyval")[0].version
# The full version, including alpha/beta/rc tags.
# release = hifis_surveyval.__release__
# The full version, including alpha/beta/rc tags
release = hifis_surveyval.__release__
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
......
# hifis-surveyval
# Framework to help developing analysis scripts for the HIFIS Software survey.
#
# SPDX-FileCopyrightText: 2021 HIFIS Software <support@hifis.net>
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
METADATA: ../analysis/metadata/Survey_XYZ.yml
OUTPUT_FOLDER: output
OUTPUT_FORMAT: PNG
SCRIPT_FOLDER: ../analysis/scripts
SCRIPT_NAMES: []
......@@ -28,5 +28,3 @@ This project is used to develop analysis scripts for the HIFIS Software survey.
.. currentmodule:: hifis_surveyval
.. moduleauthor:: HIFIS Software <software@hifis.net>
"""
from .version import __release__, __version__ # noqa
......@@ -41,21 +41,21 @@ It can be used as a handy facility for running the task from a command line.
.. moduleauthor:: HIFIS Software <software@hifis.net>
"""
import logging
import sys
from pathlib import Path
from typing import List
import click
import pandas
import pkg_resources
from hifis_surveyval import dispatch, globals
from hifis_surveyval.core import environment, settings
from hifis_surveyval.metadata import (
construct_questions_from_metadata,
fetch_participant_answers,
)
from hifis_surveyval.settings import OutputFormat
from .__init__ import __version__
setting: settings.Settings = settings.get_settings()
@click.group()
......@@ -68,71 +68,35 @@ from .__init__ import __version__
help="Enable verbose output. "
"Increase verbosity by setting this option up to 3 times.",
)
@click.option(
"--scripts",
"-s",
default="scripts",
show_default=True,
help="Select the folder containing analysis scripts.",
)
@click.option(
"--names",
"-n",
multiple=True,
default=[],
help="Optionally select the specific script names contained "
"in the scripts folder (while omitting file endings) "
"which should be executed.",
)
@click.option(
"--output-folder",
"-o",
default="output",
show_default=True,
help="Select the folder to put the generated output like plots " "into.",
)
@click.option(
"--output-format",
"-f",
default="PNG",
show_default=True,
help=f"Designate output format. "
f"Supported values are: {OutputFormat.list_supported()}.",
)
def cli(
verbose: int, scripts: str, names: List[str], output_folder: str, output_format: str
) -> None:
def cli(verbose: int) -> None:
"""Analyze a given CSV file with a set of independent python scripts."""
# NOTE that click takes above documentation for generating help text
# Thus the documentation refers to the application per se and not the
# function (as it should)
set_verbosity(verbose)
set_output_format(output_format)
logging.info(f"Selected script folder: {scripts}")
globals.settings.script_folder = Path(scripts)
logging.info(f"Selected output folder: {output_folder}")
globals.settings.output_folder = Path(output_folder)
# Set a list of selected module names contained in the module folder.
globals.settings.script_names = names
sys.path.insert(0, scripts)
globals.settings = setting
@cli.command()
def version() -> None:
"""Get the library version."""
click.echo(click.style(f"{__version__}", bold=True))
version = pkg_resources.require("hifis_surveyval")[0].version
click.echo(click.style(f"{version}", bold=True))
@cli.command()
def init() -> None:
"""
Create a default configuration in a .env file.
It will overwrite any existing .env file.
"""
settings.create_config_file()
@click.argument("file_name", type=click.File(mode="r"))
@click.option(
"--metadata",
"-m",
default="metadata/meta.yml",
help="Give file name which contains survey metadata.",
)
def analyze(file_name, metadata: str) -> None:
@cli.command()
def analyze(file_name) -> None:
"""
Read the given files into global data and metadata objects.
......@@ -141,12 +105,13 @@ def analyze(file_name, metadata: str) -> None:
If the metadata file can not be parsed, an error will be printed and
the program will abort.
"""
environment.prepare_environment()
logging.info(f"Analyzing file {file_name.name}")
try:
frame: pandas.DataFrame = pandas.read_csv(
file_name,
true_values=globals.settings.true_values,
false_values=globals.settings.false_values,
true_values=globals.settings.TRUE_VALUES,
false_values=globals.settings.FALSE_VALUES,
)
logging.debug("\n" + str(frame))
......@@ -157,14 +122,14 @@ def analyze(file_name, metadata: str) -> None:
logging.error("Could not parse the given file as CSV")
exit(1)
logging.info(f"Attempt to load metadata from {metadata}")
logging.info(f"Attempt to load metadata from {globals.settings.METADATA}")
# Load survey metadata from given YAML file.
try:
construct_questions_from_metadata(Path(metadata))
construct_questions_from_metadata(Path(globals.settings.METADATA))
# When debugging, print all parsed Questions
if globals.settings.verbosity == logging.DEBUG:
if globals.settings.VERBOSITY == logging.DEBUG:
logging.debug("Parsed Questions:")
for question in globals.survey_questions.values():
logging.debug(question)
......@@ -175,7 +140,7 @@ def analyze(file_name, metadata: str) -> None:
exit(1)
dispatcher = dispatch.Dispatcher(
globals.settings.script_folder, globals.settings.script_names
globals.settings.SCRIPT_FOLDER, globals.settings.SCRIPT_NAMES
)
dispatcher.discover()
dispatcher.load_all_modules()
......@@ -211,6 +176,7 @@ def set_verbosity(verbose_count: int) -> None:
)
new_level: int = verbosity_options[option_index]
logging.basicConfig(
level=new_level,
format="%(asctime)s "
......@@ -218,7 +184,8 @@ def set_verbosity(verbose_count: int) -> None:
"%(module)s.%(funcName)s(): "
"%(message)s",
)
globals.settings.verbosity = new_level
setting.VERBOSITY = new_level
if not new_level == logging.ERROR:
click.echo(
......@@ -228,24 +195,3 @@ def set_verbosity(verbose_count: int) -> None:
fg="yellow",
)
)
def set_output_format(output_format: str) -> None:
"""
Attempt to determine the desired output format from the given string.
The format name is not case sensitive.
Args:
output_format: A textual representation of the desired format.
"""
try:
chosen_format: OutputFormat = OutputFormat[output_format.upper()]
globals.settings.output_format = chosen_format
logging.info(f"Output format set to {chosen_format.name}")
except KeyError:
logging.error(
f"Output Format {output_format} not recognized. "
f"Supported values are: {OutputFormat.list_supported()}"
)
exit(2)
#!/usr/bin/env python
# hifis-surveyval
# Framework to help developing analysis scripts for the HIFIS Software survey.
#
......@@ -20,14 +18,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -*- coding: utf-8 -*-
"""
This module contains project version information.
This package provides core functionalities.
.. currentmodule:: hifis_surveyval.version
.. currentmodule:: hifis_surveyval.core
.. moduleauthor:: HIFIS Software <software@hifis.net>
"""
__version__ = "0.2.0" #: the working version
__release__ = "0.2.0" #: the release version
# hifis-surveyval
# Framework to help developing analysis scripts for the HIFIS Software survey.
#
# SPDX-FileCopyrightText: 2021 HIFIS Software <support@hifis.net>
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This module provides the functions to set up the environment.
It adds the script folder to PATH and creates required folders.
.. currentmodule:: hifis_surveyval.core.environment
.. moduleauthor:: HIFIS Software <software@hifis.net>
"""
import sys
from hifis_surveyval.globals import settings
def prepare_environment() -> None:
"""
Prepare the runtime environment.
* setting sys path to load scripts
* creating output folder to save images
"""
# set syspath to later on load scripts
sys.path.insert(0, settings.SCRIPT_FOLDER)
# create folder to output the results
if settings.ANALYSIS_OUTPUT_PATH is not None:
if not settings.ANALYSIS_OUTPUT_PATH.exists():
settings.ANALYSIS_OUTPUT_PATH.mkdir(parents=True)
# hifis-surveyval
# Framework to help developing analysis scripts for the HIFIS Software survey.
#
# SPDX-FileCopyrightText: 2021 HIFIS Software <support@hifis.net>
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This module handles settings.
It provides:
* settings classes
* getter for settings
* an export function to create a file
.. currentmodule:: hifis_surveyval.core.settings
.. moduleauthor:: HIFIS Software <software@hifis.net>
"""
import logging
from datetime import datetime
from enum import Enum, auto, unique
from pathlib import Path
from typing import Any, Dict, List, Set
import yaml
from pydantic import BaseSettings, validator
config_filename = Path("hifis-surveyval.yml")
@unique
class OutputFormat(Enum):
"""An Abstraction of the supported output formats for generated images."""
SCREEN = auto()
PDF = auto()
PNG = auto()
SVG = auto()
@staticmethod
def list_supported() -> Set:
"""Generate a set listing the supported output formats."""
return {value.name for value in OutputFormat}
class FileSettings(BaseSettings):
"""Settings, that the user can change."""
# Path to metadata
METADATA: Path = Path("metadata/meta.yml")
# Path in which modules to be executed are located which defaults
# to "scripts" folder.
SCRIPT_FOLDER: Path = Path("scripts")
# List of selected module names to be executed which defaults to
# an empty list for all modules in the module folder.
SCRIPT_NAMES: List[str] = []
# The Format in which the data should be output
OUTPUT_FORMAT: Any = OutputFormat.PNG
@validator("OUTPUT_FORMAT", pre=True)
def set_output_format(cls, to_validate) -> OutputFormat:
"""Parse format to enum object."""
if isinstance(to_validate, OutputFormat):
return to_validate
for output_format in OutputFormat:
if to_validate == output_format.name:
return output_format
raise ValueError(
f"Wrong output format. Only {OutputFormat.list_supported()} are implemented"
)
# Folder, into which the output file goes
# if the output format is not screen
OUTPUT_FOLDER: Path = Path("output")
class Config:
"""Subclass for specification."""
case_sensitive = True
class Settings(FileSettings):
"""Settings, that are either static or can be changed via the cli interface."""
VERBOSITY: int = logging.NOTSET
# The date prefix is used to identify the run
# (e.g. for saving output images)
RUN_TIMESTAMP: str = None
@validator("RUN_TIMESTAMP", pre=True)
def set_timestamp(cls, to_validate) -> str:
"""Get the current datetime."""
return datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
# Path to sub-folder which holds all output files of a single
# analysis run
ANALYSIS_OUTPUT_PATH: Path = None
@validator("ANALYSIS_OUTPUT_PATH")
def assemble_output_path(cls, to_validate, values: Dict[str, Any]) -> Path:
"""Assemble path from user settings and datetime."""
return values.get("OUTPUT_FOLDER") / values.get("RUN_TIMESTAMP")
# Using a set for true_values and false_values to avoid duplicates and
# because order does not matter
TRUE_VALUES: Set[str] = {"True", "Yes", "Y", "On", "1"}
"""
A set of strings to be interpreted as boolean 'True' when
parsing the input data.
"""
FALSE_VALUES: Set[str] = {"False", "No", "N", "Off", "0"}
"""
A set of strings to be interpreted as boolean 'False' when
parsing the input data.
"""
@validator("FALSE_VALUES", "TRUE_VALUES", pre=True)
def case_insensitive_values(cls, to_validate) -> Set:
"""Extend list of values to match all cases."""
additional_lower: Set[str] = set(map(str.lower, to_validate))
additional_upper: Set[str] = set(map(str.upper, to_validate))
to_validate.update(additional_lower.union(additional_upper))
return to_validate
def create_config_file():
"""Create a file to store the config."""
default_settings = Settings()
config_dict = {}
for key in FileSettings.__fields__:
value = default_settings.__getattribute__(key)
if isinstance(value, OutputFormat):
config_dict[key] = value.name
elif isinstance(value, Path):
config_dict[key] = str(value)
else:
config_dict[key] = value
with open(config_filename, "w") as config_file:
yaml.dump(config_dict, config_file)
def get_settings() -> Settings:
"""Return an instance of Settingss."""
settings: Settings = Settings()
if Path.is_file(config_filename):
with open(config_filename, "r") as config_file:
config = yaml.load(config_file, Loader=yaml.FullLoader)
settings = Settings(**config)
return settings
......@@ -32,7 +32,7 @@ from typing import Dict
from pandas import DataFrame
from .answer import AnswerType
from hifis_surveyval.answer import AnswerType
class DataContainer(object):
......
......@@ -26,9 +26,9 @@ This module provides the global definitions for the project.
"""
from typing import Dict
from hifis_surveyval.core.settings import Settings
from hifis_surveyval.data import DataContainer
from hifis_surveyval.question import AbstractQuestion
from hifis_surveyval.settings import Settings
#: A global copy-on-read container for providing the survey data
#: to the analysis functions
......
......@@ -40,10 +40,9 @@ import numpy
import yaml
from hifis_surveyval import globals
from .answer import Answer, AnswerType, ValidAnswerTypes
from .data import DataContainer
from .question import AbstractQuestion, Question, QuestionCollection
from hifis_surveyval.answer import Answer, AnswerType, ValidAnswerTypes
from hifis_surveyval.data import DataContainer
from hifis_surveyval.question import AbstractQuestion, Question, QuestionCollection
# The YAML dictionary has a recursive type
YamlDict = Dict[str, Optional[Union[str, "YamlDict"]]]
......
......@@ -41,8 +41,8 @@ from matplotlib import pyplot, rcParams
from matplotlib.colors import ListedColormap
from pandas import DataFrame
from hifis_surveyval.core.settings import OutputFormat
from hifis_surveyval.globals import settings
from hifis_surveyval.settings import OutputFormat
def _output_pyplot_image(output_file_stem: str = "") -> None:
......@@ -61,7 +61,7 @@ def _output_pyplot_image(output_file_stem: str = "") -> None:
automatic generation of a file name from the date
of the run and the module producing the image.
"""
if settings.output_format == OutputFormat.SCREEN:
if settings.OUTPUT_FORMAT == OutputFormat.SCREEN:
pyplot.show()
pyplot.close()
return
......@@ -81,10 +81,10 @@ def _output_pyplot_image(output_file_stem: str = "") -> None:
output_file_stem: str = f"{calling_module_name}"
file_ending: str = settings.output_format.name.lower()
file_ending: str = settings.OUTPUT_FORMAT.name.lower()
file_name: str = f"{output_file_stem}.{file_ending}"
output_path: Path = settings.analysis_output_path / file_name
output_path: Path = settings.ANALYSIS_OUTPUT_PATH / file_name
if output_path.exists():
logging.warning(f"Overriding existing output file {output_path}")
......
......@@ -36,7 +36,7 @@ from typing import Dict, List, Optional, Tuple
import numpy
from pandas import DataFrame, Series
from .answer import Answer, AnswerType
from hifis_surveyval.answer import Answer, AnswerType
class AbstractQuestion(ABC):
......
# hifis-surveyval
# Framework to help developing analysis scripts for the HIFIS Software survey.
#
# SPDX-FileCopyrightText: 2021 HIFIS Software <support@hifis.net>
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This module provides the definitions for a settings container.
.. currentmodule:: hifis_surveyval.settings
.. moduleauthor:: HIFIS Software <software@hifis.net>
"""
import logging
from datetime import datetime
from enum import Enum, auto, unique