Commit 16584970 authored by Uwe Jandt (DESY, HIFIS)'s avatar Uwe Jandt (DESY, HIFIS)
Browse files

Plot all available data (pdf, png, svg)

parent e83cd8b2
Pipeline #97980 passed with stage
in 3 minutes and 21 seconds
plots/
\ No newline at end of file
plots/
plot_general/plot.pdf
# Ignore all intermediate data that can pile up in local repositories
data/*/*.csv
data/*/*.plt
data/*/*.svg
data/*/*.png
data/*/*.pdf
2021*/*.pdf
2021*/*.png
\ No newline at end of file
......@@ -12,7 +12,7 @@ before_script:
- export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR
- apt-get update && apt-get -o dir::cache::archives="$APT_CACHE_DIR" -y install gnuplot gawk imagemagick poppler-utils git
- git submodule sync --recursive
- git submodule update --remote --recursive
- git submodule update --remote --recursive
- echo "Before script section END."
after_script:
......@@ -28,10 +28,9 @@ plot_general:
GIT_SUBMODULE_STRATEGY: recursive
script:
- mkdir -v plots
- ls -laR > plots/ls.txt
- cd plot_general
- bash ./plot.sh
- mv -v plot.pdf plot.png ../plots
- bash ./plot_replotgraphs.sh
artifacts:
paths:
- plots
......@@ -8,7 +8,7 @@ set yrange [0:*]
set xdata time
set timefmt "%Y-%m-%d %H:%M"
set format x "%Y-%m-%d"
set xrange ["2019-07-01":"2021-07-02"]
set xrange ["2019-07-01":"2022-01-01"]
set grid
set xtics rotate by 45
set key top left
......
......@@ -8,7 +8,7 @@ set yrange [0:*]
set xdata time
set timefmt "%Y-%m-%d %H:%M"
set format x "%Y-%m-%d"
set xrange ["2019-07-01":"2021-07-02"]
set xrange ["2019-07-01":"2022-01-01"]
set grid
set xtics rotate by 45 offset -4,-3
set key top left
......
......@@ -8,7 +8,7 @@ set yrange [0:*]
set xdata time
set timefmt "%Y-%m-%d %H:%M"
set format x "%Y-%m-%d"
set xrange ["2019-07-01":"2021-07-02"]
set xrange ["2019-07-01":"2022-01-01"]
set xtics rotate by 45 offset -4,-3
set ytics nomirror
set key top left
......
#!/bin/gnuplot
set datafile separator ","
set output './plot.pdf'
set yrange [0:*]
set xdata time
set timefmt "%Y-%m-%d %H:%M"
set format x "%Y-%m-%d"
set xrange ["2019-09-01":"2021-10-02"]
set xtics rotate by 45 offset -4,-3
set ytics nomirror
set key top left
set border 3
set terminal pdf color enhanced size 4,6 font "Helvetica, 16"
if (!exists("MP_LEFT")) MP_LEFT = 0.16
if (!exists("MP_RIGHT")) MP_RIGHT = .82
if (!exists("MP_BOTTOM")) MP_BOTTOM = .15
if (!exists("MP_TOP")) MP_TOP = .98
if (!exists("MP_xGAP")) MP_xGAP = 0.07
if (!exists("MP_yGAP")) MP_yGAP = 0.03
set multiplot layout 2,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
set timefmt "%d.%m.%Y"
set ylabel "Helmholtz AAI users" offset 1
set xtics textcolor rgbcolor "white" nomirror
plot '../stats/aai/aai_users.csv' u 1:2 w l lc black lw 2 notitle
set border 11
set datafile separator ","
set ylabel "Cumulative course participant-hours"
set y2label "Per-month participant-hours" offset -1
set y2tics
set xtics textcolor rgbcolor "black" offset -4,-4 nomirror
set timefmt "%Y-%m"
set style fill pattern 6
plot '../stats/workshops/hifis_all_workshop_statistics.csv' u 1:2 w boxes axis x1y2 lc rgb "#8CB423" lw 0.8 title 'per month' ,\
'' u 1:3 w l lc black lw 2 title 'cumulative'
unset multiplot
#!/bin/bash
gnuplot plot.plt
for fn in *.pdf; do
pdftoppm $fn plot_temp -singlefile -aa no -aaVector no -r 300 -png
convert plot_temp.png ${fn%.pdf}.png
rm plot_temp.png
done
# Service Usage Plots
Collection of service usage for reporting with crude data plotting and conversion.
In this project, plots are generated from service usage data.
The data is incorporated via submodules from separate projects.
* Helmholtz Backbone (VPN)
* Helmholtz AAI
* Gitlab (redo from [here](https://gitlab.hzdr.de/fwcc-management/gitlab-statistics)), for the sake of completeness.
#### CI generated plot
* [Browse latest build](https://gitlab.hzdr.de/jandt-desy/service-usage-plots-ci/-/jobs/artifacts/master/browse/plots?job=plot_general)
* ![pic](https://gitlab.hzdr.de/jandt-desy/service-usage-plots-ci/-/jobs/artifacts/master/raw/plots/plot.png?job=plot_general)
## Generated Plots
The plots are generated automatically when changes are detected or the generation pipeline is triggered.
* [**Latest build, all graphs**](https://gitlab.hzdr.de/jandt-desy/service-usage-plots-ci/-/jobs/artifacts/master/browse/plots?job=plot_general)
* [Reported graphs](example_graphs.md), used in reports, reproduced with most recent data.
* Example of inline svg (AAI users):
![AAI users](https://gitlab.hzdr.de/jandt-desy/service-usage-plots-ci/-/jobs/artifacts/master/raw/plots/svg/aai_users-plot_2.svg?job=plot_general)
## Services
This is the list of services currently being considered.
* Helmholtz AAI
* Helmholtz Backbone (VPN)
* Helmholtz Codebase - (Gitlab at HZDR)
* Nubes (HZB)
* Mattermost (HZDR)
* Openstack (FZJ)
* bwSync&Share (KIT)
#### Local execution
## Local execution
Clone this repository *recursively, i.e. including the data*, with:
```
git clone --recursive git@gitlab.hzdr.de:jandt-desy/service-usage-plots-ci.git
```
If you have cloned the repository already, you might need to update the submodules (i.e., data) with
```
git submodule sync --recursive
git submodule update --remote --recursive
```
Necessary for converting and plotting: `gnuplot`, `awk`, imagemagick `convert`.
Necessary for converting and plotting: `gnuplot`, `gawk`, `imagemagick`, `poppler-utils`.
```code
cd 202102_Backbone # or any other of the sub-folders
cd plot_general # or any other of the sub-folders
./plot.sh
```
## Structure
* `subprojects`: Linked projects providing the source data. Internal structure is *very* heterogenous atm.
* `data`: Collected raw data from subprojects.
- `servicename`: One Level of subfolders with one service per folder.
- `data.csv` (mandatory): Raw data with one date/time column, and at least one data column
- `timefmt.txt` (optional): Provide specific date/time format, if needed. Standard is `%Y-%m-%d %H:%M`.
- `filter.sh` (optional): Provide filter/data integration script, if needed. Otherwise raw data is plotted.
* `plots`: Collection of all plots to be output => Artifacts folder
../../subprojects/AAI/stats/aai/aai_centers.csv
\ No newline at end of file
../../subprojects/AAI/stats/aai/aai_services.csv
\ No newline at end of file
# Use specific date/time format, overriding standard format.
set timefmt "%d.%m.%y"
\ No newline at end of file
../../subprojects/AAI/stats/aai/aai_userstats.csv
\ No newline at end of file
#!/bin/bash
# summing up columns 2 and 3, combined
in=data.csv
out=data_plot.csv
if [ $in -nt $out ]; then
cat $in | awk 'BEGIN{FS=","}{print $1","a; a+=$2+$3}' > $out
fi
# Use specific date/time format, overriding standard format.
set timefmt "%d.%m.%Y"
\ No newline at end of file
../../subprojects/AAI/stats/aai/aai_vo.csv
\ No newline at end of file
# Use specific date/time format, overriding standard format.
set timefmt "%d.%m.%Y"
\ No newline at end of file
# Use specific date/time format, overriding standard format.
set timefmt "%d.%m.%Y"
\ No newline at end of file
../../subprojects/AAI/stats/gitlab/active_users.csv
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment