Commit 0ffd9210 authored by Uwe Jandt (DESY, HIFIS)'s avatar Uwe Jandt (DESY, HIFIS)
Browse files

Merge branch 'backbone-aai-data-only' into 'master'

raw data only - remove plots, scripts and filtered data

See merge request hifis/overall/communication/service-usage-plots!20
parents bfa34ae2 05f496ff
#!/bin/gnuplot
set datafile separator ","
set output 'plot.pdf'
set yrange [0:*]
set xdata time
set timefmt "%Y-%m-%d %H:%M"
set format x "%Y-%m-%d"
set xrange ["2019-07-01":"2021-07-02"]
set grid
set xtics rotate by 45
set key top left
set terminal pdf color enhanced size 7,5 dashed
if (!exists("MP_LEFT")) MP_LEFT = .1
if (!exists("MP_RIGHT")) MP_RIGHT = .95
if (!exists("MP_BOTTOM")) MP_BOTTOM = .14
if (!exists("MP_TOP")) MP_TOP = .95
if (!exists("MP_xGAP")) MP_xGAP = 0.1
if (!exists("MP_yGAP")) MP_yGAP = 0.02
set multiplot layout 2,2 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
set linetype 2 dashtype 2
set key bottom left
set xtics textcolor rgbcolor "white"
set ylabel "{/Arial:Bold B2a}: AAI-connected centres \n with users in AAI"
set yrange [0:20]
plot '../stats/aai/aai_centers.csv' u 1:($2>0?$2:1/0) w lp pt 7 ps 0.5 lc black notitle ,\
'' u 1:($2<=11?$2:1/0) w lp pt 6 ps 0.5 lc black lt 2 notitle
set yrange [0:*]
set timefmt "%d.%m.%Y"
set xtics textcolor rgbcolor "black" offset -4,-3
set ylabel '{/Arial:Bold B2c}: Registered end users'
plot '../stats/aai/aai_users.csv' u 1:2 w lp pt 7 ps 0.5 lc black notitle
set timefmt "%d.%m.%y"
set xtics textcolor rgbcolor "white"
set ylabel "{/Arial:Bold B2b}: AAI-connected services \n (incl. \"associated services\")"
plot '../stats/aai/aai_services.csv' u 1:2 w lp pt 7 ps 0.5 lc black notitle
set key top left
set timefmt "%d.%m.%Y"
set xtics textcolor rgbcolor "black" offset -4,-3
set ylabel '{/Arial:Bold B2d}: Registered VOs (root)'
plot '../stats/aai/aai_vo.csv' u 1:2 w lp pt 6 ps 0.5 lt 2 lc black title 'VO - total' ,\
'' u 1:3 w lp pt 7 ps 0.5 lt 1 lc black title 'VO - HIFIS'
set timefmt "%Y-%m-%d %H:%M"
unset multiplot
set key bottom left
set ylabel '{/Arial:Bold B2a}: AAI-connected centres'
set xtics textcolor rgbcolor "black" offset -4,0
set yrange [0:20]
plot '../stats/aai/aai_centers.csv' u 1:2 w lp pt 7 ps 0.5 lc black title 'IdPs with users in AAI' ,\
'../stats/aai/aai_idp_attribute_query.csv' u 1:2 w lp pt 6 ps 0.5 lc black title 'IdPs supporting attribute query'
set logscale y
set yrange [10:4000]
set timefmt "%d.%m.%Y"
set ylabel '{/Arial:Bold B2c}: Registered end users'
plot '../stats/aai/aai_users.csv' u 1:2 w lp pt 7 ps 0.5 lc black notitle
unset logscale y
set yrange [0:*]
set timefmt "%Y-%m-%d %H:%M"
set ylabel 'B1: Centres connected to Backbone'
plot '../stats/vpn/vpn_centers.csv' u 1:2 w lp pt 7 ps 0.5 lc black notitle
set datafile separator ";"
set ylabel 'Gitlab Usage'
plot '../stats/gitlab/active_users.csv' u 1:2 w l title 'Total' ,\
'' u 1:4 w l title 'last 7 days' ,\
'' u 1:7 w l title 'last 60 days'
plot '../stats/gitlab/projects_statistics.csv' u 1:2 w l title 'Total'
#!/bin/bash
pushd ../stats/aai/
# summing up columns 2 and 3, combined
in=aai_userstats.csv
out=aai_users.csv
if [ $in -nt $out ]; then
cat $in | awk 'BEGIN{FS=","}{print $1","a; a+=$2+$3}' > $out
fi
# summing up columns 2,3,4 and 5
# leaving first (date) and columns after 5 (service name[s]) unchanged.
in=HIFIS_AAI_SP.csv
out=aai_services.csv
if [ $in -nt $out ]; then
cat $in | awk 'BEGIN{FS=","}{
printf ("%s,",$1);
for (a=2; a<=5; a++)
{
sum[a]+=$a;
printf ("%g,",sum[a]);
}
for (a=6; a<=NF; a++)
printf ("%s,",$a);
printf("\n");
}' > $out
fi
popd
gnuplot ./plot.plt
pdftoppm plot.pdf plot_temp -singlefile -aa no -aaVector no -r 600 -mono -png
convert plot_temp.png -monochrome plot.png
rm plot_temp.png
#!/bin/gnuplot
set datafile separator ","
set output './Report2020_plot1.pdf'
set yrange [0:*]
set xdata time
set timefmt "%Y-%m-%d %H:%M"
set format x "%Y-%m-%d"
set xrange ["2019-07-01":"2021-07-02"]
set grid
set xtics rotate by 45 offset -4,-3
set key top left
set terminal pdf color enhanced size 11,3
if (!exists("MP_LEFT")) MP_LEFT = 0.06
if (!exists("MP_RIGHT")) MP_RIGHT = .97
if (!exists("MP_BOTTOM")) MP_BOTTOM = .2
if (!exists("MP_TOP")) MP_TOP = .95
if (!exists("MP_xGAP")) MP_xGAP = 0.07
if (!exists("MP_yGAP")) MP_yGAP = 0.02
set multiplot layout 1,3 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
set timefmt "%d.%m.%Y"
set ylabel "Helmholtz AAI users"
plot '../stats/aai/aai_users.csv' u 1:2 w l lc black lw 2 title 'total'
set datafile separator ";"
set ylabel "Helmholtz Gitlab users\n(incl. local users)"
set timefmt "%Y-%m-%d %H:%M"
plot '../stats/gitlab/active_users.csv' u 1:2 w l lc black lw 2 title 'total' ,\
'' u 1:7 w l lc black lw 0.8 title 'last 60d'
set datafile separator ","
set ylabel "Course participant-hours"
set timefmt "%Y-%m"
set style fill pattern 2
plot '../stats/workshops/hifis_all_workshop_statistics.csv' u 1:3 w l lc black lw 2 title 'cumulative' ,\
'' u 1:2 w boxes lc black lw 0.8 title 'per month'
unset multiplot
#!/bin/bash
gnuplot plot.plt
for fn in *.pdf; do
pdftoppm $fn plot_temp -singlefile -aa no -aaVector no -r 600 -mono -png
convert plot_temp.png -monochrome ${fn%.pdf}.png
rm plot_temp.png
done
#!/bin/gnuplot
set datafile separator ","
set output './plot.pdf'
set yrange [0:*]
set xdata time
set timefmt "%Y-%m-%d %H:%M"
set format x "%Y-%m-%d"
set xrange ["2019-07-01":"2021-07-02"]
set xtics rotate by 45 offset -4,-3
set ytics nomirror
set key top left
set border 3
set terminal pdf color enhanced size 4,6 font "Helvetica, 16"
if (!exists("MP_LEFT")) MP_LEFT = 0.16
if (!exists("MP_RIGHT")) MP_RIGHT = .82
if (!exists("MP_BOTTOM")) MP_BOTTOM = .15
if (!exists("MP_TOP")) MP_TOP = .98
if (!exists("MP_xGAP")) MP_xGAP = 0.07
if (!exists("MP_yGAP")) MP_yGAP = 0.03
set multiplot layout 2,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
set timefmt "%d.%m.%Y"
set ylabel "Helmholtz AAI users" offset 1
set xtics textcolor rgbcolor "white" nomirror
plot '../stats/aai/aai_users.csv' u 1:2 w l lc black lw 2 notitle
set border 11
set datafile separator ","
set ylabel "Cumulative course participant-hours"
set y2label "Per-month participant-hours"
set y2tics
set xtics textcolor rgbcolor "black" offset -4,-4 nomirror
set timefmt "%Y-%m"
set style fill pattern 6
plot '../stats/workshops/hifis_all_workshop_statistics.csv' u 1:2 w boxes axis x1y2 lc rgb "#8CB423" lw 0.8 title 'per month' ,\
'' u 1:3 w l lc black lw 2 title 'cumulative'
unset multiplot
#!/bin/bash
gnuplot plot.plt
for fn in *.pdf; do
pdftoppm $fn plot_temp -singlefile -aa no -aaVector no -r 300 -png
convert plot_temp.png ${fn%.pdf}.png
rm plot_temp.png
done
# Service Usage Plots
# Service Usage
Collection of service usage for reporting with crude data plotting and conversion.
Collection of (some) service usage for reporting.
Plotting will be performed in separate project.
* Helmholtz Backbone (VPN)
* Helmholtz AAI
* Gitlab (redo from [here](https://gitlab.hzdr.de/fwcc-management/gitlab-statistics)), for the sake of completeness.
#### How-to
#### Old plots
Necessary for converting and plotting: `gnuplot`, `awk`, imagemagick `convert`.
```code
cd 202102_Backbone # or any other of the sub-folders
./plot.sh
```
#### Backbone Cluster: KPI plot
This is an update to the plot as provided for the [HIFIS Report 2020](https://nubes.helmholtz-berlin.de/s/Y79kf2s3xyiZnX2/download), Backbone Cluster.
![202102_Backbone](202102_Backbone/plot.png)
#### HIFIS General, KPI plot
This is an update to the plot as provided for the HIFIS Report 2020, Overall Section.
![202103_Report_Overall](202103_Report_Overall/plot.png)
#### AAI users vs. Education participants
This is an update to the plot as provided for the MV 202104.
![202104_MV](202104_MV/plot.png)
The old plots can still be accessed in branch [`backup-of-plot-data`](https://gitlab.hzdr.de/hifis/overall/communication/service-usage-plots/-/tree/backup-of-plot-data)
date,SP,HIFIS SP,Proxy,OIDC agent,commentar
24.10.18,1,0,0,0,FEUDAL
6.11.18,1,0,0,0,FEUDAL
12.12.18,1,1,0,0,OpenStack JSC
28.1.19,1,0,0,1,oidc agent
29.1.19,1,0,0,0,FEUDAL
28.2.19,1,0,0,1,oidc agent
6.3.19,1,0,0,1,oidc agent
14.5.19,1,0,0,0,Orpheus KIT
16.5.19,1,0,0,0,Prometheus DESY
25.6.19,1,0,0,0,FEUDAL
27.6.19,1,0,0,0,atmocat
11.9.19,1,0,0,1,oidc agent
12.11.19,2,0,0,1,Dcache view DESY; oidc agent
1.2.20,1,0,0,0,Dcache view DESY
4.2.20,1,0,0,1,oidc agent
6.2.20,1,0,0,0,Jülich Data
17.2.20,1,0,0,1,oidc agent
19.2.20,1,0,0,0,Jülich Data
25.2.20,1,0,0,1,oidc agent
26.3.20,30,0,0,0,Atmocat1-30
1.4.20,1,0,0,0,WaTTS
24.4.20,1,0,0,0,webdav KIT
7.5.20,1,1,0,0,HIFIS Software Helpdesk
11.5.20,1,1,0,0,HIFIS Marktplatz
13.5.20,1,0,0,0,SonarQube DESY
18.5.20,1,1,0,0,HIFIS Helpdesk
20.5.20,1,1,0,0,Gitlab HZDR
9.6.20,1,0,0,0,webdav KIT
25.6.20,1,0,0,0,Mailping KIT
26.6.20,1,1,0,0,HIFIS Marktplatz
20.7.20,1,1,0,0,FTS DESY
22.7.20,2,0,1,1,unity JSC; oidc agent
23.7.20,1,0,0,1,oidc agent
27.7.20,1,1,0,0,Nubes dev
3.8.20,1,0,0,0,arpoc
5.8.20,1,0,0,1,oidc agent
1.9.20,2,1,0,0,FTS EGI (HIFIS); Gitlab DESY
2.9.20,1,1,0,0,FTS Cern
3.9.20,1,1,0,0,JuChat
8.9.20,1,0,1,0,Keycloak DESY
6.10.20,1,0,0,0,Helmholtz AI Voucher
16.10.20,1,0,0,0,Helmholtz AI Voucher OIDC
4.11.20,2,1,0,0,nubes (HIFIS); Gitlab DESY
5.11.20,1,0,0,0,HIP Modalities
30.11.20,3,0,1,0,HIP Modalities Development; Gitlab GFZ enterprise; AWI Gateway OIDC
2.12.20,1,0,1,0,AWI login Gateway
3.12.20,1,0,1,0,Keycloak DESY
4.12.20,1,0,0,0,DZD cloud HMGU
16.12.20,1,1,0,0,FTS HZDR
17.12.20,1,1,0,0,OpenStack DKFZ
8.1.21,1,0,0,0,Gitlab GFZ community
2.2.21,2,0,0,0,CarrerCheck DKFZ; CarrerCheck DKFZ dev
4.2.21,1,0,0,0,Helmholtz DataHub Observatory at GFZ
Date,
01.11.2015,0
23.02.2018,0
27.02.2018,1
14.03.2018,2
15.03.2018,3
24.10.2018,4
29.10.2018,6
31.10.2018,8
06.11.2018,9
30.11.2018,10
12.12.2018,11
07.01.2019,12
14.01.2019,13
15.01.2019,14
18.01.2019,15
28.01.2019,16
29.01.2019,17
01.02.2019,18
02.02.2019,19
20.02.2019,20
28.02.2019,22
06.03.2019,23
25.03.2019,24
15.04.2019,26
29.04.2019,27
30.04.2019,28
06.05.2019,29
14.05.2019,30
16.05.2019,33
17.05.2019,35
20.05.2019,36
22.05.2019,37
27.05.2019,38
28.05.2019,39
17.06.2019,40
25.06.2019,42
27.06.2019,43
10.07.2019,44
24.07.2019,47
02.08.2019,48
02.09.2019,49
11.09.2019,50
19.09.2019,51
14.10.2019,52
22.10.2019,53
23.10.2019,54
05.11.2019,55
07.11.2019,56
12.11.2019,57
18.11.2019,59
19.11.2019,60
21.11.2019,61
22.11.2019,62
26.11.2019,63
02.01.2020,64
06.01.2020,65
07.01.2020,66
08.01.2020,67
14.01.2020,68
23.01.2020,69
24.01.2020,70
28.01.2020,71
29.01.2020,74
30.01.2020,75
01.02.2020,76
04.02.2020,77
06.02.2020,79
11.02.2020,80
12.02.2020,82
13.02.2020,84
14.02.2020,85
17.02.2020,87
19.02.2020,88
20.02.2020,90
21.02.2020,92
25.02.2020,94
26.02.2020,97
27.02.2020,99
01.03.2020,101
02.03.2020,102
03.03.2020,103
04.03.2020,104
05.03.2020,105
09.03.2020,106
16.03.2020,107
26.03.2020,108
27.03.2020,138
30.03.2020,139
01.04.2020,140
03.04.2020,142
06.04.2020,143
15.04.2020,145
17.04.2020,146
21.04.2020,147
24.04.2020,148
27.04.2020,149
28.04.2020,150
02.05.2020,151
06.05.2020,152
07.05.2020,154
08.05.2020,159
10.05.2020,160
11.05.2020,161
13.05.2020,164
14.05.2020,165
18.05.2020,166
19.05.2020,167
20.05.2020,172
22.05.2020,173
26.05.2020,174
27.05.2020,176
28.05.2020,177
03.06.2020,181
04.06.2020,182
05.06.2020,183
08.06.2020,184
09.06.2020,186
10.06.2020,192
11.06.2020,195
15.06.2020,197
16.06.2020,202
17.06.2020,207
18.06.2020,212
22.06.2020,214
23.06.2020,217
24.06.2020,222
25.06.2020,223
26.06.2020,226
28.06.2020,230
29.06.2020,231
30.06.2020,232
01.07.2020,235
03.07.2020,236
06.07.2020,239
07.07.2020,240
08.07.2020,243
10.07.2020,245
13.07.2020,247
14.07.2020,249
15.07.2020,251
16.07.2020,254
20.07.2020,257
22.07.2020,259
23.07.2020,261
24.07.2020,266
26.07.2020,267
27.07.2020,268
28.07.2020,269
29.07.2020,270
30.07.2020,271
31.07.2020,272
03.08.2020,274
04.08.2020,276
05.08.2020,277
06.08.2020,279
07.08.2020,282
10.08.2020,284
11.08.2020,287
12.08.2020,294
13.08.2020,298
14.08.2020,303
17.08.2020,312
18.08.2020,314
19.08.2020,317
20.08.2020,319
25.08.2020,322
26.08.2020,326
27.08.2020,329
28.08.2020,330
31.08.2020,332
01.09.2020,334
02.09.2020,337
03.09.2020,338
04.09.2020,339
07.09.2020,340
08.09.2020,343
09.09.2020,347
11.09.2020,352
15.09.2020,353
16.09.2020,355
17.09.2020,356
18.09.2020,358
21.09.2020,359
22.09.2020,362
23.09.2020,367
24.09.2020,368
25.09.2020,373
28.09.2020,374
29.09.2020,379
30.09.2020,381
01.10.2020,391
02.10.2020,393
05.10.2020,396
07.10.2020,399
08.10.2020,401
09.10.2020,402
10.10.2020,406
12.10.2020,407
13.10.2020,410
14.10.2020,412
15.10.2020,415
16.10.2020,418
19.10.2020,420
20.10.2020,424
21.10.2020,425
22.10.2020,426
26.10.2020,429
27.10.2020,432
28.10.2020,434
29.10.2020,440
30.10.2020,443
31.10.2020,445
02.11.2020,446
03.11.2020,449
04.11.2020,452
05.11.2020,459
06.11.2020,467
09.11.2020,469
10.11.2020,470
11.11.2020,472
12.11.2020,475
13.11.2020,478
16.11.2020,482
18.11.2020,484
19.11.2020,486
20.11.2020,488
23.11.2020,491
24.11.2020,492
25.11.2020,498
26.11.2020,513
27.11.2020,519
29.11.2020,523
30.11.2020,524
01.12.2020,537
02.12.2020,564
03.12.2020,589
04.12.2020,617
07.12.2020,635
08.12.2020,644
09.12.2020,675
10.12.2020,687
11.12.2020,698
14.12.2020,703
15.12.2020,715
16.12.2020,728
17.12.2020,736
18.12.2020,751
21.12.2020,758
22.12.2020,761
04.01.2021,762
05.01.2021,770
06.01.2021,784
07.01.2021,791
08.01.2021,796
10.01.2021,803
11.01.2021,804
12.01.2021,812
13.01.2021,819
14.01.2021,824
15.01.2021,846
17.01.2021,861
18.01.2021,862
19.01.2021,896
20.01.2021,925
21.01.2021,953
22.01.2021,969
25.01.2021,987
26.01.2021,1005
27.01.2021,1016
28.01.2021,1032
29.01.2021,1045
30.01.2021,1053
01.02.2021,1054
02.02.2021,1078
03.02.2021,1093
04.02.2021,1106
05.02.2021,1128
06.02.2021,1144
07.02.2021,1146
08.02.2021,1147
09.02.2021,1162
10.02.2021,1173
11.02.2021,1193
12.02.2021,1215
13.02.2021,1233
14.02.2021,1234
15.02.2021,1235
16.02.2021,1250
17.02.2021,1263
18.02.2021,1284
19.02.2021,1300
22.02.2021,1317
23.02.2021,1326
24.02.2021,1334
25.02.2021,1347
26.02.2021,1366