cleanup

.gitignore

+# plot files
 *.pdf
-users_*.csv
+
+# local data only, so far
+raw/
+plots/
+
+# data not necessarily to be reported to HIFIS
+stats/users_all.csv
+
+# faulty data so far, to be debugged
+stats/users_lastseen.csv

README.md

-# Desy Syncnshare
+# Desy Sync&Share
 
+The data reported here contains:
+
+- 'users.csv': number of registered users
+  - with log in via Helmholtz AAI (update interval: every new registration).
+- 'consumption.csv': consumed storage - number of files, stored data in MB
+  - for users logged in via Helmholtz AAI.
+
+Extraction of data from local reports, and pushing of the extracted/filtered data is done manually so far.

scripts/eval.sh

+#!/bin/bash
+
+# path to raw data file
+fn="raw/data-raw.csv"
+
+# extract registered aai users
+{
+    # header
+    echo "date-time,\"Users authenticated via Helmholtz AAI\""
+
+    # filter and sort data
+    cat "$fn" | awk 'BEGIN{FS=","}{
+        if ($5=="Database" && $1=="owncloud")
+            print $0
+    }' |
+    sort -t ',' -k 3 |
+    awk 'BEGIN{FS=","}{
+        count++;
+        print $3","count;
+    }'
+} > stats/users_aai.csv
+
+# extract ALL users
+{
+    # header
+    echo "date-time,\"Registered users\""
+
+    # filter and sort data
+    cat "$fn" | awk 'BEGIN{FS=","}{
+            print $0
+    }' |
+    sort -t ',' -k 3 |
+    awk 'BEGIN{FS=","}{
+        count++;
+        print $3","count;
+    }'
+} > stats/users_all.csv
+
+
+
+if [ ! -d plots ]; then mkdir -v plots; fi
+gnuplot scripts/plot.plt

stats/plot.plt → scripts/plot.plt

@@ -3,7 +3,7 @@
 set datafile separator ","
 
 set terminal pdf  color enhanced size 7,5 dashed
-set output 'plot.pdf'
+set output 'plots/plot.pdf'
 
 
 set xdata time
@@ -30,20 +30,12 @@ set key top left
 set linetype 2 dashtype 2
 
 set multiplot layout 1,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
-plot 'users_aai.csv' u 1:3 w l lc black notitle
+plot 'stats/users_aai.csv' u 1:2 w l lc black notitle
 unset multiplot
-
 set multiplot layout 1,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
-plot 'users_aai.csv' u 2:3 w l lc black notitle
+plot 'stats/users_all.csv' u 1:2 w l lc black notitle
 unset multiplot
-
-
 set multiplot layout 1,1 columnsfirst margins screen MP_LEFT, MP_RIGHT, MP_BOTTOM, MP_TOP spacing screen MP_xGAP, MP_yGAP
-set title 'Creation time vs last seen for AAI users'
-set xlabel 'Creation time'
-set ylabel 'Last seen'
-set ydata time
-set format y "%Y-%m-%d"
-set yrange ["2020-07-01":"2022-03-01"]
-plot 'users_aai.csv' u 1:2 notitle
+plot 'stats/users_aai.csv' u 1:2 w l lc black notitle ,\
+     'stats/users_all.csv' u 1:2 w l lc black notitle
 unset multiplot

stats/eval.sh

-#!/bin/bash
-
-# extract registered aai users
-cat data-raw.csv | awk 'BEGIN{FS=","}{
-    if ($5=="Database" && $1=="owncloud")
-        print $0
-}' |
-sort -t ',' -k 3 |
-awk 'BEGIN{FS=","}{
-    count++;
-    print $3","$4","count;
-}' > users_aai.csv
-
-gnuplot plot.plt