little scripts

install-rdl-programs.sh

@@ -0,0 +1 @@
+install programs/* /usr/local/bin

programs/rdl_prometheus

@@ -64,3 +64,5 @@ allowed_characters='^[a-z_]+$'
 
 mv --force /tmp/"$NAME".prom.$$ "$PROM_DIR"/"$NAME".prom
 chown prometheus:prometheus "$PROM_DIR"/"$NAME".prom
+
+exit $RETURN

programs/rdl_restart_network

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+for N in {1..3}; do sudo ip link set dev eno$N down; done
+
+ip route delete default
+
+sleep 10
+
+for N in {1..3}; do sudo ip link set dev eno$N up; done
+
+ip route add default via 10.23.1.1 dev eno3

programs/rdl_ssh_touch

@@ -0,0 +1,44 @@
+#!/bin/bash
+# rdl_ssh_touch -- try and fix connection
+
+login=$1
+file=$2
+
+RDL_PROM_LINES+="
+job rdl_ssh_touch
+login $login
+file $file
+"
+
+try_connection() {
+    timeout 20 ssh "$login" touch "$file"
+}
+
+should_reset() {
+    last_run=$(< /var/rdl_ssh_touch/"$login")
+
+    if ! test "$last_run" ; then
+	echo Can\'t get a last timestamp in echo /var/rdl_ssh_touch/"$login"
+	mkdir -v /var/rdl_ssh_touch
+	return 0
+    else
+	# (time now) - (last time) > 6 hours?
+	if test $(( $(date +%s) - "$last_run" )) -gt $(( 60 * 60 * 6 )); then
+	    return 0
+	else
+	    echo Already resetted at $(date -d @"$last_run" +"%F %T")
+	    return 1
+	fi
+    fi
+}
+
+if ! rdl_prometheus try_connection; then
+    echo Can\'t ssh "$login" touch "$file".
+    if should_reset; then
+	echo Resetting...
+	date +%s > /var/rdl_ssh_touch/"$login" \
+	    && rdl_prometheus rdl_restart_network \
+	    && echo Ok
+    fi
+fi
+