progress

scripts/healthcheck/storage_health_check.sh

@@ -8,11 +8,17 @@ export PYTHONPATH=${RootDir}
 
 
 # Temporary
-MmSender=
-MmSender="${MmSender} ${HOME}/.pyenv/python3.10-venv/bin/python3"
-MmSender="${MmSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py"
-MmSender="${MmSender} --channel=Status-CVTT"
-MmSender="${MmSender} --log_level=ERROR 2>&1 > /dev/null"
+MmStatusSender=
+MmStatusSender="${MmStatusSender} ${HOME}/.pyenv/python3.10-venv/bin/python3"
+MmStatusSender="${MmStatusSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py"
+MmStatusSender="${MmStatusSender} --channel=Status-CVTT"
+MmStatusSender="${MmStatusSender} --log_level=ERROR"
+
+MmAlertSender=
+MmAlertSender="${MmAlertSender} ${HOME}/.pyenv/python3.10-venv/bin/python3"
+MmAlertSender="${MmAlertSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py"
+MmAlertSender="${MmAlertSender} --channel=Alerts-CVTT"
+MmAlertSender="${MmAlertSender} --log_level=ERROR"
 
 Hosts=
 Hosts="${Hosts} cloud11.cvtt.vpn"
@@ -29,45 +35,65 @@ Hosts="${Hosts} cryptoval2.cvtt.vpn"
 Hosts="${Hosts} cryptoval3.cvtt.vpn"
 
 Hosts="${Hosts} homestore.cvtt.vpn"
+Hosts="${Hosts} nsbackup.cvtt.vpn"
 Hosts="${Hosts} dtvmhost.cvtt.vpn"
-
-# With Ports
-# Hosts="${Hosts} cloud14.cryptovaltrading.com"
-# Hosts="${Hosts} cloud18.cryptovaltrading.com"
+Hosts="${Hosts} ops-server.cvtt.vpn"
 
 
-ALERT_USAGE=75%
-
-echo "### [$(date +'%Y-%m-%d %H:%M:%S')] Storage Health Check"
-function checkSpace() {
-    fs=$1
-    space_used=$2
 
+function space_alert() {
+    ALERT_USAGE=75%
+    for ln in "${Measurements[@]}"
+    do
+        IFS=$' '; args=($ln); unset IFS
+        host=${args[0]}
+        fs=${args[1]}
+        space_used=${args[2]}
         if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
-        echo ":bangbang: ALERT!!! Filesystem **${fs}** is using **${space_used}** :bangbang:"
+            echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
         fi
+    done
 }
 
 function storage_check() {
 
+    echo "## :card_file_box: STORAGE HEALTH CHECK"
+    echo
+    echo "| host | filesystem | usage |"
+    echo "| --- | --- | --- |"
     for host in ${Hosts}
     do
         Cmd="ssh $host"
-        Cmd="${Cmd} eval \"df -hTl -x squashfs | grep -v tmpfs | grep -v Filesystem\""
-        IFS=$'\n' ; lines=$(eval ${Cmd}) # | grep -v 'Use%')
+        Cmd="${Cmd} eval \"df -hTl"
+        Cmd="${Cmd} -x squashfs"
+        Cmd="${Cmd} | grep -v tmpfs"
+        Cmd="${Cmd} | grep -v Filesystem\""
+
+        IFS=$'\n' ; lines=$(eval ${Cmd})
         for ln in $lines
         do
             IFS=$' '; args=($ln); unset IFS
-            echo "**${host}**:***${args[6]}***  *${args[5]}*"
-            checkSpace "${host}:${args[6]}" ${args[5]}
+            echo "| **${host}** | ***${args[6]}*** | *${args[5]}* |"
+            Measurements+=("${host} ${args[6]} ${args[5]}")
         done
         unset IFS
-        echo '-'
+        echo "| | | |"
     done
-
 }
 
-Cmd="storage_check 2>&1 | ${MmSender}"
-echo ${Cmd}
-eval ${Cmd}
+Measurements=()
+tmpfile=$(mktemp)
+storage_check > ${tmpfile}
+cat ${tmpfile} | ${MmStatusSender}
+
+tmpfile=$(mktemp)
+space_alert > ${tmpfile}
+cat ${tmpfile}
+if [ -s ${tmpfile} ]
+then
+    (echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${MmAlertSender}
+else
+    echo File ${tmpfile} is empty
+fi
+rm ${tmpfile}