diff --git a/scripts/healthcheck/storage_health_check.sh b/scripts/healthcheck/storage_health_check.sh index ba23c64..336b09e 100755 --- a/scripts/healthcheck/storage_health_check.sh +++ b/scripts/healthcheck/storage_health_check.sh @@ -8,11 +8,17 @@ export PYTHONPATH=${RootDir} # Temporary -MmSender= -MmSender="${MmSender} ${HOME}/.pyenv/python3.10-venv/bin/python3" -MmSender="${MmSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py" -MmSender="${MmSender} --channel=Status-CVTT" -MmSender="${MmSender} --log_level=ERROR 2>&1 > /dev/null" +MmStatusSender= +MmStatusSender="${MmStatusSender} ${HOME}/.pyenv/python3.10-venv/bin/python3" +MmStatusSender="${MmStatusSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py" +MmStatusSender="${MmStatusSender} --channel=Status-CVTT" +MmStatusSender="${MmStatusSender} --log_level=ERROR" + +MmAlertSender= +MmAlertSender="${MmAlertSender} ${HOME}/.pyenv/python3.10-venv/bin/python3" +MmAlertSender="${MmAlertSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py" +MmAlertSender="${MmAlertSender} --channel=Alerts-CVTT" +MmAlertSender="${MmAlertSender} --log_level=ERROR" Hosts= Hosts="${Hosts} cloud11.cvtt.vpn" @@ -29,45 +35,65 @@ Hosts="${Hosts} cryptoval2.cvtt.vpn" Hosts="${Hosts} cryptoval3.cvtt.vpn" Hosts="${Hosts} homestore.cvtt.vpn" +Hosts="${Hosts} nsbackup.cvtt.vpn" Hosts="${Hosts} dtvmhost.cvtt.vpn" - -# With Ports -# Hosts="${Hosts} cloud14.cryptovaltrading.com" -# Hosts="${Hosts} cloud18.cryptovaltrading.com" +Hosts="${Hosts} ops-server.cvtt.vpn" -ALERT_USAGE=75% -echo "### [$(date +'%Y-%m-%d %H:%M:%S')] Storage Health Check" -function checkSpace() { - fs=$1 - space_used=$2 - - if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then - echo ":bangbang: ALERT!!! Filesystem **${fs}** is using **${space_used}** :bangbang:" - fi +function space_alert() { + ALERT_USAGE=75% + for ln in "${Measurements[@]}" + do + IFS=$' '; args=($ln); unset IFS + host=${args[0]} + fs=${args[1]} + space_used=${args[2]} + if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then + echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:" + fi + done } function storage_check() { + echo "## :card_file_box: STORAGE HEALTH CHECK" + echo + echo "| host | filesystem | usage |" + echo "| --- | --- | --- |" for host in ${Hosts} do Cmd="ssh $host" - Cmd="${Cmd} eval \"df -hTl -x squashfs | grep -v tmpfs | grep -v Filesystem\"" - IFS=$'\n' ; lines=$(eval ${Cmd}) # | grep -v 'Use%') + Cmd="${Cmd} eval \"df -hTl" + Cmd="${Cmd} -x squashfs" + Cmd="${Cmd} | grep -v tmpfs" + Cmd="${Cmd} | grep -v Filesystem\"" + + IFS=$'\n' ; lines=$(eval ${Cmd}) for ln in $lines do IFS=$' '; args=($ln); unset IFS - echo "**${host}**:***${args[6]}*** *${args[5]}*" - checkSpace "${host}:${args[6]}" ${args[5]} + echo "| **${host}** | ***${args[6]}*** | *${args[5]}* |" + Measurements+=("${host} ${args[6]} ${args[5]}") done unset IFS - echo '-' + echo "| | | |" done - } -Cmd="storage_check 2>&1 | ${MmSender}" -echo ${Cmd} -eval ${Cmd} +Measurements=() +tmpfile=$(mktemp) +storage_check > ${tmpfile} +cat ${tmpfile} | ${MmStatusSender} + +tmpfile=$(mktemp) +space_alert > ${tmpfile} +cat ${tmpfile} +if [ -s ${tmpfile} ] +then + (echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${MmAlertSender} +else + echo File ${tmpfile} is empty +fi +rm ${tmpfile}