archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Oleg Sheynin 2023-12-27 20:54:29 -05:00
parent aa7cf6f08c
commit c6198d310c
1 changed files with 53 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
scripts/healthcheck/storage_health_check.sh
View File

@ -8,11 +8,17 @@ export PYTHONPATH=${RootDir}
# Temporary # Temporary
MmSender= MmStatusSender=
MmSender="${MmSender} ${HOME}/.pyenv/python3.10-venv/bin/python3" MmStatusSender="${MmStatusSender} ${HOME}/.pyenv/python3.10-venv/bin/python3"
MmSender="${MmSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py" MmStatusSender="${MmStatusSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py"
MmSender="${MmSender} --channel=Status-CVTT" MmStatusSender="${MmStatusSender} --channel=Status-CVTT"
MmSender="${MmSender} --log_level=ERROR 2>&1 > /dev/null" MmStatusSender="${MmStatusSender} --log_level=ERROR"
MmAlertSender=
MmAlertSender="${MmAlertSender} ${HOME}/.pyenv/python3.10-venv/bin/python3"
MmAlertSender="${MmAlertSender} ${RootDir}/cvttpy/apps/utils/mmost_sender.py"
MmAlertSender="${MmAlertSender} --channel=Alerts-CVTT"
MmAlertSender="${MmAlertSender} --log_level=ERROR"
Hosts= Hosts=
Hosts="${Hosts} cloud11.cvtt.vpn" Hosts="${Hosts} cloud11.cvtt.vpn"
@ -29,45 +35,65 @@ Hosts="${Hosts} cryptoval2.cvtt.vpn"
Hosts="${Hosts} cryptoval3.cvtt.vpn" Hosts="${Hosts} cryptoval3.cvtt.vpn"
Hosts="${Hosts} homestore.cvtt.vpn" Hosts="${Hosts} homestore.cvtt.vpn"
Hosts="${Hosts} nsbackup.cvtt.vpn"
Hosts="${Hosts} dtvmhost.cvtt.vpn" Hosts="${Hosts} dtvmhost.cvtt.vpn"
Hosts="${Hosts} ops-server.cvtt.vpn"
# With Ports
# Hosts="${Hosts} cloud14.cryptovaltrading.com"
# Hosts="${Hosts} cloud18.cryptovaltrading.com"
ALERT_USAGE=75%
echo "### [$(date +'%Y-%m-%d %H:%M:%S')] Storage Health Check" function space_alert() {
function checkSpace() { ALERT_USAGE=75%
fs=$1 for ln in "${Measurements[@]}"
space_used=$2 do
IFS=$' '; args=($ln); unset IFS
if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then host=${args[0]}
echo ":bangbang: ALERT!!! Filesystem **${fs}** is using **${space_used}** :bangbang:" fs=${args[1]}
fi space_used=${args[2]}
if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
fi
done
} }
function storage_check() { function storage_check() {
echo "## :card_file_box: STORAGE HEALTH CHECK"
echo
echo "| host | filesystem | usage |"
echo "| --- | --- | --- |"
for host in ${Hosts} for host in ${Hosts}
do do
Cmd="ssh $host" Cmd="ssh $host"
Cmd="${Cmd} eval \"df -hTl -x squashfs | grep -v tmpfs | grep -v Filesystem\"" Cmd="${Cmd} eval \"df -hTl"
IFS=$'\n' ; lines=$(eval ${Cmd}) # | grep -v 'Use%') Cmd="${Cmd} -x squashfs"
Cmd="${Cmd} | grep -v tmpfs"
Cmd="${Cmd} | grep -v Filesystem\""
IFS=$'\n' ; lines=$(eval ${Cmd})
for ln in $lines for ln in $lines
do do
IFS=$' '; args=($ln); unset IFS IFS=$' '; args=($ln); unset IFS
echo "**${host}**:***${args[6]}*** *${args[5]}*" echo "| **${host}** | ***${args[6]}*** | *${args[5]}* |"
checkSpace "${host}:${args[6]}" ${args[5]} Measurements+=("${host} ${args[6]} ${args[5]}")
done done
unset IFS unset IFS
echo '-' echo "| | | |"
done done
} }
Cmd="storage_check 2>&1 | ${MmSender}" Measurements=()
echo ${Cmd} tmpfile=$(mktemp)
eval ${Cmd} storage_check > ${tmpfile}
cat ${tmpfile} | ${MmStatusSender}
tmpfile=$(mktemp)
space_alert > ${tmpfile}
cat ${tmpfile}
if [ -s ${tmpfile} ]
then
(echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${MmAlertSender}
else
echo File ${tmpfile} is empty
fi
rm ${tmpfile}