129 lines
2.9 KiB
Bash
Executable File
129 lines
2.9 KiB
Bash
Executable File
#!/bin/sh
|
|
echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
|
|
|
|
RootDir="${HOME}/prod"
|
|
|
|
export PYTHONPATH=${RootDir}
|
|
|
|
|
|
StatusChannel=Status-CVTT
|
|
AlertChannel=Alerts-CVTT
|
|
Sender=${RootDir}/ops/utils/send_mmost.sh
|
|
|
|
# Sender=cat
|
|
# StatusChannel=
|
|
|
|
Hosts=
|
|
Hosts="${Hosts} cloud11.cvtt.vpn"
|
|
Hosts="${Hosts} cloud15.cvtt.vpn"
|
|
Hosts="${Hosts} cloud16.cvtt.vpn"
|
|
Hosts="${Hosts} cloud17.cvtt.vpn"
|
|
Hosts="${Hosts} cloud21.cvtt.vpn"
|
|
Hosts="${Hosts} cloud22.cryptovaltrading.com"
|
|
|
|
Hosts="${Hosts} cvttdata.cvtt.vpn"
|
|
Hosts="${Hosts} cryptoval2.cvtt.vpn"
|
|
Hosts="${Hosts} cryptoval3.cvtt.vpn"
|
|
|
|
Hosts="${Hosts} homestore.cvtt.vpn"
|
|
Hosts="${Hosts} nsbackup.sheynin.home"
|
|
Hosts="${Hosts} dtvmhost.cvtt.vpn"
|
|
Hosts="${Hosts} ops-server.cvtt.vpn"
|
|
|
|
Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
|
|
Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
|
|
Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
|
|
|
|
Metrics=()
|
|
TempFiles=
|
|
|
|
function cleanup {
|
|
if [ "" != "${TempFiles}" ]; then
|
|
echo rm -f ${TempFiles}
|
|
rm -f ${TempFiles}
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
|
|
function new_tempfile {
|
|
tmpfile=$(mktemp)
|
|
TempFiles="${TempFiles} ${tmpfile}"
|
|
echo ${tmpfile}
|
|
}
|
|
|
|
function space_alert() {
|
|
ALERT_USAGE=75%
|
|
for metric in "${Metrics[@]}"
|
|
do
|
|
IFS=$' '; args=($metric); unset IFS
|
|
host=${args[0]}
|
|
fs=${args[1]}
|
|
space_used=${args[2]}
|
|
|
|
if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
|
|
echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function storage_check() {
|
|
|
|
result_lines=()
|
|
|
|
declare -a SingleMeas
|
|
for host in ${Hosts}
|
|
do
|
|
if [[ "${host}" == *"cryptovaltrading.com" ]]; then
|
|
port=7822
|
|
else
|
|
port=22
|
|
fi
|
|
Cmd="ssh -p ${port} $host"
|
|
Cmd="${Cmd} eval \"df -hTl"
|
|
Cmd="${Cmd} -x squashfs"
|
|
Cmd="${Cmd} | grep -v tmpfs"
|
|
Cmd="${Cmd} | grep -v Filesystem\""
|
|
|
|
IFS=$'\n' ; lines=$(eval ${Cmd})
|
|
for ln in $lines
|
|
do
|
|
IFS=$' '; args=($ln); unset IFS
|
|
res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |"
|
|
|
|
result_lines+=("$res")
|
|
|
|
fs=${args[6]}
|
|
used=${args[5]}
|
|
Metrics+=("$host $fs $used")
|
|
done
|
|
unset IFS
|
|
done
|
|
for ln in "${result_lines[@]}"
|
|
do
|
|
echo "${ln}"
|
|
done
|
|
}
|
|
|
|
tmpfile=$(new_tempfile)
|
|
tmpfile2=$(new_tempfile)
|
|
storage_check > ${tmpfile2}
|
|
|
|
echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
|
|
echo >> ${tmpfile}
|
|
echo "| host | filesystem | usage |" >> ${tmpfile}
|
|
echo "| --- | --- | --- |" >> ${tmpfile}
|
|
cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
|
|
cat ${tmpfile} | ${Sender} ${StatusChannel}
|
|
|
|
Measurements=()
|
|
tmpfile=$(new_tempfile)
|
|
space_alert > ${tmpfile}
|
|
if [ -s ${tmpfile} ]
|
|
then
|
|
(echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel}
|
|
else
|
|
echo "No Storage Alerts"
|
|
fi
|
|
|