#!/bin/bash echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]' RootDir="${HOME}/prod" export PYTHONPATH=${RootDir} StatusChannel=Status-CVTT AlertChannel=Alerts-CVTT Sender=${RootDir}/ops/utils/send_mmost.sh # ----- For DEBUGGING # Sender=cat # StatusChannel= get_user_hosts() { local User=${1} local Domain=${2} Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts" Cmd+=" | ${HOME}/bin/hjson -j" Cmd+=" | jq -r" Cmd+=" --arg domain \"${Domain}\"" Cmd+=" --arg usr \"${User}\"" Cmd+=" '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'" Cmd+=" | sed 's/\$/.${Domain}/'" eval ${Cmd} } function cleanup { echo Cleaing up temporary files: ${TempFiles} if [ "" != "${TempFiles}" ]; then rm -f ${TempFiles} fi } trap cleanup EXIT function space_alert() { ALERT_USAGE=75% for metric in "${Metrics[@]}" do IFS=$' '; args=($metric); unset IFS host=${args[0]} fs=${args[1]} space_used=${args[2]} if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:" fi done } function storage_check() { local Hosts=("${@}") result_lines=() declare -a SingleMeas for host in ${Hosts[@]} do echo "storage_check host=${host}" >&2 if [[ "${host}" == *"cryptovaltrading.com" ]]; then port=7822 else port=22 fi Cmd="ssh -p ${port}" Cmd+=" -o StrictHostKeyChecking=no" Cmd+=" -o UserKnownHostsFile=/dev/null" Cmd+=" $host" Cmd+=" eval \"df -hTl" Cmd+=" -x squashfs" Cmd+=" -x tmpfs" Cmd+=" -x vfat" Cmd+=" -x devtmpfs" Cmd+=" | grep -v Filesystem\"" IFS=$'\n' ; lines=$(eval ${Cmd}) for ln in $lines do IFS=$' '; args=($ln); unset IFS res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |" result_lines+=("$res") fs=${args[6]} used=${args[5]} Metrics+=("$host $fs $used") done unset IFS done for ln in "${result_lines[@]}" do echo "${ln}" done } User=oleg Metrics=() TempFiles= Hosts=() for Domain in cvtt.vpn cryptovaltrading.com ; do Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})") done tmpfile=$(mktemp) TempFiles="${TempFiles} ${tmpfile}" tmpfile2=$(mktemp) TempFiles="${TempFiles} ${tmpfile2}" storage_check "${Hosts[@]}" > ${tmpfile2} echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile} echo >> ${tmpfile} echo "| host | filesystem | usage |" >> ${tmpfile} echo "| --- | --- | --- |" >> ${tmpfile} cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile} cat ${tmpfile} | ${Sender} ${StatusChannel} tmpfile=$(mktemp) TempFiles="${TempFiles} ${tmpfile}" space_alert > ${tmpfile} if [ -s ${tmpfile} ] then (echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel} else echo "No Storage Alerts" fi