moved/initial
This commit is contained in:
Cryptoval Trading Technologies
commit
5e9b704ae1
79
healthcheck/hosts_check.sh
Executable file
79
healthcheck/hosts_check.sh
Executable file
@ -0,0 +1,79 @@
|
||||
#!/bin/bash
|
||||
|
||||
# "cryptovaltrading.com": {
|
||||
# "cloud18": {
|
||||
# "users": ["oleg"],
|
||||
# "type": "cloud",
|
||||
# "ssh_port": 7822,
|
||||
# "to_check": "false"
|
||||
# "timeout_sec": 5
|
||||
# },
|
||||
|
||||
RootDir="${HOME}/prod"
|
||||
|
||||
AlertChannel=Alerts-CVTT
|
||||
Sender=${RootDir}/ops/utils/send_mmost.sh
|
||||
ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts
|
||||
|
||||
HOSTS_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
|
||||
|
||||
get_domains() {
|
||||
echo ${HOSTS_CONFIG} | jq -r '. | keys[]'
|
||||
}
|
||||
get_user_hosts() {
|
||||
local User=${1}
|
||||
local Domain=${2}
|
||||
|
||||
jdcmd="jq -r --arg domain \"$Domain\""
|
||||
jdcmd="$jdcmd --arg usr \"$User\""
|
||||
jdcmd="$jdcmd '.[\$domain]"
|
||||
jdcmd="$jdcmd | to_entries[]"
|
||||
jdcmd="$jdcmd | select(.value.users[]"
|
||||
jdcmd="$jdcmd | contains(\$usr))"
|
||||
jdcmd="$jdcmd | .key'"
|
||||
echo ${HOSTS_CONFIG} | eval ${jdcmd} |sed "s/$/.$Domain/" # >&2
|
||||
}
|
||||
|
||||
function host_alert() {
|
||||
alert=${1}
|
||||
if [ "${alert}" != "" ]
|
||||
then
|
||||
echo -e "### :fire: HOST ALERT \n${alert}" | ${Sender} ${AlertChannel}
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
User=oleg
|
||||
Hosts=()
|
||||
DEFAULT_SSH_PORT=22
|
||||
DEFAULT_TIMEOUT=5
|
||||
DEFAULT_TO_CHECK="true"
|
||||
Domains=("${Domains[@]}" "$(get_domains)")
|
||||
|
||||
for Domain in ${Domains[@]} ; do
|
||||
Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
|
||||
done
|
||||
|
||||
for Host in ${Hosts[@]} ; do
|
||||
host=$(echo $Host | cut -d'.' -f1)
|
||||
Domain=$(echo $Host | cut -d'.' -f2-)
|
||||
|
||||
|
||||
PortSSH=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].ssh_port // '"$DEFAULT_SSH_PORT"'')
|
||||
|
||||
Timeout=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].timeout_sec // '"$DEFAULT_TIMEOUT"'')
|
||||
|
||||
ToCheck=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].to_check // '"$DEFAULT_TO_CHECK"'')
|
||||
to_check="${ToCheck^^}"
|
||||
if [ "${to_check}" == "TRUE" -o "${to_check}" == "YES" -o "${to_check}" == "Y" -o "${to_check}" == "T" ] ; then
|
||||
echo "Checking host: $Host on port $PortSSH"
|
||||
else
|
||||
continue
|
||||
fi
|
||||
|
||||
# Use nc to check if the specified port is open
|
||||
if ! nc -z -w ${Timeout} "$Host" "$PortSSH"; then
|
||||
echo "Host $Host is not available on port $PortSSH"
|
||||
host_alert "Host $Host is not available on port $PortSSH"
|
||||
fi
|
||||
done
|
||||
60
healthcheck/md/crypto_md_chcklst.sh
Executable file
60
healthcheck/md/crypto_md_chcklst.sh
Executable file
@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
usage() {
|
||||
echo -n "Usage: ${0}"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
RootDir="${HOME}/prod"
|
||||
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
|
||||
|
||||
export PYTHONPATH=${RootDir}
|
||||
|
||||
StatusChannel="MD-Status"
|
||||
Sender=${RootDir}/ops/utils/send_mmost.sh
|
||||
|
||||
# ----- For DEBUGGING
|
||||
# Sender=cat
|
||||
# StatusChannel=
|
||||
|
||||
run_checklist() {
|
||||
yr=$(date -d 'yesterday' '+%Y')
|
||||
mn=$(date -d 'yesterday' '+%m')
|
||||
|
||||
declare -A Commands
|
||||
Commands=(
|
||||
["hs01:cloud24"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud24/${yr}/${mn} | tail -5"
|
||||
["hs01:cloud28"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud28/${yr}/${mn} | tail -5"
|
||||
["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim/ | tail -5"
|
||||
["cloud21:cloud24"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud24/${yr}/${mn} | tail -5"
|
||||
["cloud21:cloud28"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud28/${yr}/${mn} | tail -5"
|
||||
["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim | tail -5"
|
||||
["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
|
||||
)
|
||||
|
||||
for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
|
||||
Cmd=${Commands[${name}]}
|
||||
echo "------- ${name}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
}
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
function cleanup {
|
||||
cd ${HOME}
|
||||
rm -f ${tmpfile}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "## :hearts: CRYPTO MD HEALTH CHECK (cvtt-md.cvtt.vpn)" >> ${tmpfile}
|
||||
echo '```' >> ${tmpfile}
|
||||
run_checklist >> ${tmpfile}
|
||||
echo '```' >> ${tmpfile}
|
||||
|
||||
cat ${tmpfile} | ${Sender} ${StatusChannel}
|
||||
|
||||
cat $tmpfile
|
||||
|
||||
73
healthcheck/md/eqty_md_chcklst.sh
Executable file
73
healthcheck/md/eqty_md_chcklst.sh
Executable file
@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
usage() {
|
||||
echo -n "Usage: ${0}"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
RootDir="${HOME}/prod"
|
||||
|
||||
export PYTHONPATH=${RootDir}
|
||||
|
||||
StatusChannel="MD-Status"
|
||||
Sender=${RootDir}/ops/utils/send_mmost.sh
|
||||
|
||||
# ----- For DEBUGGING
|
||||
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
|
||||
# Sender=cat #### D E B U G
|
||||
# StatusChannel= #### D E B U G
|
||||
|
||||
run_checklist() {
|
||||
|
||||
|
||||
yr=$(date -d 'yesterday' '+%Y')
|
||||
CheckSymbols="A/AAPL N/NVDA M/META"
|
||||
declare -A Commands
|
||||
Commands["hs01"]=""
|
||||
for sym in ${CheckSymbols}; do
|
||||
Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
|
||||
done
|
||||
Commands["hs01"]+="echo"
|
||||
Commands["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5"
|
||||
|
||||
Commands["cloud21"]=""
|
||||
for sym in ${CheckSymbols}; do
|
||||
Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
|
||||
done
|
||||
Commands["cloud21"]+="echo"
|
||||
Commands["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5"
|
||||
|
||||
Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
|
||||
|
||||
for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
|
||||
echo "------- ${name}"
|
||||
Cmd=${Commands[${name}]}
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
}
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
function cleanup {
|
||||
cd ${HOME}
|
||||
rm -f ${tmpfile}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# run_checklist
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
function cleanup {
|
||||
cd ${HOME}
|
||||
rm -f ${tmpfile}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "## :fire: EQUITY MD HEALTH CHECK (cryptoval4)" >> ${tmpfile}
|
||||
echo '```' | tee -a ${tmpfile}
|
||||
run_checklist | tee -a ${tmpfile}
|
||||
echo '```' | tee -a ${tmpfile}
|
||||
|
||||
echo "Sending result to ${Sender} ${StatusChannel}"
|
||||
cat ${tmpfile} | ${Sender} ${StatusChannel}
|
||||
58
healthcheck/services_check.sh
Executable file
58
healthcheck/services_check.sh
Executable file
@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
# "Jenkins" {
|
||||
# "to_check": "Yes",
|
||||
# "protocol": "http",
|
||||
# "host": "cvtt-build.cvtt.vpn",
|
||||
# "port": 8080
|
||||
# },
|
||||
|
||||
RootDir="${HOME}/prod"
|
||||
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
|
||||
|
||||
AlertChannel=Alerts-CVTT
|
||||
Sender=${RootDir}/ops/utils/send_mmost.sh
|
||||
ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_services
|
||||
|
||||
SERVICES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
|
||||
|
||||
echo $Sender
|
||||
echo $AlertChannel
|
||||
|
||||
|
||||
function service_alert() {
|
||||
alert="${1}"
|
||||
if [ "${alert}" != "" ]
|
||||
then
|
||||
#it may contain quotes
|
||||
alert=$(echo "${alert}" | sed 's/"/\\"/g')
|
||||
echo -e "### :boom: SERVICE ALERT\n${alert}" | ${Sender} ${AlertChannel}
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
User=oleg
|
||||
Hosts=()
|
||||
DEFAULT_TO_CHECK=Yes
|
||||
|
||||
mapfile -t SvcNames < <(echo ${SERVICES_CONFIG} | jq -r '. | keys[]')
|
||||
|
||||
DEFAULT_TO_CHECK=Yes
|
||||
for SvcName in "${SvcNames[@]}" ; do
|
||||
ToCheck=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].to_check // "Yes"')
|
||||
if [ "${ToCheck^^}" == "NO" ]; then
|
||||
continue
|
||||
fi
|
||||
Host=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].host ')
|
||||
Port=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].port ')
|
||||
echo "Checking \"$SvcName\" (${Host}:${Port})"
|
||||
|
||||
|
||||
# Use nc to check if the specified port is open
|
||||
if ! nc -z -w5 "$Host" "$Port"; then
|
||||
msg="Service \"${SvcName}\" (${Host}:${Port}) is not available"
|
||||
echo ${msg}
|
||||
service_alert "${msg}"
|
||||
fi
|
||||
done
|
||||
|
||||
134
healthcheck/storage_health_check.sh
Executable file
134
healthcheck/storage_health_check.sh
Executable file
@ -0,0 +1,134 @@
|
||||
#!/bin/bash
|
||||
echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
|
||||
|
||||
RootDir="${HOME}/prod"
|
||||
|
||||
export PYTHONPATH=${RootDir}
|
||||
|
||||
|
||||
StatusChannel=Status-CVTT
|
||||
AlertChannel=Alerts-CVTT
|
||||
Sender=${RootDir}/ops/utils/send_mmost.sh
|
||||
|
||||
# ----- For DEBUGGING
|
||||
# Sender=cat
|
||||
# StatusChannel=
|
||||
|
||||
get_user_hosts() {
|
||||
local User=${1}
|
||||
local Domain=${2}
|
||||
|
||||
Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts"
|
||||
Cmd+=" | ${HOME}/bin/hjson -j"
|
||||
Cmd+=" | jq -r"
|
||||
Cmd+=" --arg domain \"${Domain}\""
|
||||
Cmd+=" --arg usr \"${User}\""
|
||||
Cmd+=" '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
|
||||
|
||||
Cmd+=" | sed 's/\$/.${Domain}/'"
|
||||
eval ${Cmd}
|
||||
}
|
||||
|
||||
function cleanup {
|
||||
echo Cleaing up temporary files: ${TempFiles}
|
||||
if [ "" != "${TempFiles}" ]; then
|
||||
rm -f ${TempFiles}
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
function space_alert() {
|
||||
ALERT_USAGE=75%
|
||||
for metric in "${Metrics[@]}"
|
||||
do
|
||||
IFS=$' '; args=($metric); unset IFS
|
||||
host=${args[0]}
|
||||
fs=${args[1]}
|
||||
space_used=${args[2]}
|
||||
|
||||
if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
|
||||
echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function storage_check() {
|
||||
local Hosts=("${@}")
|
||||
|
||||
result_lines=()
|
||||
|
||||
declare -a SingleMeas
|
||||
for host in ${Hosts[@]}
|
||||
do
|
||||
echo "storage_check host=${host}" >&2
|
||||
if [[ "${host}" == *"cryptovaltrading.com" ]]; then
|
||||
port=7822
|
||||
else
|
||||
port=22
|
||||
fi
|
||||
Cmd="ssh -p ${port}"
|
||||
Cmd+=" -o StrictHostKeyChecking=no"
|
||||
Cmd+=" -o UserKnownHostsFile=/dev/null"
|
||||
Cmd+=" $host"
|
||||
Cmd+=" eval \"df -hTl"
|
||||
Cmd+=" -x squashfs"
|
||||
Cmd+=" -x tmpfs"
|
||||
Cmd+=" -x vfat"
|
||||
Cmd+=" -x devtmpfs"
|
||||
Cmd+=" | grep -v Filesystem\""
|
||||
|
||||
IFS=$'\n' ; lines=$(eval ${Cmd})
|
||||
for ln in $lines
|
||||
do
|
||||
IFS=$' '; args=($ln); unset IFS
|
||||
res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |"
|
||||
|
||||
result_lines+=("$res")
|
||||
|
||||
fs=${args[6]}
|
||||
used=${args[5]}
|
||||
Metrics+=("$host $fs $used")
|
||||
done
|
||||
unset IFS
|
||||
done
|
||||
for ln in "${result_lines[@]}"
|
||||
do
|
||||
echo "${ln}"
|
||||
done
|
||||
}
|
||||
|
||||
User=oleg
|
||||
Metrics=()
|
||||
TempFiles=
|
||||
|
||||
Hosts=()
|
||||
for Domain in cvtt.vpn cryptovaltrading.com ; do
|
||||
Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
|
||||
done
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
TempFiles="${TempFiles} ${tmpfile}"
|
||||
|
||||
tmpfile2=$(mktemp)
|
||||
TempFiles="${TempFiles} ${tmpfile2}"
|
||||
|
||||
storage_check "${Hosts[@]}" > ${tmpfile2}
|
||||
|
||||
echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
|
||||
echo >> ${tmpfile}
|
||||
echo "| host | filesystem | usage |" >> ${tmpfile}
|
||||
echo "| --- | --- | --- |" >> ${tmpfile}
|
||||
cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
|
||||
cat ${tmpfile} | ${Sender} ${StatusChannel}
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
TempFiles="${TempFiles} ${tmpfile}"
|
||||
|
||||
space_alert > ${tmpfile}
|
||||
if [ -s ${tmpfile} ]
|
||||
then
|
||||
(echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel}
|
||||
else
|
||||
echo "No Storage Alerts"
|
||||
fi
|
||||
|
||||
70
save_cronjobs.sh
Executable file
70
save_cronjobs.sh
Executable file
@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: ${0} <local_root_dir>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
get_user_hosts() {
|
||||
local User=${1}
|
||||
local Domain=${2}
|
||||
|
||||
Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts"
|
||||
Cmd="${Cmd} | ${HOME}/bin/hjson -j"
|
||||
Cmd="${Cmd} | jq -r"
|
||||
Cmd="${Cmd} --arg domain \"${Domain}\""
|
||||
Cmd="${Cmd} --arg usr \"${User}\""
|
||||
Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
|
||||
# echo $Cmd ### - FOR DEBUG ONLY
|
||||
eval ${Cmd}
|
||||
}
|
||||
|
||||
Domain=cvtt.vpn
|
||||
echo "Started ${0} ${*} ..."
|
||||
|
||||
RootDir=${1}
|
||||
|
||||
if [ ! -d "${RootDir}" ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
cd ${RootDir}
|
||||
Cmd="git pull"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
for User in cvtt oleg
|
||||
do
|
||||
echo "User=$User"
|
||||
Cmd="mkdir -p ${RootDir}/${User}"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
hosts=$(get_user_hosts ${User} ${Domain})
|
||||
echo ${hosts}
|
||||
|
||||
for host in ${hosts}
|
||||
do
|
||||
echo "===================="
|
||||
echo "${host}:${User}"
|
||||
Cmd="ssh ${User}@${host}.${Domain} 'crontab -l' | tee ${RootDir}/${User}/${host}.cron"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
done
|
||||
done
|
||||
|
||||
cd ${RootDir}
|
||||
|
||||
Cmd="find -type f -empty -delete -print"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
Cmd="find -type d -empty -delete -print"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
Cmd="git add ."
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
Cmd="git commit -a -m '$(date)'"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
Cmd="git pushall"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
echo "Done ${0} ${*}"
|
||||
Loading…
x
Reference in New Issue
Block a user