moved/initial

This commit is contained in:
Cryptoval Trading Technologies 2025-05-19 19:42:49 +00:00
commit 5e9b704ae1
6 changed files with 474 additions and 0 deletions

79
healthcheck/hosts_check.sh Executable file
View File

@ -0,0 +1,79 @@
#!/bin/bash
# "cryptovaltrading.com": {
# "cloud18": {
# "users": ["oleg"],
# "type": "cloud",
# "ssh_port": 7822,
# "to_check": "false"
# "timeout_sec": 5
# },
RootDir="${HOME}/prod"
AlertChannel=Alerts-CVTT
Sender=${RootDir}/ops/utils/send_mmost.sh
ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts
HOSTS_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
get_domains() {
echo ${HOSTS_CONFIG} | jq -r '. | keys[]'
}
get_user_hosts() {
local User=${1}
local Domain=${2}
jdcmd="jq -r --arg domain \"$Domain\""
jdcmd="$jdcmd --arg usr \"$User\""
jdcmd="$jdcmd '.[\$domain]"
jdcmd="$jdcmd | to_entries[]"
jdcmd="$jdcmd | select(.value.users[]"
jdcmd="$jdcmd | contains(\$usr))"
jdcmd="$jdcmd | .key'"
echo ${HOSTS_CONFIG} | eval ${jdcmd} |sed "s/$/.$Domain/" # >&2
}
function host_alert() {
alert=${1}
if [ "${alert}" != "" ]
then
echo -e "### :fire: HOST ALERT \n${alert}" | ${Sender} ${AlertChannel}
fi
}
User=oleg
Hosts=()
DEFAULT_SSH_PORT=22
DEFAULT_TIMEOUT=5
DEFAULT_TO_CHECK="true"
Domains=("${Domains[@]}" "$(get_domains)")
for Domain in ${Domains[@]} ; do
Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
done
for Host in ${Hosts[@]} ; do
host=$(echo $Host | cut -d'.' -f1)
Domain=$(echo $Host | cut -d'.' -f2-)
PortSSH=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].ssh_port // '"$DEFAULT_SSH_PORT"'')
Timeout=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].timeout_sec // '"$DEFAULT_TIMEOUT"'')
ToCheck=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].to_check // '"$DEFAULT_TO_CHECK"'')
to_check="${ToCheck^^}"
if [ "${to_check}" == "TRUE" -o "${to_check}" == "YES" -o "${to_check}" == "Y" -o "${to_check}" == "T" ] ; then
echo "Checking host: $Host on port $PortSSH"
else
continue
fi
# Use nc to check if the specified port is open
if ! nc -z -w ${Timeout} "$Host" "$PortSSH"; then
echo "Host $Host is not available on port $PortSSH"
host_alert "Host $Host is not available on port $PortSSH"
fi
done

View File

@ -0,0 +1,60 @@
#!/usr/bin/env bash
usage() {
echo -n "Usage: ${0}"
echo
exit 1
}
RootDir="${HOME}/prod"
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
export PYTHONPATH=${RootDir}
StatusChannel="MD-Status"
Sender=${RootDir}/ops/utils/send_mmost.sh
# ----- For DEBUGGING
# Sender=cat
# StatusChannel=
run_checklist() {
yr=$(date -d 'yesterday' '+%Y')
mn=$(date -d 'yesterday' '+%m')
declare -A Commands
Commands=(
["hs01:cloud24"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud24/${yr}/${mn} | tail -5"
["hs01:cloud28"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud28/${yr}/${mn} | tail -5"
["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim/ | tail -5"
["cloud21:cloud24"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud24/${yr}/${mn} | tail -5"
["cloud21:cloud28"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud28/${yr}/${mn} | tail -5"
["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim | tail -5"
["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
)
for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
Cmd=${Commands[${name}]}
echo "------- ${name}"
echo ${Cmd}
eval ${Cmd}
done
}
tmpfile=$(mktemp)
function cleanup {
cd ${HOME}
rm -f ${tmpfile}
}
trap cleanup EXIT
echo "## :hearts: CRYPTO MD HEALTH CHECK (cvtt-md.cvtt.vpn)" >> ${tmpfile}
echo '```' >> ${tmpfile}
run_checklist >> ${tmpfile}
echo '```' >> ${tmpfile}
cat ${tmpfile} | ${Sender} ${StatusChannel}
cat $tmpfile

View File

@ -0,0 +1,73 @@
#!/usr/bin/env bash
usage() {
echo -n "Usage: ${0}"
echo
exit 1
}
RootDir="${HOME}/prod"
export PYTHONPATH=${RootDir}
StatusChannel="MD-Status"
Sender=${RootDir}/ops/utils/send_mmost.sh
# ----- For DEBUGGING
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
# Sender=cat #### D E B U G
# StatusChannel= #### D E B U G
run_checklist() {
yr=$(date -d 'yesterday' '+%Y')
CheckSymbols="A/AAPL N/NVDA M/META"
declare -A Commands
Commands["hs01"]=""
for sym in ${CheckSymbols}; do
Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
done
Commands["hs01"]+="echo"
Commands["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5"
Commands["cloud21"]=""
for sym in ${CheckSymbols}; do
Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
done
Commands["cloud21"]+="echo"
Commands["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5"
Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
echo "------- ${name}"
Cmd=${Commands[${name}]}
echo ${Cmd}
eval ${Cmd}
done
}
tmpfile=$(mktemp)
function cleanup {
cd ${HOME}
rm -f ${tmpfile}
}
trap cleanup EXIT
# run_checklist
tmpfile=$(mktemp)
function cleanup {
cd ${HOME}
rm -f ${tmpfile}
}
trap cleanup EXIT
echo "## :fire: EQUITY MD HEALTH CHECK (cryptoval4)" >> ${tmpfile}
echo '```' | tee -a ${tmpfile}
run_checklist | tee -a ${tmpfile}
echo '```' | tee -a ${tmpfile}
echo "Sending result to ${Sender} ${StatusChannel}"
cat ${tmpfile} | ${Sender} ${StatusChannel}

58
healthcheck/services_check.sh Executable file
View File

@ -0,0 +1,58 @@
#!/bin/bash
# "Jenkins" {
# "to_check": "Yes",
# "protocol": "http",
# "host": "cvtt-build.cvtt.vpn",
# "port": 8080
# },
RootDir="${HOME}/prod"
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
AlertChannel=Alerts-CVTT
Sender=${RootDir}/ops/utils/send_mmost.sh
ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_services
SERVICES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
echo $Sender
echo $AlertChannel
function service_alert() {
alert="${1}"
if [ "${alert}" != "" ]
then
#it may contain quotes
alert=$(echo "${alert}" | sed 's/"/\\"/g')
echo -e "### :boom: SERVICE ALERT\n${alert}" | ${Sender} ${AlertChannel}
fi
}
User=oleg
Hosts=()
DEFAULT_TO_CHECK=Yes
mapfile -t SvcNames < <(echo ${SERVICES_CONFIG} | jq -r '. | keys[]')
DEFAULT_TO_CHECK=Yes
for SvcName in "${SvcNames[@]}" ; do
ToCheck=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].to_check // "Yes"')
if [ "${ToCheck^^}" == "NO" ]; then
continue
fi
Host=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].host ')
Port=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].port ')
echo "Checking \"$SvcName\" (${Host}:${Port})"
# Use nc to check if the specified port is open
if ! nc -z -w5 "$Host" "$Port"; then
msg="Service \"${SvcName}\" (${Host}:${Port}) is not available"
echo ${msg}
service_alert "${msg}"
fi
done

View File

@ -0,0 +1,134 @@
#!/bin/bash
echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
RootDir="${HOME}/prod"
export PYTHONPATH=${RootDir}
StatusChannel=Status-CVTT
AlertChannel=Alerts-CVTT
Sender=${RootDir}/ops/utils/send_mmost.sh
# ----- For DEBUGGING
# Sender=cat
# StatusChannel=
get_user_hosts() {
local User=${1}
local Domain=${2}
Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts"
Cmd+=" | ${HOME}/bin/hjson -j"
Cmd+=" | jq -r"
Cmd+=" --arg domain \"${Domain}\""
Cmd+=" --arg usr \"${User}\""
Cmd+=" '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
Cmd+=" | sed 's/\$/.${Domain}/'"
eval ${Cmd}
}
function cleanup {
echo Cleaing up temporary files: ${TempFiles}
if [ "" != "${TempFiles}" ]; then
rm -f ${TempFiles}
fi
}
trap cleanup EXIT
function space_alert() {
ALERT_USAGE=75%
for metric in "${Metrics[@]}"
do
IFS=$' '; args=($metric); unset IFS
host=${args[0]}
fs=${args[1]}
space_used=${args[2]}
if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
fi
done
}
function storage_check() {
local Hosts=("${@}")
result_lines=()
declare -a SingleMeas
for host in ${Hosts[@]}
do
echo "storage_check host=${host}" >&2
if [[ "${host}" == *"cryptovaltrading.com" ]]; then
port=7822
else
port=22
fi
Cmd="ssh -p ${port}"
Cmd+=" -o StrictHostKeyChecking=no"
Cmd+=" -o UserKnownHostsFile=/dev/null"
Cmd+=" $host"
Cmd+=" eval \"df -hTl"
Cmd+=" -x squashfs"
Cmd+=" -x tmpfs"
Cmd+=" -x vfat"
Cmd+=" -x devtmpfs"
Cmd+=" | grep -v Filesystem\""
IFS=$'\n' ; lines=$(eval ${Cmd})
for ln in $lines
do
IFS=$' '; args=($ln); unset IFS
res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |"
result_lines+=("$res")
fs=${args[6]}
used=${args[5]}
Metrics+=("$host $fs $used")
done
unset IFS
done
for ln in "${result_lines[@]}"
do
echo "${ln}"
done
}
User=oleg
Metrics=()
TempFiles=
Hosts=()
for Domain in cvtt.vpn cryptovaltrading.com ; do
Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
done
tmpfile=$(mktemp)
TempFiles="${TempFiles} ${tmpfile}"
tmpfile2=$(mktemp)
TempFiles="${TempFiles} ${tmpfile2}"
storage_check "${Hosts[@]}" > ${tmpfile2}
echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
echo >> ${tmpfile}
echo "| host | filesystem | usage |" >> ${tmpfile}
echo "| --- | --- | --- |" >> ${tmpfile}
cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
cat ${tmpfile} | ${Sender} ${StatusChannel}
tmpfile=$(mktemp)
TempFiles="${TempFiles} ${tmpfile}"
space_alert > ${tmpfile}
if [ -s ${tmpfile} ]
then
(echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel}
else
echo "No Storage Alerts"
fi

70
save_cronjobs.sh Executable file
View File

@ -0,0 +1,70 @@
#!/bin/bash
usage() {
echo "Usage: ${0} <local_root_dir>"
exit 1
}
get_user_hosts() {
local User=${1}
local Domain=${2}
Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts"
Cmd="${Cmd} | ${HOME}/bin/hjson -j"
Cmd="${Cmd} | jq -r"
Cmd="${Cmd} --arg domain \"${Domain}\""
Cmd="${Cmd} --arg usr \"${User}\""
Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
# echo $Cmd ### - FOR DEBUG ONLY
eval ${Cmd}
}
Domain=cvtt.vpn
echo "Started ${0} ${*} ..."
RootDir=${1}
if [ ! -d "${RootDir}" ]; then
usage
fi
cd ${RootDir}
Cmd="git pull"
echo ${Cmd} && eval ${Cmd}
for User in cvtt oleg
do
echo "User=$User"
Cmd="mkdir -p ${RootDir}/${User}"
echo ${Cmd} && eval ${Cmd}
hosts=$(get_user_hosts ${User} ${Domain})
echo ${hosts}
for host in ${hosts}
do
echo "===================="
echo "${host}:${User}"
Cmd="ssh ${User}@${host}.${Domain} 'crontab -l' | tee ${RootDir}/${User}/${host}.cron"
echo ${Cmd} && eval ${Cmd}
done
done
cd ${RootDir}
Cmd="find -type f -empty -delete -print"
echo ${Cmd} && eval ${Cmd}
Cmd="find -type d -empty -delete -print"
echo ${Cmd} && eval ${Cmd}
Cmd="git add ."
echo ${Cmd} && eval ${Cmd}
Cmd="git commit -a -m '$(date)'"
echo ${Cmd} && eval ${Cmd}
Cmd="git pushall"
echo ${Cmd} && eval ${Cmd}
echo "Done ${0} ${*}"