commit 5e9b704ae12ca976f27151411436c645d017b68d
Author: Cryptoval Trading Technologies <cryptoval@cryptovaltrading.com>
Date:   Mon May 19 19:42:49 2025 +0000

    moved/initial

diff --git a/healthcheck/hosts_check.sh b/healthcheck/hosts_check.sh
new file mode 100755
index 0000000..adf9324
--- /dev/null
+++ b/healthcheck/hosts_check.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# "cryptovaltrading.com": {
+#     "cloud18": {
+#         "users": ["oleg"],
+#         "type": "cloud",
+#         "ssh_port": 7822,
+#         "to_check": "false"
+#         "timeout_sec": 5
+#     },
+
+RootDir="${HOME}/prod"
+
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts
+
+HOSTS_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
+
+get_domains() {
+    echo ${HOSTS_CONFIG} | jq -r '. | keys[]'
+}
+get_user_hosts() {
+    local User=${1}
+    local Domain=${2}
+
+    jdcmd="jq -r --arg domain \"$Domain\""
+    jdcmd="$jdcmd --arg usr \"$User\""
+    jdcmd="$jdcmd '.[\$domain]"
+    jdcmd="$jdcmd | to_entries[]"
+    jdcmd="$jdcmd | select(.value.users[]"
+    jdcmd="$jdcmd | contains(\$usr))"
+    jdcmd="$jdcmd | .key'"
+    echo ${HOSTS_CONFIG} | eval ${jdcmd} |sed "s/$/.$Domain/" # >&2
+}
+
+function host_alert() {
+    alert=${1}
+    if [ "${alert}" != "" ]
+    then
+        echo -e "### :fire: HOST ALERT \n${alert}" | ${Sender} ${AlertChannel}
+    fi
+
+}
+
+User=oleg
+Hosts=()
+DEFAULT_SSH_PORT=22
+DEFAULT_TIMEOUT=5
+DEFAULT_TO_CHECK="true"
+Domains=("${Domains[@]}" "$(get_domains)")
+
+for Domain in ${Domains[@]} ; do
+    Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
+done
+
+for Host in ${Hosts[@]} ; do
+    host=$(echo $Host | cut -d'.' -f1)
+    Domain=$(echo $Host | cut -d'.' -f2-)
+
+
+    PortSSH=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].ssh_port // '"$DEFAULT_SSH_PORT"'')
+
+    Timeout=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].timeout_sec // '"$DEFAULT_TIMEOUT"'')
+
+    ToCheck=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].to_check // '"$DEFAULT_TO_CHECK"'')
+    to_check="${ToCheck^^}"
+    if [ "${to_check}" == "TRUE" -o "${to_check}" == "YES" -o "${to_check}" == "Y" -o "${to_check}" == "T" ] ; then
+        echo "Checking host: $Host on port $PortSSH"
+    else
+        continue
+    fi
+
+    # Use nc to check if the specified port is open
+    if ! nc -z -w ${Timeout} "$Host" "$PortSSH"; then
+        echo "Host $Host is not available on port $PortSSH"
+        host_alert "Host $Host is not available on port $PortSSH"
+    fi
+done
diff --git a/healthcheck/md/crypto_md_chcklst.sh b/healthcheck/md/crypto_md_chcklst.sh
new file mode 100755
index 0000000..becbf28
--- /dev/null
+++ b/healthcheck/md/crypto_md_chcklst.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+run_checklist() {
+    yr=$(date -d 'yesterday' '+%Y')
+    mn=$(date -d 'yesterday' '+%m')
+
+    declare -A Commands
+    Commands=(
+        ["hs01:cloud24"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud24/${yr}/${mn} | tail -5"
+        ["hs01:cloud28"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud28/${yr}/${mn} | tail -5"
+        ["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim/ | tail -5"
+        ["cloud21:cloud24"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud24/${yr}/${mn} | tail -5"
+        ["cloud21:cloud28"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud28/${yr}/${mn} | tail -5"
+        ["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim | tail -5"
+        ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
+    )
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        Cmd=${Commands[${name}]}
+        echo "------- ${name}"
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :hearts: CRYPTO MD HEALTH CHECK (cvtt-md.cvtt.vpn)" >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+run_checklist  >> ${tmpfile}
+echo  '```' >> ${tmpfile}
+
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+cat $tmpfile
+
diff --git a/healthcheck/md/eqty_md_chcklst.sh b/healthcheck/md/eqty_md_chcklst.sh
new file mode 100755
index 0000000..322bc0c
--- /dev/null
+++ b/healthcheck/md/eqty_md_chcklst.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+
+export PYTHONPATH=${RootDir}
+
+StatusChannel="MD-Status"
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+# Sender=cat #### D E B U G
+# StatusChannel=  #### D E B U G
+
+run_checklist() {
+
+
+    yr=$(date -d 'yesterday' '+%Y')
+    CheckSymbols="A/AAPL N/NVDA M/META"
+    declare -A Commands
+    Commands["hs01"]=""
+    for sym in ${CheckSymbols}; do
+        Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
+    done
+    Commands["hs01"]+="echo"
+    Commands["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" 
+
+    Commands["cloud21"]=""
+    for sym in ${CheckSymbols}; do
+        Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
+    done
+    Commands["cloud21"]+="echo"
+    Commands["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" 
+
+    Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
+
+    for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
+        echo "------- ${name}"
+        Cmd=${Commands[${name}]}
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+}
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+# run_checklist
+
+tmpfile=$(mktemp)
+function cleanup {
+    cd ${HOME}
+    rm -f ${tmpfile}
+}
+trap cleanup EXIT
+
+echo "## :fire: EQUITY MD HEALTH CHECK (cryptoval4)" >> ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
+run_checklist  | tee -a ${tmpfile}
+echo  '```' | tee -a ${tmpfile}
+
+echo "Sending result to ${Sender} ${StatusChannel}"
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
\ No newline at end of file
diff --git a/healthcheck/services_check.sh b/healthcheck/services_check.sh
new file mode 100755
index 0000000..30e73ea
--- /dev/null
+++ b/healthcheck/services_check.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+    # "Jenkins" {
+    #     "to_check": "Yes",
+    #     "protocol": "http",
+    #     "host": "cvtt-build.cvtt.vpn",
+    #     "port": 8080
+    # },
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+ConfigUrl=http://cloud16.cvtt.vpn:6789/admin/cvtt_services
+
+SERVICES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
+
+echo $Sender
+echo $AlertChannel
+
+
+function service_alert() {
+    alert="${1}"
+    if [ "${alert}" != "" ]
+    then
+        #it may contain quotes
+        alert=$(echo "${alert}" | sed 's/"/\\"/g')
+        echo -e "### :boom: SERVICE ALERT\n${alert}" | ${Sender} ${AlertChannel}
+    fi
+
+}
+
+User=oleg
+Hosts=()
+DEFAULT_TO_CHECK=Yes
+
+mapfile -t SvcNames < <(echo ${SERVICES_CONFIG} | jq -r '. | keys[]')
+
+DEFAULT_TO_CHECK=Yes
+for SvcName in "${SvcNames[@]}" ; do
+    ToCheck=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].to_check // "Yes"')
+    if [ "${ToCheck^^}" == "NO" ]; then
+        continue
+    fi
+    Host=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName"  '.[$svcname].host ')
+    Port=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName"  '.[$svcname].port ')
+    echo "Checking \"$SvcName\" (${Host}:${Port})"
+
+
+    # Use nc to check if the specified port is open
+    if ! nc -z -w5 "$Host" "$Port"; then
+        msg="Service \"${SvcName}\" (${Host}:${Port}) is not available"
+        echo ${msg}
+        service_alert "${msg}"
+    fi
+done
+
diff --git a/healthcheck/storage_health_check.sh b/healthcheck/storage_health_check.sh
new file mode 100755
index 0000000..8176bb1
--- /dev/null
+++ b/healthcheck/storage_health_check.sh
@@ -0,0 +1,134 @@
+#!/bin/bash
+echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
+
+RootDir="${HOME}/prod"
+
+export PYTHONPATH=${RootDir}
+
+
+StatusChannel=Status-CVTT
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+# ----- For DEBUGGING
+# Sender=cat
+# StatusChannel=
+
+get_user_hosts() {
+    local User=${1}
+    local Domain=${2}
+
+    Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts"
+    Cmd+=" | ${HOME}/bin/hjson -j"
+    Cmd+=" | jq -r"
+    Cmd+=" --arg domain \"${Domain}\""
+    Cmd+=" --arg usr \"${User}\""
+    Cmd+=" '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
+
+    Cmd+=" | sed  's/\$/.${Domain}/'"
+    eval ${Cmd}
+}
+
+function cleanup {
+    echo Cleaing up temporary files: ${TempFiles}
+    if [ "" != "${TempFiles}" ]; then
+        rm -f ${TempFiles}
+    fi
+}
+trap cleanup EXIT
+
+function space_alert() {
+    ALERT_USAGE=75%
+    for metric in "${Metrics[@]}"
+    do
+        IFS=$' '; args=($metric); unset IFS
+        host=${args[0]}
+        fs=${args[1]}
+        space_used=${args[2]}
+
+        if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
+            echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
+        fi
+    done
+}
+
+function storage_check() {
+    local Hosts=("${@}")
+
+    result_lines=()
+
+    declare -a SingleMeas
+    for host in ${Hosts[@]}
+    do
+        echo "storage_check host=${host}" >&2
+        if [[ "${host}" == *"cryptovaltrading.com" ]]; then
+            port=7822
+        else
+            port=22
+        fi
+        Cmd="ssh -p ${port}"
+        Cmd+=" -o StrictHostKeyChecking=no"
+        Cmd+=" -o UserKnownHostsFile=/dev/null"
+        Cmd+="  $host"
+        Cmd+=" eval \"df -hTl"
+        Cmd+=" -x squashfs"
+        Cmd+=" -x tmpfs"
+        Cmd+=" -x vfat"
+        Cmd+=" -x devtmpfs"
+        Cmd+=" | grep -v Filesystem\""
+
+        IFS=$'\n' ; lines=$(eval ${Cmd})
+        for ln in $lines
+        do
+            IFS=$' '; args=($ln); unset IFS
+            res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |"
+
+            result_lines+=("$res")
+
+            fs=${args[6]}
+            used=${args[5]}
+            Metrics+=("$host $fs $used")
+        done
+        unset IFS
+    done
+    for ln in "${result_lines[@]}"
+    do
+        echo "${ln}"
+    done
+}
+
+User=oleg
+Metrics=()
+TempFiles=
+
+Hosts=()
+for Domain in cvtt.vpn cryptovaltrading.com ; do
+    Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
+done
+
+tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
+tmpfile2=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile2}"
+
+storage_check  "${Hosts[@]}" > ${tmpfile2}
+
+echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
+echo  >> ${tmpfile}
+echo "| host | filesystem | usage |"  >> ${tmpfile}
+echo "| --- | --- | --- |"  >> ${tmpfile}
+cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
+cat ${tmpfile} |  ${Sender} ${StatusChannel}
+
+tmpfile=$(mktemp)
+TempFiles="${TempFiles} ${tmpfile}"
+
+space_alert > ${tmpfile}
+if [ -s ${tmpfile} ]
+then
+    (echo "### :card_file_box: STORAGE ALERTS" && cat ${tmpfile}) | ${Sender} ${AlertChannel}
+else
+    echo "No Storage Alerts"
+fi
+
diff --git a/save_cronjobs.sh b/save_cronjobs.sh
new file mode 100755
index 0000000..227de99
--- /dev/null
+++ b/save_cronjobs.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+usage() {
+    echo "Usage: ${0} <local_root_dir>"
+    exit 1
+}
+
+get_user_hosts() {
+    local User=${1}
+    local Domain=${2}
+
+    Cmd="curl -s http://cloud16.cvtt.vpn:6789/admin/cvtt_hosts"
+    Cmd="${Cmd} | ${HOME}/bin/hjson -j"
+    Cmd="${Cmd} | jq -r"
+    Cmd="${Cmd} --arg domain \"${Domain}\""
+    Cmd="${Cmd} --arg usr \"${User}\""
+    Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
+    # echo $Cmd  ### - FOR DEBUG ONLY
+    eval ${Cmd}
+}
+
+Domain=cvtt.vpn
+echo "Started ${0} ${*} ..."
+
+RootDir=${1}
+
+if [ ! -d "${RootDir}" ]; then
+    usage
+fi
+
+cd ${RootDir}
+Cmd="git pull"
+echo ${Cmd} && eval ${Cmd}
+
+for User in cvtt oleg
+do
+    echo "User=$User"
+    Cmd="mkdir -p ${RootDir}/${User}"
+    echo ${Cmd} && eval ${Cmd}
+
+    hosts=$(get_user_hosts ${User} ${Domain})
+    echo ${hosts}
+
+    for host in ${hosts}
+    do
+        echo "===================="
+        echo "${host}:${User}"
+        Cmd="ssh ${User}@${host}.${Domain} 'crontab -l' | tee ${RootDir}/${User}/${host}.cron"
+        echo ${Cmd} && eval ${Cmd}
+    done
+done
+
+cd ${RootDir}
+
+Cmd="find -type f -empty -delete -print"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="find -type d -empty -delete -print"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git add ."
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git commit -a -m '$(date)'"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="git pushall"
+echo ${Cmd} && eval ${Cmd}
+
+echo "Done ${0} ${*}"