services avalability check

release_version.txt

@@ -1 +1 @@
-1.7.3.fx1,alpaca md HBAR
+1.7.7,services avalability check

research/ad_hoc/rearrange2.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# by chatGPT
+# ==========================================
+# homestore.cvtt.vpn SETTINGS
+# ==========================================
+
+# Define source and destination base directories
+SOURCE_BASE_DIR="/works/cvtt/md_archive/equity/alpaca_md.OLD"           # Replace with your actual source directory
+DESTINATION_BASE_DIR="/works/cvtt/md_archive/equity/alpaca_md"  # Replace with your destination directory
+
+# Loop through all .db.gz files in the source directory
+find "$SOURCE_BASE_DIR" -type f -name "*.db.gz" | while read -r file; do
+    # Extract relevant directory components from the path
+    relative_path="${file#$SOURCE_BASE_DIR/}"
+    ticker=$(echo "$relative_path" | cut -d'/' -f2)
+    year=$(echo "$relative_path" | cut -d'/' -f3)
+    original_date=$(basename "$file" | cut -d'.' -f1)
+
+    # Define the new file name and destination path
+    new_file_name="${original_date}.${ticker}.alpaca_1m_bars.db.gz"
+    destination_dir="$DESTINATION_BASE_DIR/$year/A/$ticker"
+
+    # Create the destination directory if it doesn't exist
+    mkdir -p "$destination_dir"
+    #echo DEBUG mkdir -p "$destination_dir"
+
+    # Move and rename the file
+    mv "$file" "$destination_dir/$new_file_name"
+    #echo DEBUG mv "$file" "$destination_dir/$new_file_name"
+
+    echo "Moved $file to $destination_dir/$new_file_name"
+done

research/load_eqty_md.sh

@@ -167,10 +167,10 @@ EOF
 echo "Loading files"
 for stock in "${Stocks[@]}"; do
     StockLetter="${stock:0:1}"
-    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.1min.db")
+    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.alpaca_1m_bars.db")
 
     SourceFileZip="${SourceFile}.gz"
-    SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/${StockLetter}/${stock}/%Y/${SourceFileZip}")
+    SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/%Y/${StockLetter}/${stock}/${SourceFileZip}")
     echo ${SourceFilePath}
 
     download_file ${SourceFilePath}
@@ -180,6 +180,3 @@ if [ -f ${TargetFilePath} ]; then
     chmod 600
     ls -lh ${TargetFilePath}
 fi
-
-# mkdir -p /tmp/aaa
-# cp -r ${tmp_dir}/* /tmp/aaa/

scripts/healthcheck/hosts_check.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# "cryptovaltrading.com": {
+#     "cloud18": {
+#         "users": ["oleg"],
+#         "type": "cloud",
+#         "ssh_port": 7822,
+#         # "to_check": "No"
+#     },
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts
+
+HOSTS_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
+
+get_domains() {
+    echo ${HOSTS_CONFIG} | jq -r '. | keys[]'
+}
+get_user_hosts() {
+    local User=${1}
+    local Domain=${2}
+
+    jdcmd="jq -r --arg domain \"$Domain\""
+    jdcmd="$jdcmd --arg usr \"$User\""
+    jdcmd="$jdcmd '.[\$domain]"
+    jdcmd="$jdcmd | to_entries[]"
+    jdcmd="$jdcmd | select(.value.users[]"
+    jdcmd="$jdcmd | contains(\$usr))"
+    jdcmd="$jdcmd | .key'"
+    echo ${HOSTS_CONFIG} | eval ${jdcmd} |sed "s/$/.$Domain/" # >&2
+}
+
+function host_alert() {
+    alert=${1}
+    if [ "${alert}" != "" ]
+    then
+        echo -e "### :fire: HOST ALERT \n${alert}" | ${Sender} ${AlertChannel}
+    fi
+
+}
+
+User=oleg
+Hosts=()
+DEFAULT_SSH_PORT=22
+Domains=("${Domains[@]}" "$(get_domains)")
+
+for Domain in ${Domains[@]} ; do
+    Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
+done
+
+for Host in ${Hosts[@]} ; do
+    host=$(echo $Host | cut -d'.' -f1)
+    Domain=$(echo $Host | cut -d'.' -f2-)
+    # echo "Host=$Host host=$host Domain=$Domain"
+
+    # Get SSH port for the host, or use default if not specified
+    ToCheck=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].to_check // "Yes"')
+    if [ "${ToCheck}" == "No" ]; then
+        continue
+    fi
+    PortSSH=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].ssh_port // '"$DEFAULT_SSH_PORT"'')
+
+    echo "Checking host: $Host on port $PortSSH"
+
+    # Use nc to check if the specified port is open
+    if ! nc -z -w5 "$Host" "$PortSSH"; then
+        echo "Host $Host is not available on port $PortSSH"
+        host_alert "Host $Host is not available on port $PortSSH"
+    fi
+done

scripts/healthcheck/services_check.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+    # "Jenkins" {
+    #     "to_check": "Yes",
+    #     "protocol": "http",
+    #     "host": "cvtt-build.cvtt.vpn",
+    #     "port": 8080
+    # },
+
+RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/cvtt_services
+
+SERVICES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
+
+echo $Sender
+echo $AlertChannel
+
+
+function service_alert() {
+    alert="${1}"
+    if [ "${alert}" != "" ]
+    then
+        #it may contain quotes
+        alert=$(echo "${alert}" | sed 's/"/\\"/g')
+        echo -e "### :boom: SERVICE ALERT\n${alert}" | ${Sender} ${AlertChannel}
+    fi
+
+}
+
+User=oleg
+Hosts=()
+DEFAULT_TO_CHECK=Yes
+
+mapfile -t SvcNames < <(echo ${SERVICES_CONFIG} | jq -r '. | keys[]')
+
+DEFAULT_TO_CHECK=Yes
+for SvcName in "${SvcNames[@]}" ; do
+    echo "SvcName=$SvcName"
+    ToCheck=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].to_check // "Yes"')
+    if [ "${ToCheck}" == "No" ]; then
+        continue
+    fi
+    Host=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName"  '.[$svcname].host ')
+    Port=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName"  '.[$svcname].port ')
+    echo "Checking \"$SvcName\" (${Host}:${Port})"
+
+
+    # Use nc to check if the specified port is open
+    if ! nc -z -w5 "$Host" "$Port"; then
+        msg="Service \"${SvcName}\" (${Host}:${Port}) is not available"
+        echo ${msg}
+        service_alert "${msg}"
+    fi
+done
+

scripts/healthcheck/storage_health_check.sh

@@ -69,7 +69,9 @@ function storage_check() {
         Cmd="ssh -p ${port} $host"
         Cmd="${Cmd} eval \"df -hTl"
         Cmd="${Cmd} -x squashfs"
-        Cmd="${Cmd} | grep -v tmpfs"
+        Cmd="${Cmd} -x tmpfs"
+        Cmd="${Cmd} -x vfat"
+        Cmd="${Cmd} -x devtmpfs"
         Cmd="${Cmd} | grep -v Filesystem\""
 
         IFS=$'\n' ; lines=$(eval ${Cmd})