progress: host checks

release_version.txt

@@ -1 +1 @@
-1.7.5,using new(rust) alpaca market data structure for hbar
+1.7.6,host avalability check

research/ad_hoc/rearrange2.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# by chatGPT
+# ==========================================
+# homestore.cvtt.vpn SETTINGS
+# ==========================================
+
+# Define source and destination base directories
+SOURCE_BASE_DIR="/works/cvtt/md_archive/equity/alpaca_md.OLD"           # Replace with your actual source directory
+DESTINATION_BASE_DIR="/works/cvtt/md_archive/equity/alpaca_md"  # Replace with your destination directory
+
+# Loop through all .db.gz files in the source directory
+find "$SOURCE_BASE_DIR" -type f -name "*.db.gz" | while read -r file; do
+    # Extract relevant directory components from the path
+    relative_path="${file#$SOURCE_BASE_DIR/}"
+    ticker=$(echo "$relative_path" | cut -d'/' -f2)
+    year=$(echo "$relative_path" | cut -d'/' -f3)
+    original_date=$(basename "$file" | cut -d'.' -f1)
+
+    # Define the new file name and destination path
+    new_file_name="${original_date}.${ticker}.alpaca_1m_bars.db.gz"
+    destination_dir="$DESTINATION_BASE_DIR/$year/A/$ticker"
+
+    # Create the destination directory if it doesn't exist
+    mkdir -p "$destination_dir"
+    #echo DEBUG mkdir -p "$destination_dir"
+
+    # Move and rename the file
+    mv "$file" "$destination_dir/$new_file_name"
+    #echo DEBUG mv "$file" "$destination_dir/$new_file_name"
+
+    echo "Moved $file to $destination_dir/$new_file_name"
+done

research/load_eqty_md.sh

@@ -167,7 +167,7 @@ EOF
 echo "Loading files"
 for stock in "${Stocks[@]}"; do
     StockLetter="${stock:0:1}"
-    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.alpaca_1min_bars.db")
+    SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.alpaca_1m_bars.db")
 
     SourceFileZip="${SourceFile}.gz"
     SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/%Y/${StockLetter}/${stock}/${SourceFileZip}")

scripts/healthcheck/hosts_check.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo
+    exit 1
+}
+
+RootDir="${HOME}/prod"
+RootDir="${HOME}/develop/cvtt2"
+
+AlertChannel=Alerts-CVTT
+Sender=${RootDir}/ops/utils/send_mmost.sh
+
+HOSTS_CONFIG=$(curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts | ${HOME}/bin/hjson -j)
+
+get_domains() {
+    echo ${HOSTS_CONFIG} | jq -r '. | keys[]'
+}
+get_user_hosts() {
+    local User=${1}
+    local Domain=${2}
+
+    jdcmd="jq -r --arg domain \"$Domain\""
+    jdcmd="$jdcmd --arg usr \"$User\""
+    jdcmd="$jdcmd '.[\$domain]"
+    jdcmd="$jdcmd | to_entries[]"
+    jdcmd="$jdcmd | select(.value.users[]"
+    jdcmd="$jdcmd | contains(\$usr))"
+    jdcmd="$jdcmd | .key'"
+    echo ${HOSTS_CONFIG} | eval ${jdcmd} |sed "s/$/.$Domain/" # >&2
+}
+
+function host_alert() {
+    alert=${1}
+    if [ "${alert}" != "" ]
+    then
+        echo -e "### :fire: HOST ALERT \n${alert}" | ${Sender} ${AlertChannel}
+    fi
+
+}
+
+
+User=oleg
+Hosts=()
+DEFAULT_SSH_PORT=22
+Domains=("${Domains[@]}" "$(get_domains)")
+
+for Domain in ${Domains[@]} ; do
+    Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
+done
+
+for Host in ${Hosts[@]} ; do
+    host=$(echo $Host | cut -d'.' -f1)
+    Domain=$(echo $Host | cut -d'.' -f2-)
+    # echo "Host=$Host host=$host Domain=$Domain"
+
+    # Get SSH port for the host, or use default if not specified
+    PortSSH=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].ssh_port // '"$DEFAULT_SSH_PORT"'')
+
+    echo "Checking host: $Host on port $PortSSH"
+
+    # Use nc to check if the specified port is open
+    if ! nc -z -w5 "$Host" "$PortSSH"; then
+        echo "Host $Host is not available on port $PortSSH"
+        host_alert "Host $Host is not available on port $PortSSH"
+    fi
+done
+
+
+# for Domain in $DOMAINS; do
+#     echo "Processing domain: $Domain"
+
+#     for host in $HOSTS; do
+#         # Construct the fully qualified hostname
+#         FQHN="${host}.${Domain}"
+
+#         # Get SSH port for the host, or use default if not specified
+#         SSH_PORT=$(echo "$JSON_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].ssh_port // '"$DEFAULT_SSH_PORT"'')
+
+#         echo "Checking host: $FQHN on port $SSH_PORT"
+
+#         # Use nc to check if the specified port is open
+#         if nc -z -w5 "$FQHN" "$SSH_PORT"; then
+#             echo "Host $FQHN is available on port $SSH_PORT"
+#         else
+#             echo "Host $FQHN is not available on port $SSH_PORT"
+#         fi
+#         echo "----------------------------------------"
+#     done
+