storage healthcheck - ignore known_hosts

admin/docker_images.sh

@@ -1,6 +1,6 @@
 #!/bin/env bash
 
-Registry=http://homestore.cvtt.vpn:5500
+Registry=http://hs01.cvtt.vpn:5500
 
 Catalog=${Registry}/v2/_catalog
 
@@ -11,5 +11,12 @@ jstr=$(curl -s -X GET ${Catalog})
 echo "${jstr}" | jq -r '.repositories[]' | while read repo; do
     TagListURL="${Registry}/v2/${repo}/tags/list"
     # echo $repo
-    curl -s -X GET ${TagListURL}
+    Cmd="curl -s -X GET ${TagListURL}"
+    echo ${Cmd}
+    eval ${Cmd}
+    # TagList=$(eval ${Cmd})
+    # # echo ${TagList}
+    # echo "${TagList}" | jq -r '.tags[]' | while read tag; do
+    #     echo "${Registry}/${repo}:${tag}"
+    # done
 done

build/build_release.sh

@@ -14,7 +14,7 @@ dist_user=cvttdist
 dist_host="cloud21.cvtt.vpn"
 dist_ssh_port="22"
 
-dist_locations="cloud21.cvtt.vpn:22 homestore.cvtt.vpn:22"
+dist_locations="cloud21.cvtt.vpn:22 hs01.cvtt.vpn:22"
 # ---------------- Settings
 
 # ---------------- cmdline

build/docker_images_builder.sh

@@ -66,7 +66,9 @@ build_docker_image() {
     convert_symlink_to_dir ${project}
 
     DockerDir=./docker_dev/${docker_dev_path}
-    echo "Building ${title}... on ${DockerDev}"
+    echo "================================================================================"
+    echo "Building ${title}... on ${DockerDir}"
+    echo "================================================================================"
 
     Cmd="docker build"
     Cmd+=" -t ${image_name}"
@@ -194,5 +196,6 @@ Cmd="rm -rf ${RootDir}/docker_dev"
 echo ${Cmd}
 eval ${Cmd}
 
+echo "Done ${0} ${*}"
 #===============================================================================
 #===============================================================================

docker/market_data/hist/crypto_md_day.sh

@@ -15,6 +15,7 @@ Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
 Cmd="${Cmd} --rm"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -h ${host}"
 Cmd="${Cmd} -d ${date}"

docker/market_data/hist/load_alpaca_hbar.sh

@@ -91,6 +91,7 @@ Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=${ContainerName}"
 Cmd="${Cmd} --rm"
 Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -d ${date_to_load}"
 if [ -n "${NumSymbols}" ]; then

docker/market_data/hist/load_alpaca_qat.sh

@@ -96,6 +96,7 @@ Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=${ContainerName}"
 Cmd="${Cmd} --rm"
 Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -d ${date_to_load}"
 Cmd="${Cmd} -S ${Symbols}"

docker/market_data/recorder/md_recorder.sh

@@ -49,6 +49,7 @@ Cmd="${Cmd} --rm"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/.creds:/.creds"
 Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
 Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
 Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"

docker/market_data/recorder/md_recorder_monitor.sh

@@ -27,9 +27,6 @@ is_container_running() {
     fi
 }
 
-
-
-
 Instance=${1}
 if ! is_valid "${Instance}"; then
     usage
@@ -53,6 +50,7 @@ Cmd="${Cmd} --rm"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/.creds:/.creds"
 Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
 Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
 Cmd="${Cmd} ${Instance} ${AdminPort}"

release_version.txt

@@ -1 +1 @@
-1.9.2.F1,fix: prepare eqt simdata script
+1.9.6,storage healthcheck - ignore known_hosts

research/ad_hoc/rearrange2.sh

@@ -2,7 +2,7 @@
 
 # by chatGPT
 # ==========================================
-# homestore.cvtt.vpn SETTINGS
+# hs01.cvtt.vpn SETTINGS
 # ==========================================
 
 # Define source and destination base directories

research/load_crypto_md.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 usage() {
-  echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
+  echo "Usage: $0 [-h <host (hs01*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
   exit 1
 }
 
@@ -20,7 +20,7 @@ is_valid() {
 }
 
 date=""
-host=homestore
+host=hs01
 source=cloud21
 
 while getopts ":h:d:s:" opt; do
@@ -51,7 +51,7 @@ fi
 
 echo "$date $host $source"
 
-valid_hosts=('homestore' 'cloud21')
+valid_hosts=('hs01' 'cloud21')
 if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
     echo "Host '${host}' is not valid"
     usage
@@ -67,8 +67,8 @@ if [ "${host}" == "cloud21" ] ; then
     SourceHost=cloud21.cvtt.vpn
     SourceUser=cvtt
     SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
-elif [ "${host}" == "homestore" ]; then
+elif [ "${host}" == "hs01" ]; then
-    SourceHost=homestore.cvtt.vpn
+    SourceHost=hs01.cvtt.vpn
     SourceUser=cvtt
     SourceRootDir=/works/cvtt/md_archive/crypto/${source}
 else

research/load_eqty_md.sh

@@ -2,7 +2,7 @@
 
 usage() {
   echo -n "Usage: $0"
-  echo -n " [-h <host (homestore*/cloud21)>]"
+  echo -n " [-h <host (hs01*/cloud21)>]"
   echo -n " [-d <YYYYMMDD> (yesterday*)]"
   echo -n " [-s <stocks comma separated>"
   echo -n " [-t <target directory>"
@@ -19,7 +19,7 @@ fi
 
 # ------------------ Settings
 md_date=""
-host=homestore
+host=hs01
 stocks=COIN,GBTC,SQ
 
 TargetDir="/opt/jupyter_gpu/data/eqty_md"
@@ -81,7 +81,7 @@ fi
 
 echo "$md_date $host $source"
 
-valid_hosts=('homestore' 'cloud21')
+valid_hosts=('hs01' 'cloud21')
 if ! is_valid "${host}"  "${valid_hosts[@]}" ; then
     echo "Host '${host}' is not valid"
     usage
@@ -108,8 +108,8 @@ if [ "${host}" == "cloud21" ] ; then
     SourceHost=cloud21.cvtt.vpn
     SourceUser=cvtt
     SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
-elif [ "${host}" == "homestore" ]; then
+elif [ "${host}" == "hs01" ]; then
-    SourceHost=homestore.cvtt.vpn
+    SourceHost=hs01.cvtt.vpn
     SourceUser=cvtt
     SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
 else

research/prepare_crypto_simdata.sh

@@ -152,7 +152,7 @@ sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
 INSERT OR IGNORE INTO md_trades SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -168,7 +168,7 @@ COMMIT;
 BEGIN;
 INSERT OR IGNORE INTO md_trades
 SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -189,7 +189,7 @@ sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
 INSERT OR IGNORE INTO md_quotes SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -269,7 +269,7 @@ Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
 echo ${Cmd}
 eval ${Cmd}
 

research/prepare_eqt_simdata.sh

@@ -168,7 +168,7 @@ for sym in ${Symbols[@]}; do
     sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${src_qat_db}' AS source_db;
 BEGIN;
-INSERT INTO md_trades SELECT
+INSERT OR IGNORE INTO md_trades SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -183,7 +183,7 @@ FROM source_db.md_trades;
 COMMIT;
 
 BEGIN;
-INSERT INTO md_quotes SELECT
+INSERT OR IGNORE INTO md_quotes SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -202,7 +202,7 @@ EOF
     sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${src_hbar_db}' AS source_db;
 BEGIN;
-INSERT INTO md_1min_bars SELECT
+INSERT OR IGNORE INTO md_1min_bars SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -225,13 +225,11 @@ EOF
     eval ${Cmd}
 done
 
-exit
-
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}
 

scripts/healthcheck/storage_health_check.sh

@@ -19,13 +19,13 @@ get_user_hosts() {
     local Domain=${2}
 
     Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
-    Cmd="${Cmd} | ${HOME}/bin/hjson -j"
+    Cmd+=" | ${HOME}/bin/hjson -j"
-    Cmd="${Cmd} | jq -r"
+    Cmd+=" | jq -r"
-    Cmd="${Cmd} --arg domain \"${Domain}\""
+    Cmd+=" --arg domain \"${Domain}\""
-    Cmd="${Cmd} --arg usr \"${User}\""
+    Cmd+=" --arg usr \"${User}\""
-    Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
+    Cmd+=" '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
 
-    Cmd="${Cmd} | sed  's/\$/.${Domain}/'"
+    Cmd+=" | sed  's/\$/.${Domain}/'"
     eval ${Cmd}
 }
 
@@ -66,13 +66,16 @@ function storage_check() {
         else
             port=22
         fi
-        Cmd="ssh -p ${port} $host"
+        Cmd="ssh -p ${port}"
-        Cmd="${Cmd} eval \"df -hTl"
+        Cmd+=" -o StrictHostKeyChecking=no"
-        Cmd="${Cmd} -x squashfs"
+        Cmd+=" -o UserKnownHostsFile=/dev/null"
-        Cmd="${Cmd} -x tmpfs"
+        Cmd+="  $host"
-        Cmd="${Cmd} -x vfat"
+        Cmd+=" eval \"df -hTl"
-        Cmd="${Cmd} -x devtmpfs"
+        Cmd+=" -x squashfs"
-        Cmd="${Cmd} | grep -v Filesystem\""
+        Cmd+=" -x tmpfs"
+        Cmd+=" -x vfat"
+        Cmd+=" -x devtmpfs"
+        Cmd+=" | grep -v Filesystem\""
 
         IFS=$'\n' ; lines=$(eval ${Cmd})
         for ln in $lines

scripts/mkt_data/checklists/crypto_md_chcklst.sh

@@ -25,8 +25,8 @@ run_checklist() {
 
     declare -A Commands
     Commands=(
-        ["homestore:cloud21"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
+        ["hs01:cloud21"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
-        ["homestore:cvttdata"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
+        ["hs01:cvttdata"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
         ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
         ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
         ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -24,11 +24,11 @@ run_checklist() {
     yr=$(date -d 'yesterday' '+%Y')
     CheckSymbols="A/AAPL N/NVDA M/META"
     declare -A Commands
-    Commands["homestore"]=""
+    Commands["hs01"]=""
     for sym in ${CheckSymbols}; do
-        Commands["homestore"]+="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
+        Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
     done
-    Commands["homestore"]+="echo"
+    Commands["hs01"]+="echo"
 
     Commands["cloud21"]=""
     for sym in ${CheckSymbols}; do

scripts/mkt_data/sync_market_data.sh

@@ -2,7 +2,7 @@
 
 Source=/home/cvtt/prod/archive/md_archive/
 Targets=
-Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/"
+Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/"
 Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"