From 06610de9928afcaa234e1e7028e29b39ea02fd73 Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Fri, 21 Mar 2025 17:04:23 +0000 Subject: [PATCH 1/5] OPS-10 --- VERSION | 2 +- scripts/mkt_data/checklists/crypto_md_chcklst.sh | 6 ++++-- scripts/mkt_data/checklists/eqty_md_chcklst.sh | 11 ++++++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 8b37754..c74ca0e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6,OPS-6 +2.0.7,OPS-10 diff --git a/scripts/mkt_data/checklists/crypto_md_chcklst.sh b/scripts/mkt_data/checklists/crypto_md_chcklst.sh index 0a9ec6d..a9ec92f 100755 --- a/scripts/mkt_data/checklists/crypto_md_chcklst.sh +++ b/scripts/mkt_data/checklists/crypto_md_chcklst.sh @@ -27,8 +27,10 @@ run_checklist() { Commands=( ["hs01:cloud21"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" ["hs01:cvttdata"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" - ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" - ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim/ | tail -5" + ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" + ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim | tail -5" ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'" ) diff --git a/scripts/mkt_data/checklists/eqty_md_chcklst.sh b/scripts/mkt_data/checklists/eqty_md_chcklst.sh index 0787c32..322bc0c 100755 --- a/scripts/mkt_data/checklists/eqty_md_chcklst.sh +++ b/scripts/mkt_data/checklists/eqty_md_chcklst.sh @@ -29,12 +29,14 @@ run_checklist() { Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; " done Commands["hs01"]+="echo" + Commands["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" Commands["cloud21"]="" for sym in ${CheckSymbols}; do - Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; " + Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; " done Commands["cloud21"]+="echo" + Commands["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'" @@ -46,6 +48,13 @@ run_checklist() { done } +tmpfile=$(mktemp) +function cleanup { + cd ${HOME} + rm -f ${tmpfile} +} +trap cleanup EXIT + # run_checklist tmpfile=$(mktemp) From df04d764c8d790ba140da07e94afe193649428e5 Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Mon, 24 Mar 2025 17:25:57 +0000 Subject: [PATCH 2/5] prepare crypto sim enhanced --- VERSION | 2 +- research/prepare_crypto_simdata.sh | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index c74ca0e..56d0235 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.7,OPS-10 +2.0.8,prepare crypto sim enhanced diff --git a/research/prepare_crypto_simdata.sh b/research/prepare_crypto_simdata.sh index da11ae8..ad47de8 100755 --- a/research/prepare_crypto_simdata.sh +++ b/research/prepare_crypto_simdata.sh @@ -10,17 +10,22 @@ usage() { - echo "Usage: $0 -d YYYMMDD Date> [-O ]" + echo -n "Usage: $0" + echo -n " -d YYYMMDD Date>" + echo -n " [-O ]" + echo -n " [-S ]" + echo -n " [-D (cloud21)]" + echo exit 1 } # --------------------- Settings -SourceHost=cloud21.cvtt.vpn +SourceHost=cloud21 SourceRootDir=/opt/store/cvtt/md_archive/crypto DbSource=cloud21 # --------------------- Settings -while getopts ":d:O:" opt; do +while getopts ":d:O:S:D:" opt; do case ${opt} in d ) Date=$OPTARG @@ -28,6 +33,12 @@ while getopts ":d:O:" opt; do O ) OutputDir=$OPTARG ;; + S ) + SourceHost=$OPTARG + ;; + D ) + DbSource=$OPTARG + ;; \? ) echo "Invalid option: -$OPTARG" >&2 usage @@ -46,6 +57,8 @@ fi if [ -z ${OutputDir} ] ; then OutputDir=. fi + +SourceHost+=".cvtt.vpn" mkdir -p ${OutputDir} # --- Binance From f59c729c6f6085d35745f133abaf468c8011a3a6 Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Wed, 16 Apr 2025 17:15:03 +0000 Subject: [PATCH 3/5] flexible target directory for loading sim data --- VERSION | 2 +- research/load_crypto_md.sh | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 56d0235..27bdbf2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.8,prepare crypto sim enhanced +2.0.9,prepare crypto sim enhanced diff --git a/research/load_crypto_md.sh b/research/load_crypto_md.sh index 46e3735..dd36a89 100755 --- a/research/load_crypto_md.sh +++ b/research/load_crypto_md.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash usage() { - echo "Usage: $0 [-h ] [-d (yesterday*)] [-s (cvttdata/cloud21*)>]" + echo -n "Usage: $0 [-h ]" + echo -n " [-d (yesterday*)]" + echo -n " [-s (cvttdata/cloud21*)>]" + echo " [-t (/opt/jupyter_gpu/data/crypto_md)]" exit 1 } @@ -19,11 +22,15 @@ is_valid() { return 1 } +# ------- D E F A U L T S date="" host=hs01 source=cloud21 +TargetDir="/opt/jupyter_gpu/data/crypto_md" +# ------- D E F A U L T S -while getopts ":h:d:s:" opt; do + +while getopts ":h:d:s:t:" opt; do case ${opt} in d ) date=$OPTARG @@ -34,6 +41,9 @@ while getopts ":h:d:s:" opt; do s ) source=$OPTARG ;; + t ) + TargetDir=$OPTARG + ;; \? ) echo "Invalid option: -$OPTARG" >&2 usage @@ -82,7 +92,6 @@ SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db") SourceFileZip="${SourceFile}.gz" SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}") -TargetDir="/opt/jupyter_gpu/data/crypto_md" TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db") TargetFilePath="${TargetDir}/${TargetFile}" From 52dd9997ca7f9710315bac935b6a44f90c318fb3 Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Thu, 15 May 2025 19:30:09 +0000 Subject: [PATCH 4/5] 2.1.1,archive_logs interface change --- VERSION | 2 +- utils/archive_logs.sh | 56 +++++++++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/VERSION b/VERSION index 27bdbf2..dd7ae89 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.9,prepare crypto sim enhanced +2.1.1,archive_logs interface change diff --git a/utils/archive_logs.sh b/utils/archive_logs.sh index a9f2c1c..e98b49c 100755 --- a/utils/archive_logs.sh +++ b/utils/archive_logs.sh @@ -1,36 +1,62 @@ #!/bin/bash - function usage { - echo "Usage: ${0} [days (default 2)]" + echo -n "Usage: ${0}" + echo -n " -L " + echo -n " [ -A (default /works/archive/logs)]" + echo -n " [-D (default: '2 days ago')]" + echo exit 1 } echo Starting $0 $* LogDir=${1} +LogArchiveDir=/works/logs +DateCriteria="2 days ago" + + +# ---------------- cmdline +while getopts "A:L:D:" opt; do + case ${opt} in + A ) + LogArchiveDir=$OPTARG + ;; + L ) + LogDir=$OPTARG + ;; + D ) + DateCriteria=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done +# ---------------- cmdline + + if [ "${LogDir}" == "" ] then usage fi -Days=${2} -if [ "${Days}" == "" ] -then - Days=2 -fi +Cmd="mkdir -p ${LogArchiveDir}" +echo ${Cmd} && eval ${Cmd} -DateCriteria="${Days} days ago" +echo "Looking for log files older than '${DateCriteria}' in ${LogDir}" -Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d') +Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S') -echo "Looking for log files older than ${DateCriteria} in ${LogDir}" +Cmd="find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt \"${Oldest}\"" +echo $Cmd -LogArchiveDir=${HOME}/prod/archive/logs -mkdir -p ${LogArchiveDir} - -echo "find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt ${Oldest})" -files=$(find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt ${Oldest}) +files=$(eval ${Cmd}) if [ "$files" == "" ] then echo "No files found older than ${Oldest} in ${LogDir}" From 1432794197eb9b65d9a8db715bf9b52fe0216392 Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Thu, 15 May 2025 19:40:31 +0000 Subject: [PATCH 5/5] 2.1.2,fix --- VERSION | 2 +- utils/archive_logs.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index dd7ae89..3dbac14 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.1,archive_logs interface change +2.1.2,fix diff --git a/utils/archive_logs.sh b/utils/archive_logs.sh index e98b49c..574ceeb 100755 --- a/utils/archive_logs.sh +++ b/utils/archive_logs.sh @@ -12,7 +12,7 @@ function usage { echo Starting $0 $* LogDir=${1} -LogArchiveDir=/works/logs +LogArchiveDir=/works/archive/logs DateCriteria="2 days ago"