diff --git a/VERSION b/VERSION index 8b37754..3dbac14 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.6,OPS-6 +2.1.2,fix diff --git a/research/load_crypto_md.sh b/research/load_crypto_md.sh index 46e3735..dd36a89 100755 --- a/research/load_crypto_md.sh +++ b/research/load_crypto_md.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash usage() { - echo "Usage: $0 [-h ] [-d (yesterday*)] [-s (cvttdata/cloud21*)>]" + echo -n "Usage: $0 [-h ]" + echo -n " [-d (yesterday*)]" + echo -n " [-s (cvttdata/cloud21*)>]" + echo " [-t (/opt/jupyter_gpu/data/crypto_md)]" exit 1 } @@ -19,11 +22,15 @@ is_valid() { return 1 } +# ------- D E F A U L T S date="" host=hs01 source=cloud21 +TargetDir="/opt/jupyter_gpu/data/crypto_md" +# ------- D E F A U L T S -while getopts ":h:d:s:" opt; do + +while getopts ":h:d:s:t:" opt; do case ${opt} in d ) date=$OPTARG @@ -34,6 +41,9 @@ while getopts ":h:d:s:" opt; do s ) source=$OPTARG ;; + t ) + TargetDir=$OPTARG + ;; \? ) echo "Invalid option: -$OPTARG" >&2 usage @@ -82,7 +92,6 @@ SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db") SourceFileZip="${SourceFile}.gz" SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}") -TargetDir="/opt/jupyter_gpu/data/crypto_md" TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db") TargetFilePath="${TargetDir}/${TargetFile}" diff --git a/research/prepare_crypto_simdata.sh b/research/prepare_crypto_simdata.sh index da11ae8..ad47de8 100755 --- a/research/prepare_crypto_simdata.sh +++ b/research/prepare_crypto_simdata.sh @@ -10,17 +10,22 @@ usage() { - echo "Usage: $0 -d YYYMMDD Date> [-O ]" + echo -n "Usage: $0" + echo -n " -d YYYMMDD Date>" + echo -n " [-O ]" + echo -n " [-S ]" + echo -n " [-D (cloud21)]" + echo exit 1 } # --------------------- Settings -SourceHost=cloud21.cvtt.vpn +SourceHost=cloud21 SourceRootDir=/opt/store/cvtt/md_archive/crypto DbSource=cloud21 # --------------------- Settings -while getopts ":d:O:" opt; do +while getopts ":d:O:S:D:" opt; do case ${opt} in d ) Date=$OPTARG @@ -28,6 +33,12 @@ while getopts ":d:O:" opt; do O ) OutputDir=$OPTARG ;; + S ) + SourceHost=$OPTARG + ;; + D ) + DbSource=$OPTARG + ;; \? ) echo "Invalid option: -$OPTARG" >&2 usage @@ -46,6 +57,8 @@ fi if [ -z ${OutputDir} ] ; then OutputDir=. fi + +SourceHost+=".cvtt.vpn" mkdir -p ${OutputDir} # --- Binance diff --git a/scripts/mkt_data/checklists/crypto_md_chcklst.sh b/scripts/mkt_data/checklists/crypto_md_chcklst.sh index 0a9ec6d..a9ec92f 100755 --- a/scripts/mkt_data/checklists/crypto_md_chcklst.sh +++ b/scripts/mkt_data/checklists/crypto_md_chcklst.sh @@ -27,8 +27,10 @@ run_checklist() { Commands=( ["hs01:cloud21"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" ["hs01:cvttdata"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" - ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" - ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim/ | tail -5" + ["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5" + ["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5" + ["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim | tail -5" ["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'" ) diff --git a/scripts/mkt_data/checklists/eqty_md_chcklst.sh b/scripts/mkt_data/checklists/eqty_md_chcklst.sh index 0787c32..322bc0c 100755 --- a/scripts/mkt_data/checklists/eqty_md_chcklst.sh +++ b/scripts/mkt_data/checklists/eqty_md_chcklst.sh @@ -29,12 +29,14 @@ run_checklist() { Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; " done Commands["hs01"]+="echo" + Commands["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" Commands["cloud21"]="" for sym in ${CheckSymbols}; do - Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; " + Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; " done Commands["cloud21"]+="echo" + Commands["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5" Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'" @@ -46,6 +48,13 @@ run_checklist() { done } +tmpfile=$(mktemp) +function cleanup { + cd ${HOME} + rm -f ${tmpfile} +} +trap cleanup EXIT + # run_checklist tmpfile=$(mktemp) diff --git a/utils/archive_logs.sh b/utils/archive_logs.sh index a9f2c1c..574ceeb 100755 --- a/utils/archive_logs.sh +++ b/utils/archive_logs.sh @@ -1,36 +1,62 @@ #!/bin/bash - function usage { - echo "Usage: ${0} [days (default 2)]" + echo -n "Usage: ${0}" + echo -n " -L " + echo -n " [ -A (default /works/archive/logs)]" + echo -n " [-D (default: '2 days ago')]" + echo exit 1 } echo Starting $0 $* LogDir=${1} +LogArchiveDir=/works/archive/logs +DateCriteria="2 days ago" + + +# ---------------- cmdline +while getopts "A:L:D:" opt; do + case ${opt} in + A ) + LogArchiveDir=$OPTARG + ;; + L ) + LogDir=$OPTARG + ;; + D ) + DateCriteria=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done +# ---------------- cmdline + + if [ "${LogDir}" == "" ] then usage fi -Days=${2} -if [ "${Days}" == "" ] -then - Days=2 -fi +Cmd="mkdir -p ${LogArchiveDir}" +echo ${Cmd} && eval ${Cmd} -DateCriteria="${Days} days ago" +echo "Looking for log files older than '${DateCriteria}' in ${LogDir}" -Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d') +Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S') -echo "Looking for log files older than ${DateCriteria} in ${LogDir}" +Cmd="find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt \"${Oldest}\"" +echo $Cmd -LogArchiveDir=${HOME}/prod/archive/logs -mkdir -p ${LogArchiveDir} - -echo "find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt ${Oldest})" -files=$(find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt ${Oldest}) +files=$(eval ${Cmd}) if [ "$files" == "" ] then echo "No files found older than ${Oldest} in ${LogDir}"