archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of cloud21.cvtt.vpn:/opt/store/git/cvtt2/ops

Browse Source
This commit is contained in:
Oleg Sheynin 2025-05-15 18:28:08 -04:00
commit 5c3fd357ab
6 changed files with 84 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
VERSION
View File

@ -1 +1 @@
2.0.6,OPS-6
2.1.2,fix

15
research/load_crypto_md.sh
View File

@ -1,7 +1,10 @@
#!/usr/bin/env bash
usage() {
echo "Usage: $0 [-h <host (hs01*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
echo -n "Usage: $0 [-h <host (hs01*/cloud21)>]"
echo -n " [-d <YYYYMMDD> (yesterday*)]"
echo -n " [-s <source> (cvttdata/cloud21*)>]"
echo " [-t <target_dir> (/opt/jupyter_gpu/data/crypto_md)]"
exit 1
}
@ -19,11 +22,15 @@ is_valid() {
return 1
}
# ------- D E F A U L T S
date=""
host=hs01
source=cloud21
TargetDir="/opt/jupyter_gpu/data/crypto_md"
# ------- D E F A U L T S
while getopts ":h:d:s:" opt; do
while getopts ":h:d:s:t:" opt; do
case ${opt} in
d )
date=$OPTARG
@ -34,6 +41,9 @@ while getopts ":h:d:s:" opt; do
s )
source=$OPTARG
;;
t )
TargetDir=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
@ -82,7 +92,6 @@ SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
SourceFileZip="${SourceFile}.gz"
SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
TargetDir="/opt/jupyter_gpu/data/crypto_md"
TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
TargetFilePath="${TargetDir}/${TargetFile}"

19
research/prepare_crypto_simdata.sh
View File

@ -10,17 +10,22 @@
usage() {
echo "Usage: $0 -d YYYMMDD Date> [-O <output dir (./) >]"
echo -n "Usage: $0"
echo -n " -d YYYMMDD Date>"
echo -n " [-O <output dir (./) >]"
echo -n " [-S <archive_source_host (cloud21)>]"
echo -n " [-D <orig_db_source> (cloud21)]"
echo
exit 1
}
# --------------------- Settings
SourceHost=cloud21.cvtt.vpn
SourceHost=cloud21
SourceRootDir=/opt/store/cvtt/md_archive/crypto
DbSource=cloud21
# --------------------- Settings
while getopts ":d:O:" opt; do
while getopts ":d:O:S:D:" opt; do
case ${opt} in
d )
Date=$OPTARG
@ -28,6 +33,12 @@ while getopts ":d:O:" opt; do
O )
OutputDir=$OPTARG
;;
S )
SourceHost=$OPTARG
;;
D )
DbSource=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
@ -46,6 +57,8 @@ fi
if [ -z ${OutputDir} ] ; then
OutputDir=.
fi
SourceHost+=".cvtt.vpn"
mkdir -p ${OutputDir}
# --- Binance

6
scripts/mkt_data/checklists/crypto_md_chcklst.sh
View File

@ -27,8 +27,10 @@ run_checklist() {
Commands=(
["hs01:cloud21"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["hs01:cvttdata"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim/ | tail -5"
["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/sim | tail -5"
["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
)

11
scripts/mkt_data/checklists/eqty_md_chcklst.sh
View File

@ -29,12 +29,14 @@ run_checklist() {
Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
done
Commands["hs01"]+="echo"
Commands["hs01:sim"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5"
Commands["cloud21"]=""
for sym in ${CheckSymbols}; do
Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
done
Commands["cloud21"]+="echo"
Commands["cloud21:sim"]="ssh cvtt@cloud21.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/sim | tail -5"
Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
@ -46,6 +48,13 @@ run_checklist() {
done
}
tmpfile=$(mktemp)
function cleanup {
cd ${HOME}
rm -f ${tmpfile}
}
trap cleanup EXIT
# run_checklist
tmpfile=$(mktemp)

56
utils/archive_logs.sh
View File

@ -1,36 +1,62 @@
#!/bin/bash
function usage {
echo "Usage: ${0} <log directory> [days (default 2)]"
echo -n "Usage: ${0}"
echo -n " -L <log directory>"
echo -n " [ -A <archive_logs_dir> (default /works/archive/logs)]"
echo -n " [-D <older than time criteria> (default: '2 days ago')]"
echo
exit 1
}
echo Starting $0 $*
LogDir=${1}
LogArchiveDir=/works/archive/logs
DateCriteria="2 days ago"
# ---------------- cmdline
while getopts "A:L:D:" opt; do
case ${opt} in
A )
LogArchiveDir=$OPTARG
;;
L )
LogDir=$OPTARG
;;
D )
DateCriteria=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
# ---------------- cmdline
if [ "${LogDir}" == "" ]
then
usage
fi
Days=${2}
if [ "${Days}" == "" ]
then
Days=2
fi
Cmd="mkdir -p ${LogArchiveDir}"
echo ${Cmd} && eval ${Cmd}
DateCriteria="${Days} days ago"
echo "Looking for log files older than '${DateCriteria}' in ${LogDir}"
Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d')
Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S')
echo "Looking for log files older than ${DateCriteria} in ${LogDir}"
Cmd="find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt \"${Oldest}\""
echo $Cmd
LogArchiveDir=${HOME}/prod/archive/logs
mkdir -p ${LogArchiveDir}
echo "find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt ${Oldest})"
files=$(find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt ${Oldest})
files=$(eval ${Cmd})
if [ "$files" == "" ]
then
echo "No files found older than ${Oldest} in ${LogDir}"