Compare commits

...

35 Commits

Author SHA1 Message Date
oleg 9d9b9c17f6 renamed2 2024-08-18 17:05:06 -04:00
oleg 5ad576b704 renaming checklist script 2024-08-18 17:02:42 -04:00
oleg 6d0e32f6ad progress. md checklists 2024-08-18 16:44:19 -04:00
oleg b57fb2257f progress - intermediate for md checklists 2024-08-18 16:29:40 -04:00
oleg 1dafab8971 fix 2024-08-17 14:39:07 -04:00
oleg 9dfdc615de fix 2024-08-17 14:25:03 -04:00
oleg 17f29a3a67 fix 2024-08-17 14:24:32 -04:00
oleg 1de85ffe73 fix 2024-08-17 14:02:13 -04:00
oleg 4ce11b12ce clean 2024-08-17 14:00:25 -04:00
oleg 4962d59d46 cleaning 2024-08-17 13:59:30 -04:00
oleg e444f697eb progress 2024-08-14 20:08:12 -04:00
oleg 087e536bdf progress 2024-08-14 15:31:42 -04:00
oleg c3218eac2e progress 2024-08-13 23:03:24 -04:00
oleg e5716c29a3 progress 2024-08-12 21:35:58 -04:00
oleg 7c7248efd3 progress 2024-08-12 20:30:55 -04:00
oleg ff7f8c0e01 progress 2024-08-10 23:33:30 -04:00
oleg 7c34d6e918 fix 2024-08-10 23:28:07 -04:00
oleg 55782dbd44 progress 2024-08-10 23:24:46 -04:00
oleg 72c9cb2e00 progress 2024-08-09 22:54:33 -04:00
oleg a610ab5462 progress 2024-08-09 22:43:44 -04:00
oleg 969360642a debugging crypto_md_day 2024-08-08 20:51:45 -04:00
oleg 939cc0197d progress: build script 2024-08-08 18:17:00 -04:00
oleg 0766a07b79 fix 2024-08-08 13:04:16 -04:00
oleg 9874aaeac7 fix 2024-08-06 23:59:35 -04:00
oleg 6458974866 progress: storage check uses hosts from config service 2024-08-06 16:05:55 -04:00
oleg df761027ec progress 2024-08-05 22:41:52 -04:00
oleg 90fa146190 fix 2024-08-05 18:17:12 -04:00
oleg c3ed03e865 hjson and jq in bash 2024-08-05 17:38:12 -04:00
oleg 87250d5c77 minor 2024-08-05 14:28:05 -04:00
oleg b1fe4f38f0 progress 2024-07-31 12:44:24 -04:00
oleg c7e717a64f fix 2024-07-31 12:37:32 -04:00
oleg c6b99f6678 progress 2024-07-31 12:24:00 -04:00
oleg 6c771ad1a1 progress 2024-07-31 11:34:58 -04:00
oleg 066a9fbdf1 progress 2024-07-31 11:24:32 -04:00
oleg e7d7af1889 progress 2024-07-30 23:32:40 -04:00
33 changed files with 700 additions and 657 deletions
@@ -127,7 +127,8 @@ if [ "${?}" != "0" ]; then
fi fi
release_version=$(cat release_version.txt) release_version=$(cat release_version.txt | awk -F',' '{print $1}')
whats_new=$(cat release_version.txt | awk -F',' '{print $2}')
echo "--------------------------------" echo "--------------------------------"
@@ -136,7 +137,7 @@ echo "Release version: ${release_version}"
confirm confirm
version_tag="v${release_version}" version_tag="v${release_version}"
version_comment="${version_tag}_${project}_${branch}_$(date +%Y%m%d)" version_comment="'${version_tag} ${project} ${branch} $(date +%Y-%m-%d)\n${whats_new}'"
cmd_arr=() cmd_arr=()
Cmd="git tag -a ${version_tag} -m ${version_comment}" Cmd="git tag -a ${version_tag} -m ${version_comment}"
@@ -178,4 +179,8 @@ do
pwd && echo ${cmd} && eval ${cmd} pwd && echo ${cmd} && eval ${cmd}
done done
echo "PROJECT ${project} **** P U L L N E W T A G S F R O M o r i g i n ****"
echo "----"
echo "(cd ${project}; git pull --tags; git pushall)"
echo "----"
echo "$0 Done ${project} ${release_version}" echo "$0 Done ${project} ${release_version}"
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
usage() {
echo "Usage: $0 <project name>"
exit 1
}
prj=${1}
if [ "" == "${prj}" ] ; then
usage
fi
Cmd="pushd /home/oleg/develop/cvtt2"
Cmd="${Cmd} && (cd ${prj}"
Cmd="${Cmd} && git pushall)"
Cmd="${Cmd} && ./build_release.sh -p ${prj}"
Cmd="${Cmd} && (cd ${prj}"
Cmd="${Cmd} && git pull --tags"
Cmd="${Cmd} && git pushall)"
Cmd="${Cmd} && popd"
echo ${Cmd}
eval ${Cmd}
-10
View File
@@ -1,10 +0,0 @@
0 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart config_service 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).config_service.restart.log
#
#
#---------------------- Utils
0 5 * * * /home/cvtt/prod/utils/config_svc_backup.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).config_svc_backup.log
0 6 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).archive_logs.log
10 1 * * * /home/cvtt/prod/utils/move_archives.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).move_archives.log
#
#---------------------- T E S T
-11
View File
@@ -1,11 +0,0 @@
0 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart all 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).bnbspot_md_recorder.restart.log
#
#10 6 * * * /usr/bin/supervisorctl -c /home/cvtt/prod/config/supervisor/supervisord.conf restart executor 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).executor_app.restart.log
#
#---------------------- Utils
0 6 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cronjob_log_archive.log
10 1 * * * /home/cvtt/prod/utils/move_archives.sh cloud17 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).move_archives.log
#
# ----------------- # TEST
#
# ----------------- # TEST
-8
View File
@@ -1,8 +0,0 @@
20 5 * * SAT /home/cvtt/prod/run/load_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /home/cvtt/prod/logs/$(date +\
20 6 * * SAT /home/cvtt/prod/run/prune_eqt_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /home/cvtt/prod/logs/$(da
#
0 6 * * SUN /home/cvtt/prod/run/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/p
#
# T E S T
#----------------------------------------------------------------------
#----------------------------------------------------------------------
-11
View File
@@ -1,11 +0,0 @@
#
#---------------------- Market Data
0 6 * * * /home/cvtt/prod/run/archive_yesterday_md.sh coinbase,bnbspot,bnbfut 2>&1 > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).archive_yestd_md.log
#---------------------- Utils
0 7 * * * /home/cvtt/prod/utils/backup_md_archive.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).backup_md_archive.log
#
30 7 * * * /home/cvtt/prod/utils/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' > /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cronjob_log_archive.log
#
# ----------------- # TEST
#
# ----------------- # TEST
+70
View File
@@ -0,0 +1,70 @@
#!/bin/bash
usage() {
echo "Usage: ${0} <local_root_dir>"
exit 1
}
get_user_hosts() {
local User=${1}
local Domain=${2}
Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
Cmd="${Cmd} | ${HOME}/bin/hjson -j"
Cmd="${Cmd} | jq -r"
Cmd="${Cmd} --arg domain \"${Domain}\""
Cmd="${Cmd} --arg usr \"${User}\""
Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
# echo $Cmd ### - FOR DEBUG ONLY
eval ${Cmd}
}
Domain=cvtt.vpn
echo "Started ${0} ${*} ..."
RootDir=${1}
if [ ! -d "${RootDir}" ]; then
usage
fi
cd ${RootDir}
Cmd="git pull"
echo ${Cmd} && eval ${Cmd}
for User in cvtt oleg
do
echo "User=$User"
Cmd="mkdir -p ${RootDir}/${User}"
echo ${Cmd} && eval ${Cmd}
hosts=$(get_user_hosts ${User} ${Domain})
echo ${hosts}
for host in ${hosts}
do
echo "===================="
echo "${host}:${User}"
Cmd="ssh ${User}@${host}.${Domain} 'crontab -l' | tee ${RootDir}/${User}/${host}.cron"
echo ${Cmd} && eval ${Cmd}
done
done
cd ${RootDir}
Cmd="find -type f -empty -delete -print"
echo ${Cmd} && eval ${Cmd}
Cmd="find -type d -empty -delete -print"
echo ${Cmd} && eval ${Cmd}
Cmd="git add ."
echo ${Cmd} && eval ${Cmd}
Cmd="git commit -a -m '$(date)'"
echo ${Cmd} && eval ${Cmd}
Cmd="git pushall"
echo ${Cmd} && eval ${Cmd}
echo "Done ${0} ${*}"
+2 -1
View File
@@ -18,7 +18,8 @@ Cmd="${Cmd} --rm"
Cmd="${Cmd} ${DockerImage}" Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} -h ${host}" Cmd="${Cmd} -h ${host}"
Cmd="${Cmd} -d ${date}" Cmd="${Cmd} -d ${date}"
Cmd="${Cmd} -s coinbase,bnbspot,bnbfut" # Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
Cmd="${Cmd} -s coinbase,bnbspot"
echo $Cmd echo $Cmd
eval $Cmd eval $Cmd
+2
View File
@@ -0,0 +1,2 @@
ROOT_DIR=/home/cvtt/prod
ADMIN_PORT=7220
@@ -0,0 +1,20 @@
version: '3.8'
#==================================================
# Relies on the file `.env` content for varables:
# ROOT_DIR
# JUPYTER_PORT
#==================================================
services:
md_portal:
image: cloud21.cvtt.vpn:5500/md_portal:latest
network_mode: host
container_name: md_portal
restart: unless-stopped
environment:
- JUPYTER_ENABLE_LAB=yes
volumes:
- ${ROOT_DIR}/logs:/logs
ports:
- "${ADMIN_PORT}:${ADMIN_PORT}"
shm_size: "8gb"
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
args=${*}
ConfigServer=cloud23.cvtt.vpn:6789
ActiveExchanges=ALPACA_SNDBX
PortalName=MD_PORTAL_ALPACA
AdminPort=7220
while getopts ":c:e:a:n:" opt; do
case ${opt} in
c )
ConfigServer=$OPTARG
;;
e )
ActiveExchanges=$OPTARG
;;
a )
AdminPort=$OPTARG
;;
n )
PortalName=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
echo "ConfigServer=${ConfigServer}"
echo "ActiveExchanges=${ActiveExchanges}"
echo "PortalName=${PortalName}"
echo "AdminPort=${AdminPort}"
ImageName=md_portal
DockerRegistry=cloud21.cvtt.vpn:5500
DockerImage=${DockerRegistry}/${ImageName}
if [ -z ${date} ] ; then
date=$(date -d "yesterday" +%Y%m%d)
fi
Cmd="docker run"
Cmd="${Cmd} --detach"
Cmd="${Cmd} --restart=unless-stopped"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --name=${ImageName}.${PortalName}"
Cmd="${Cmd} --volume /home/cvtt/prod/logs:/logs"
Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} ${args}"
echo $Cmd
eval $Cmd
+1 -1
View File
@@ -1 +1 @@
1.2.5 1.5.1A,md checklists
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env bash
usage() {
echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
exit 1
}
is_valid() {
local inst=$1
shift
local valid_instances=("$@")
for valid_inst in "${valid_instances[@]}";
do
if [[ "$inst" == "$valid_inst" ]]; then
return 0
fi
done
return 1
}
date=""
host=homestore
source=cloud21
while getopts ":h:d:s:" opt; do
case ${opt} in
d )
date=$OPTARG
;;
h )
host=$OPTARG
;;
s )
source=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ "${date}" == "" ] ; then
date="yesterday"
fi
echo "$date $host $source"
valid_hosts=('homestore' 'cloud21')
if ! is_valid "${host}" "${valid_hosts[@]}" ; then
echo "Host '${host}' is not valid"
usage
fi
valid_sources=('cloud21' 'cvttdata')
if ! is_valid ${source} ${valid_sources[@]} ; then
echo "Source '${source}' is not valid"
usage
fi
if [ "${host}" == "cloud21" ] ; then
SourceHost=cloud21.cvtt.vpn
SourceUser=cvtt
SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
elif [ "${host}" == "homestore" ]; then
SourceHost=homestore.cvtt.vpn
SourceUser=oleg
SourceRootDir=/works/cvtt/md_archive/crypto/${source}
else
usage
fi
echo "${SourceHost} ${SourceUser} ${SourceRootDir}"
SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
SourceFileZip="${SourceFile}.gz"
SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
TargetDir="/opt/jupyter_gpu/data/crypto_md"
TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
TargetFilePath="${TargetDir}/${TargetFile}"
# Tables=(bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min)
Tables=(bnbspot_ohlcv_1min coinbase_ohlcv_1min)
echo ${SourceFile}
tmp_dir=$(mktemp -d)
function cleanup {
cd ${HOME}
rm -rf ${tmp_dir}
}
trap cleanup EXIT
function download_file {
Cmd="rsync"
Cmd="${Cmd} -ahv"
if tty -s; then
Cmd="${Cmd} --progress=info2"
fi
Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
echo ${Cmd}
eval ${Cmd}
ls -lh ${tmp_dir}
Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
echo ${Cmd} && eval ${Cmd}
ls -lh ${tmp_dir}
rm -f ${TargetFilePath}
touch ${TargetFilePath}
for table in "${Tables[@]}"
do
Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
echo ${Cmd}
eval ${Cmd}
done
chmod 600 ${TargetFilePath}
ls -lh ${TargetFilePath}
}
download_file
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env bash
usage() {
echo -n "Usage: $0"
echo -n " [-h <host (homestore*/cloud21)>]"
echo -n " [-d <YYYYMMDD> (yesterday*)]"
echo -n " [-s <stocks comma separated>"
echo -n " [-t <target directory>"
echo
exit 1
}
if [[ "$(uname)" == "Darwin" ]]; then
# macOS
date='gdate'
else
date='date'
fi
# ------------------ Settings
md_date=""
host=homestore
stocks=COIN,GBTC,SQ
TargetDir="/opt/jupyter_gpu/data/eqty_md"
mkdir -p ${TargetDir}
Table=md_1min_bars
# ------------------ Settings
is_valid() {
local inst=$1
shift
local valid_instances=("$@")
for valid_inst in "${valid_instances[@]}";
do
if [[ "$inst" == "$valid_inst" ]]; then
return 0
fi
done
return 1
}
while getopts ":h:d:s:t:u" opt; do
case ${opt} in
d )
md_date=$OPTARG
;;
h )
host=$OPTARG
;;
s )
stocks=$OPTARG
;;
t )
TargetDir=$OPTARG
;;
u )
usage
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ "${md_date}" == "" ] ; then
md_date="yesterday"
fi
echo "$md_date $host $source"
valid_hosts=('homestore' 'cloud21')
if ! is_valid "${host}" "${valid_hosts[@]}" ; then
echo "Host '${host}' is not valid"
usage
fi
TargetFile=$(${date} -d ${md_date} "+%Y%m%d.eqty.mktdata.ohlcv.db")
TargetFilePath="${TargetDir}/${TargetFile}"
mv ${TargetFilePath} "${TargetFilePath}.saved.$(${date} '+%Y%m%d_%H%M%S')"
touch ${TargetFilePath}
# ---- temp dir
echo ${SourceFile}
tmp_dir=$(mktemp -d)
function cleanup {
cd ${HOME}
rm -rf ${tmp_dir}
}
trap cleanup EXIT
# ---- temp dir
if [ "${host}" == "cloud21" ] ; then
SourceHost=cloud21.cvtt.vpn
SourceUser=cvtt
SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
elif [ "${host}" == "homestore" ]; then
SourceHost=homestore.cvtt.vpn
SourceUser=oleg
SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
else
usage
fi
OLDIFS=$IFS
IFS=','
read -ra Stocks <<< "$stocks"
IFS=$OLDIFS
echo "${SourceHost} ${SourceUser} ${SourceRootDir} ${Stocks[@]}"
function download_file {
local SourceFilePath=${1}
Cmd="rsync"
Cmd="${Cmd} -ahv"
if tty -s; then
Cmd="${Cmd} --progress=info2"
fi
Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
echo ${Cmd}
eval ${Cmd}
ls -lh ${tmp_dir}
Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
echo ${Cmd} && eval ${Cmd}
DbFile=${tmp_dir}/${SourceFile}
# --- T E M P - for files older than 20240813
sqlite3 ${DbFile} <<EOF
DROP INDEX md_1min_bars_idx;
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_idx on md_1min_bars(tstamp, exchange_id, instrument_id);
EOF
# --- T E M P
Cmd="sqlite3 ${DbFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
echo ${Cmd}
eval ${Cmd}
}
echo "Loading files"
for stock in "${Stocks[@]}"; do
StockLetter="${stock:0:1}"
SourceFile=$(${date} -d ${md_date} "+%Y%m%d.${stock}.1min.db")
SourceFileZip="${SourceFile}.gz"
SourceFilePath=$(${date} -d ${md_date} "+${SourceRootDir}/${StockLetter}/${stock}/%Y/${SourceFileZip}")
echo ${SourceFilePath}
download_file ${SourceFilePath}
done
chmod 600 ${TargetFilePath}
ls -lh ${TargetFilePath}
# mkdir -p /tmp/aaa
# cp -r ${tmp_dir}/* /tmp/aaa/
@@ -1,156 +0,0 @@
#!/bin/bash
Script="${0} ${*}"
function usage {
echo "Usage: ${0} --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
exit 1
}
for arg in "${@}"
do
case ${arg} in
--restart)
Restart=Y
shift
;;
--stop)
Stop="Y"
shift
;;
--once)
Once="Y"
shift
;;
--inst_set=*)
InstrSet="${arg#*=}"
shift
;;
--sleep_sec=*)
SleepSec="${arg#*=}"
shift
;;
-*|--*)
usage
;;
*)
;;
esac
done
declare -A InstrSetInstrs
InstrSetInstrs[1]=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
if [ "${InstrSet}" == "" ]
then
usage
fi
if [ "${SleepSec}" == "" ]
then
SleepSec=3
fi
Instruments=${InstrSetInstrs[${InstrSet}]}
if [ "${Instruments}" == "" ]
then
echo "Unrecognized instrument settings: ${InstrSet}"
usage
fi
# -------------------- S e t t i n g s
RootDir=/home/cvtt/prod
# ConfigFile="${RootDir}/config/md_recorder.cfg"
ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
AdminPort="721${InstrSet}"
Exchange=BNBSPOT
DbCredKey=TSDB_COINBS_1
Name=MD_RECORDER_BNBSPOT
# -------------------- S e t t i n g s
LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
source ${HOME}/.pyenv/python3.10-venv/bin/activate
export PYTHONPATH=${RootDir}
Cmd="nohup"
Cmd="${Cmd} python3"
Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
Cmd="${Cmd} --config=${ConfigFile}"
Cmd="${Cmd} --active_exchanges=${Exchange}"
Cmd="${Cmd} --instruments=${Instruments}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_level=INFO"
Cmd="${Cmd} --log_file=${LogFile}"
Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
Cmd="${Cmd} &"
function start_it {
echo ${Cmd}
eval ${Cmd} &
sleep 10
}
function check_it {
if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
then
echo "${Script} doesn't respond. restarting..."
pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
start_it
fi
}
function kill_it {
pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut -d' ' -f2)
if [ "${pids}" != "" ]
then
echo "Killing ${pids} ..."
kill -9 ${pids}
fi
while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
do
echo "Shutting down localhost:${AdminPort}/shutdown ..."
timeout 5 curl -s localhost:${AdminPort}/shutdown
pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
done
}
if [ "${Stop}" == "Y" ] ; then
kill_it
exit 0
fi
if [ "${Once}" == "Y" ] ; then
start_it
exit 0
fi
if [ "${Restart}" == "Y" ]
then
kill_it
fi
start_it
while true
do
check_it
echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
sleep ${SleepSec}
done
@@ -1,161 +0,0 @@
#!/bin/bash
Script="${0} ${*}"
function usage {
echo "Usage: ${0} --inst_set=<setting #> [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
exit 1
}
for arg in "${@}"
do
case ${arg} in
--restart)
Restart=Y
shift
;;
--stop)
Stop="Y"
shift
;;
--once)
Once="Y"
shift
;;
--inst_set=*)
InstrSet="${arg#*=}"
shift
;;
--sleep_sec=*)
SleepSec="${arg#*=}"
shift
;;
-*|--*)
usage
;;
*)
;;
esac
done
declare -A InstrSetInstrs
InstrSetInstrs[1]=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
InstrSetInstrs[2]=PAIR-USDT-USD,PAIR-MATIC-USD
InstrSetInstrs[3]=PAIR-DOGE-USD,PAIR-AVAX-USD,
InstrSetInstrs[4]=PAIR-BCH-USD,PAIR-UNI-USD,PAIR-AAVE-USD,PAIR-LINK-USD,PAIR-SOL-USD
if [ "${InstrSet}" == "" ]
then
usage
fi
if [ "${SleepSec}" == "" ]
then
SleepSec=3
fi
Instruments=${InstrSetInstrs[${InstrSet}]}
if [ "${Instruments}" == "" ]
then
echo "Unrecognized instrument settings: ${InstrSet}"
usage
fi
# -------------------- S e t t i n g s
RootDir=/home/cvtt/prod
# ConfigFile="${RootDir}/config/md_recorder.cfg"
ConfigFile="http://cloud11.cvtt.vpn:6789/apps/md_recorder"
AdminPort="720${InstrSet}"
Exchange=COINBASE_AT
DbCredKey=TSDB_COINBS_1
Name=MD_RECORDER_COINBASE
# -------------------- S e t t i n g s
LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}_${InstrSet}.log"
source ${HOME}/.pyenv/python3.10-venv/bin/activate
export PYTHONPATH=${RootDir}
Cmd="nohup"
Cmd="${Cmd} python3"
Cmd="${Cmd} ${RootDir}/cvttpy/apps/md/md_recorder.py"
Cmd="${Cmd} --config=${ConfigFile}"
Cmd="${Cmd} --active_exchanges=${Exchange}"
Cmd="${Cmd} --instruments=${Instruments}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_level=INFO"
Cmd="${Cmd} --log_file=${LogFile}"
Cmd="${Cmd} --db_credentials_key=${DbCredKey}"
Cmd="${Cmd} &"
function start_it {
echo ${Cmd}
eval ${Cmd} &
sleep 10
}
function check_it {
if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
then
echo "${Script} doesn't respond. restarting..."
pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
start_it
fi
}
function kill_it {
pids=$(ps -ef |grep "inst_set=${InstrSet}" |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut -d' ' -f2)
if [ "${pids}" != "" ]
then
echo "Killing ${pids} ..."
kill -9 ${pids}
fi
while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
do
echo "Shutting down localhost:${AdminPort}/shutdown ..."
timeout 5 curl -s localhost:${AdminPort}/shutdown
pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
done
}
if [ "${Stop}" == "Y" ] ; then
kill_it
exit 0
fi
if [ "${Once}" == "Y" ] ; then
start_it
exit 0
fi
if [ "${Restart}" == "Y" ]
then
kill_it
fi
start_it
while true
do
check_it
echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
sleep ${SleepSec}
done
@@ -1,128 +0,0 @@
#!/bin/bash
Script="${0} ${*}"
function usage {
echo "Usage: ${0} [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
exit 1
}
for arg in "${@}"
do
case ${arg} in
--restart)
Restart=Y
shift
;;
--stop)
Stop="Y"
shift
;;
--once)
Once="Y"
shift
;;
--sleep_sec=*)
SleepSec="${arg#*=}"
shift
;;
-*|--*)
usage
;;
*)
;;
esac
done
if [ "${SleepSec}" == "" ]
then
SleepSec=5
fi
RootDir=/home/cvtt/prod
CfgSvcDir=${RootDir}/config_service
export PYTHONPATH="${RootDir}:${PYTHONPATH}"
Name=config_server
ServerPy="${RootDir}/cvttpy/apps/utils/config_server.py"
ServicePort=6789
ServiceDir=${CfgSvcDir}/data
LogDir=${RootDir}/logs
source ${HOME}/.pyenv/python3.10-venv/bin/activate
export PYTHONPATH="${RootDir}:${PYTHONPATH}"
Cmd="nohup"
Cmd="${Cmd} python3"
Cmd="${Cmd} ${ServerPy}"
Cmd="${Cmd} --port=${ServicePort}"
Cmd="${Cmd} --root=${ServiceDir}"
Cmd="${Cmd} --log_file=${LogDir}/%T.config_service.log"
function start_it {
echo ${Cmd}
eval ${Cmd} &
sleep 10
}
function check_it {
if !(timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
then
echo "${Script} doesn't respond. restarting..."
pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
start_it
fi
}
function kill_it {
pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut -d' ' -f2)
if [ "${pids}" != "" ]
then
echo "Killing ${pids} ..."
kill -9 ${pids}
fi
while (timeout 3 curl -s localhost:${ServicePort}/ping > /dev/null)
do
echo "Shutting down localhost:${ServicePort}/__shutdown__ ..."
timeout 5 curl -s localhost:${ServicePort}/__shutdown__
pids=$(ps -ef | grep "port=${ServicePort}" | grep "root=${ServiceDir}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
done
}
if [ "${Stop}" == "Y" ] ; then
kill_it
exit 0
fi
if [ "${Once}" == "Y" ] ; then
start_it
exit 0
fi
if [ "${Restart}" == "Y" ]
then
kill_it
fi
start_it
while true
do
check_it
echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
sleep ${SleepSec}
done
@@ -1,144 +0,0 @@
#!/bin/bash
Script="${0} ${*}"
function usage {
echo "Usage: ${0} [--restart] [--stop] [--once] [--sleep_sec=<num_seconds>]"
exit 1
}
for arg in "${@}"
do
case ${arg} in
--restart)
Restart=Y
shift
;;
--stop)
Stop="Y"
shift
;;
--once)
Once="Y"
shift
;;
--inst_set=*)
InstrSet="${arg#*=}"
shift
;;
--sleep_sec=*)
SleepSec="${arg#*=}"
shift
;;
-*|--*)
usage
;;
*)
;;
esac
done
if [ "${SleepSec}" == "" ]
then
SleepSec=3
fi
# -------------------- S e t t i n g s
ConfigServiceHost=cloud11.cvtt.vpn
ConfigServicePort=6789
RootDir=/home/cvtt/prod
ConfigFile="http://${ConfigServiceHost}:${ConfigServicePort}/apps/executor_app"
AdminPort="7210"
Exchange=COINBASE_AT
Name=EXECUTOR_APP
# -------------------- S e t t i n g s
LogFile="${RootDir}/logs/$(date '+%Y%m%d_%H%M%S').${Name}.log"
source ${HOME}/.pyenv/python3.10-venv/bin/activate
export PYTHONPATH=${RootDir}
Cmd="nohup"
Cmd="${Cmd} python3"
Cmd="${Cmd} ${RootDir}/cvttpy/apps/executor_app.py"
Cmd="${Cmd} --config=${ConfigFile}"
Cmd="${Cmd} --active_exchanges=${Exchange}"
Cmd="${Cmd} --admin_port=${AdminPort}"
Cmd="${Cmd} --log_level=INFO"
Cmd="${Cmd} --log_file=${LogFile}"
Cmd="${Cmd} &"
function start_it {
echo ${Cmd}
eval ${Cmd} &
sleep 10
}
function check_it {
if !(timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
then
echo "${Script} doesn't respond. restarting..."
pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
start_it
fi
}
function kill_it {
pids=$(ps -ef |grep $(basename ${0}) |grep -v ${$} |grep -v grep |tr -s ' ' |cut -d' ' -f2)
if [ "${pids}" != "" ]
then
echo "Killing ${pids} ..."
kill -9 ${pids}
fi
while (timeout 3 curl -s localhost:${AdminPort}/ping > /dev/null)
do
echo "Shutting down localhost:${AdminPort}/shutdown ..."
timeout 5 curl -s localhost:${AdminPort}/shutdown
sleep 3
pids=$(ps -ef | grep "admin_port=${AdminPort}" | grep -v grep | tr -s ' ' |cut -d' ' -f2)
echo pids=${pids}
if [ "${pids}" != "" ]
then
kill -9 ${pids}
fi
done
}
if [ "${Stop}" == "Y" ] ; then
kill_it
exit 0
fi
if [ "${Once}" == "Y" ] ; then
start_it
exit 0
fi
if [ "${Restart}" == "Y" ]
then
kill_it
fi
start_it
while true
do
check_it
echo "${Script} is checked" | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
sleep ${SleepSec}
done
+27 -24
View File
@@ -1,4 +1,4 @@
#!/bin/sh #!/bin/bash
echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]' echo $0 $* | /usr/bin/ts '[%Y-%m-%d %H:%M:%S]'
RootDir="${HOME}/prod" RootDir="${HOME}/prod"
@@ -10,32 +10,24 @@ StatusChannel=Status-CVTT
AlertChannel=Alerts-CVTT AlertChannel=Alerts-CVTT
Sender=${RootDir}/ops/utils/send_mmost.sh Sender=${RootDir}/ops/utils/send_mmost.sh
# ----- For DEBUGGING
# Sender=cat # Sender=cat
# StatusChannel= # StatusChannel=
Hosts= get_user_hosts() {
Hosts="${Hosts} cloud11.cvtt.vpn" local User=${1}
Hosts="${Hosts} cloud15.cvtt.vpn" local Domain=${2}
Hosts="${Hosts} cloud16.cvtt.vpn"
Hosts="${Hosts} cloud17.cvtt.vpn"
Hosts="${Hosts} cloud21.cvtt.vpn"
Hosts="${Hosts} cloud22.cryptovaltrading.com"
Hosts="${Hosts} cvttdata.cvtt.vpn" Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
Hosts="${Hosts} cryptoval2.cvtt.vpn" Cmd="${Cmd} | ${HOME}/bin/hjson -j"
Hosts="${Hosts} cryptoval3.cvtt.vpn" Cmd="${Cmd} | jq -r"
Cmd="${Cmd} --arg domain \"${Domain}\""
Cmd="${Cmd} --arg usr \"${User}\""
Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
Hosts="${Hosts} homestore.cvtt.vpn" Cmd="${Cmd} | sed 's/\$/.${Domain}/'"
Hosts="${Hosts} nsbackup.sheynin.home" eval ${Cmd}
Hosts="${Hosts} dtvmhost.cvtt.vpn" }
Hosts="${Hosts} ops-server.cvtt.vpn"
Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
Metrics=()
TempFiles=
function cleanup { function cleanup {
echo Cleaing up temporary files: ${TempFiles} echo Cleaing up temporary files: ${TempFiles}
@@ -61,12 +53,14 @@ function space_alert() {
} }
function storage_check() { function storage_check() {
local Hosts=("${@}")
result_lines=() result_lines=()
declare -a SingleMeas declare -a SingleMeas
for host in ${Hosts} for host in ${Hosts[@]}
do do
echo "storage_check host=${host}" >&2
if [[ "${host}" == *"cryptovaltrading.com" ]]; then if [[ "${host}" == *"cryptovaltrading.com" ]]; then
port=7822 port=7822
else else
@@ -98,13 +92,22 @@ function storage_check() {
done done
} }
User=oleg
Metrics=()
TempFiles=
Hosts=()
for Domain in cvtt.vpn cryptovaltrading.com ; do
Hosts=("${Hosts[@]}" "$(get_user_hosts ${User} ${Domain})")
done
tmpfile=$(mktemp) tmpfile=$(mktemp)
TempFiles="${TempFiles} ${tmpfile}" TempFiles="${TempFiles} ${tmpfile}"
tmpfile2=$(mktemp) tmpfile2=$(mktemp)
TempFiles="${TempFiles} ${tmpfile2}" TempFiles="${TempFiles} ${tmpfile2}"
storage_check > ${tmpfile2} storage_check "${Hosts[@]}" > ${tmpfile2}
echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile} echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
echo >> ${tmpfile} echo >> ${tmpfile}
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
usage() {
echo -n "Usage: ${0}"
echo
exit 1
}
RootDir="${HOME}/prod"
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
export PYTHONPATH=${RootDir}
StatusChannel="MD-Status"
Sender=${RootDir}/ops/utils/send_mmost.sh
# ----- For DEBUGGING
# Sender=cat
# StatusChannel=
run_checklist() {
yr=$(date -d 'yesterday' '+%Y')
mn=$(date -d 'yesterday' '+%m')
declare -A Commands
Commands=(
["homestore:cloud21"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["homestore:cvttdata"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
)
for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
Cmd=${Commands[${name}]}
echo "------- ${name}"
echo ${Cmd}
eval ${Cmd}
done
}
tmpfile=$(mktemp)
function cleanup {
cd ${HOME}
rm -f ${tmpfile}
}
trap cleanup EXIT
echo "## :hearts: CRYPTO MD HEALTH CHECK" >> ${tmpfile}
echo '```' >> ${tmpfile}
run_checklist >> ${tmpfile}
echo '```' >> ${tmpfile}
cat ${tmpfile} | ${Sender} ${StatusChannel}
cat $tmpfile
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
usage() {
echo -n "Usage: ${0}"
echo
exit 1
}
RootDir="${HOME}/prod"
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
export PYTHONPATH=${RootDir}
StatusChannel="MD-Status"
Sender=${RootDir}/ops/utils/send_mmost.sh
# ----- For DEBUGGING
# Sender=cat
# StatusChannel=
run_checklist() {
yr=$(date -d 'yesterday' '+%Y')
declare -A Commands
Commands["homestore"]="ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
Commands["homestore"]+="; echo"
Commands["homestore"]+="; ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
Commands["cloud21"]+="; echo"
Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
Cmd=${Commands[${name}]}
echo "------- ${name}"
echo ${Cmd}
eval ${Cmd}
done
}
# run_checklist
tmpfile=$(mktemp)
function cleanup {
cd ${HOME}
rm -f ${tmpfile}
}
trap cleanup EXIT
echo "## :fire: EQUITY MD HEALTH CHECK" >> ${tmpfile}
echo '```' >> ${tmpfile}
run_checklist >> ${tmpfile}
echo '```' >> ${tmpfile}
cat ${tmpfile} | ${Sender} ${StatusChannel}
exit
# homestore
echo homestore
ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
echo
ssh oleg@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
echo cloud21
ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3
echo
ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3
echo "------ gpushnik"
ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'
+49
View File
@@ -0,0 +1,49 @@
#!/bin/bash
# ---------------------
# user "git" owned remote repository
#
# must run on remote server
# change GitTopDir, HostName and RemoteName
# ---------------------
# R e s u l t E x a m p l e:
# sudo bash -c 'mkdir /opt/store/git/cvtt2/tests.git && cd /opt/store/git/cvtt2/tests.git && git init --bare && chown -R git:git /opt/store/git/cvtt2/tests.git'
# ---------------------
#
# ----- S e t t i n g s
GitTopDir=/ops/store/git
HostName=cloud21.cvtt.vpn
RemoteName=origin
# ----- S e t t i n g s
usage() {
echo "Usage: ${0} <repo_name> [<repo_parent_dir> *${GitTopDir}/cvtt2]"
exit 1
}
RepoName=${1}
if [ "" == "${RepoName}" ] ; then
usage
fi
RepoParentDir=${GitTopDir}/cvtt2
if [ "${2}" != "" ] ; then
RepoParentDir=${2}
fi
RepoDir="${RepoParentDir}/${RepoName}.git"
Cmd="sudo bash -c 'mkdir ${RepoDir}"
Cmd="${Cmd} && cd ${RepoDir}"
Cmd="${Cmd} && git init --bare "
Cmd="${Cmd} && chown -R git:git ${RepoDir}'"
echo ${Cmd}
eval ${Cmd} || exit 1
echo "====================="
echo "Repository is created. Use this command to add it:"
echo " git remote add ${RemoteName} git@${HostName}:${RepoDir}"
echo "====================="