prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress with retrofit

Browse Source
This commit is contained in:
Cryptoval Trading Technologies 2025-11-28 19:45:31 +00:00
parent 95056f252a
commit b3fb3e01b3
7 changed files with 80 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.env
View File

@ -1,7 +1,7 @@
CREDS_FILE=/config/.creds
# ==================================== daily_crypto_md
CRYPTO_IMAGE_VERSION=0.0.4
CRYPTO_IMAGE_VERSION=0.0.6
CRYPTO_CONFIG_FILE=http://cloud16.cvtt.vpn:6789/apps/utils/mktdata/daily_md_crypto
CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
CRYPTO_DB_SCHEMAS=coinbase,bnbspot
@ -9,7 +9,7 @@ CRYPTO_DB_SCHEMAS=coinbase,bnbspot
CRYPTO_MD_DATE=
CRYPTO_RSYNC_TARGETS=
CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/crypto"
CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
CRYPTO_ADD_ARGS="--log_stdout"
@ -20,7 +20,7 @@ CRYPTO_PYTHON_DEBUG_WAIT=Y
CRYPTO_PYTHON_DEBUG_PORT=45678
# ==================================== daily_equity_md
EQUITY_IMAGE_VERSION=0.0.2
EQUITY_IMAGE_VERSION=0.0.7
EQUITY_CONTAINER_NAME_PFX=daily_equity_md
EQUITY_HBAR_NUM_SYMBOLS=
EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA
@ -28,22 +28,19 @@ EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA
EQUITY_HBAR_DATE=
EQUITY_RSYNC_TARGETS=
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md"
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md"
# ==================================== daily_sim_crypto
SIM_CRYPTO_IMAGE_VERSION=0.0.3
DAILY_SIM_CRYPTO_DATE=
#DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn
#DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto
DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn
DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto
DAILY_SIM_CRYPTO_DB_SOURCES=cloud28,cloud29
DAILY_SIM_CRYPTO_RSYNC_TARGETS=
DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim"
DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim"
DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/crypto/sim"
# ==================================== daily_sim_equity
@ -51,10 +48,10 @@ SIM_EQUITY_IMAGE_VERSION=0.0.2
DAILY_SIM_EQUITY_DATE=
DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn
DAILY_SIM_EQUITY_SOURCE_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
DAILY_SIM_EQUITY_SOURCE_DIR=/works/cvtt/md_archive/equity/alpaca_md
DAILY_SIM_EQUITY_SYMBOL_LIST=COIN #,CRCL,GBTC,HOOD,MSTR
DAILY_SIM_EQUITY_RSYNC_TARGETS=
DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim"
DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim"
DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim"

1
.gitignore vendored
View File

@ -3,3 +3,4 @@ logs/
nohup.out
.history/
__OLD__/
__DELETE__/

24
retrofit/crypto_missing_dates.sh
View File

@ -1,11 +1,5 @@
#!/bin/bash
SourceHost=${1}
if [ "${SourceHost}" == "" ]; then
echo "Usage: ${0} <source_host>"
exit 1
fi
function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
source ${function_file}
@ -15,8 +9,23 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
BackDaysCount=${BACKDAYS_COUNT:-30}
for SourceHost in cloud28 cloud29; do
echo "============= ${SourceHost}"
Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
echo $Cmd
Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
DATES=$($Cmd)
for dt in ${DATES}; do
echo $dt
done
done
DEFAULT_FGLOB=*.crypto_sim_md.db.gz
FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
echo "============= sim"
Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
echo $Cmd
DATES=$($Cmd)
@ -24,3 +33,4 @@ DATES=$($Cmd)
for dt in ${DATES}; do
echo $dt
done

22
retrofit/crypto_sim_missing_dates.sh
View File

@ -1,22 +0,0 @@
#!/bin/bash
# DAILY_SIM_CRYPTO_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_crypto
#
#
source $(realpath $(dirname $0))/get_retrofit_dates.sh
DEFAULT_FGLOB=*.crypto_sim_md.db.gz
Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
BackDaysCount=${BACKDAYS_COUNT:-30}
Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
echo $Cmd
DATES=$($Cmd)
for dt in ${DATES}; do
echo $dt
done

24
retrofit/equity_missing_dates.sh
View File

@ -11,6 +11,7 @@ Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
BackDaysCount=${BACKDAYS_COUNT:-50}
echo "======================= Alpaca MD"
Cmd="get_equity_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}"
echo $Cmd
@ -19,3 +20,26 @@ DATES=$($Cmd)
for dt in ${DATES}; do
echo $dt
done
# ======================================
DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
echo "======================= sim"
Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
echo $Cmd
DATES=$($Cmd)
for dt in ${DATES}; do
weekday=$(date -d "$dt" +%u)
(( weekday >= 6 )) && continue
is_equity_business_date "$dt"
rc=$?
if (( rc != 0 )); then
continue
fi
fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
echo $fdt
done

25
retrofit/equity_sim_missing_dates.sh
View File

@ -1,25 +0,0 @@
#!/bin/bash
#
# DAILY_SIM_EQUITY_DATE=20250723 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_sim_equity
#
source $(realpath $(dirname $0))/get_retrofit_dates.sh
DEFAULT_FGLOB=*.alpaca_sim_md.db.gz
Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
BackDaysCount=${BACKDAYS_COUNT:-50}
Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/sim ${FileGlob} ${BackDaysCount}"
echo $Cmd
DATES=$($Cmd)
for dt in ${DATES}; do
weekday=$(date -d "$dt" +%u)
(( weekday >= 6 )) && continue
fdt=$(date -d "$dt" +%Y-%m-%d 2>/dev/null)
echo $fdt
done

31
retrofit/get_retrofit_dates.sh
View File

@ -61,6 +61,30 @@ get_retrofit_dates() {
printf '%s\n' "${missing_dates[@]}"
}
is_equity_business_date() {
local ymd=${1:-}
if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then
echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2
return 2
fi
local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}"
local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}"
local response
if ! response=$(curl -fsS "$url" 2>/dev/null); then
echo "is_equity_business_date: failed to query trading calendar API" >&2
return 1
fi
if [[ "$response" =~ \[\s*\] ]]; then
return 1
fi
return 0
}
get_equity_retrofit_dates() {
local host=${1:-}
local root_dir=${2:-}
@ -107,10 +131,15 @@ get_equity_retrofit_dates() {
fi
local -a missing_dates=()
local idx dt tmp
local idx dt tmp rc
for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
dt=${target_dates[$idx]}
[[ -n "${existing_dates[$dt]:-}" ]] && continue
is_equity_business_date "$dt"
rc=$?
if (( rc != 0 )); then
continue
fi
if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then
echo "invalid date \"$dt\"" >&2
return 1