prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress and fixes

Browse Source
This commit is contained in:
Cryptoval Trading Technologies 2026-01-07 22:16:00 +00:00
parent ee4d264ad2
commit a821e8f7a8
5 changed files with 18 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env
View File

@ -45,7 +45,7 @@ DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.c
# ==================================== daily_sim_equity # ==================================== daily_sim_equity
SIM_EQUITY_IMAGE_VERSION=0.0.2 SIM_EQUITY_IMAGE_VERSION=0.0.5
DAILY_SIM_EQUITY_DATE= DAILY_SIM_EQUITY_DATE=
DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn

11
docker-compose.yml
View File

@ -5,9 +5,10 @@
#SIM_CRYPTO_IMAGE_VERSION=0.0.2 #SIM_CRYPTO_IMAGE_VERSION=0.0.2
#SIM_EQUITY_IMAGE_VERSION=0.0.2 #SIM_EQUITY_IMAGE_VERSION=0.0.2
name: daily_mktdata_services
services: services:
daily_crypto_cloud28: daily_crypto_cloud28:
image: cloud21.cvtt.vpn:5500/daily_crypto_md:${CRYPTO_IMAGE_VERSION} image: cloud16.cvtt.vpn:5543/daily_crypto_md:${CRYPTO_IMAGE_VERSION}
container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud28 container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud28
user: ${CVTT_USER:-1001:1001} user: ${CVTT_USER:-1001:1001}
pull_policy: always pull_policy: always
@ -37,7 +38,7 @@ services:
- 45678:5678 - 45678:5678
daily_crypto_cloud29: daily_crypto_cloud29:
image: cloud21.cvtt.vpn:5500/daily_crypto_md:${CRYPTO_IMAGE_VERSION} image: cloud16.cvtt.vpn:5543/daily_crypto_md:${CRYPTO_IMAGE_VERSION}
container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud29 container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud29
user: ${CVTT_USER:-1001:1001} user: ${CVTT_USER:-1001:1001}
pull_policy: always pull_policy: always
@ -67,7 +68,7 @@ services:
ports: ports:
- 45679:5678 - 45679:5678
daily_equity_hbar: daily_equity_hbar:
image: cloud21.cvtt.vpn:5500/daily_equity_md:${EQUITY_IMAGE_VERSION} image: cloud16.cvtt.vpn:5543/daily_equity_md:${EQUITY_IMAGE_VERSION}
container_name: alpaca_md_hbar_loader container_name: alpaca_md_hbar_loader
user: ${CVTT_USER:-1001:1001} user: ${CVTT_USER:-1001:1001}
pull_policy: always pull_policy: always
@ -89,7 +90,7 @@ services:
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
daily_equity_qat: daily_equity_qat:
image: cloud21.cvtt.vpn:5500/daily_equity_md:${EQUITY_IMAGE_VERSION} image: cloud16.cvtt.vpn:5543/daily_equity_md:${EQUITY_IMAGE_VERSION}
container_name: alpaca_md_qat_loader container_name: alpaca_md_qat_loader
user: ${CVTT_USER:-1001:1001} user: ${CVTT_USER:-1001:1001}
pull_policy: always pull_policy: always
@ -132,7 +133,7 @@ services:
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
daily_sim_equity: daily_sim_equity:
image: cloud21.cvtt.vpn:5500/daily_sim:${SIM_EQUITY_IMAGE_VERSION} image: cloud16.cvtt.vpn:5543/daily_sim:${SIM_EQUITY_IMAGE_VERSION}
container_name: daily_sim_equity container_name: daily_sim_equity
user: ${CVTT_USER:-1001:1001} user: ${CVTT_USER:-1001:1001}
pull_policy: always pull_policy: always

5
retrofit/crypto_retrofit.sh
View File

@ -21,11 +21,12 @@ run_retrofit() {
mkdir -p ${LogDir} mkdir -p ${LogDir}
Cmd="nohup ${RunDir}/${Script} ${Source} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.${Source}.log 2>&1 &" Cmd="nohup ${RunDir}/${Script} ${Source} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.${Source}.log 2>&1 &"
print_and_run ${Cmd} echo ${Cmd}
eval ${Cmd}
} }
for src in cloud28 cloud29; do for src in cloud28 cloud29; do
run_retrofit retrofit_crypto.sh $src run_retrofit retrofit_crypto.sh $src
done done
run_retrofit sim_crypto_retrofit.sh #run_retrofit sim_crypto_retrofit.sh

2
retrofit/equity_missing_dates.sh
View File

@ -12,7 +12,7 @@ FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
BackDaysCount=${BACKDAYS_COUNT:-50} BackDaysCount=${BACKDAYS_COUNT:-50}
echo "======================= Alpaca MD" echo "======================= Alpaca MD"
Cmd="get_equity_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}" Cmd="get_equity_retrofit_dates ${Source} /works/cvtt/md_archive/equity/alpaca_md/*/N/NVDA/${FileGlob} ${BackDaysCount}"
echo $Cmd echo $Cmd
DATES=$($Cmd) DATES=$($Cmd)

13
retrofit/get_retrofit_dates.sh
View File

@ -86,12 +86,12 @@ is_equity_business_date() {
} }
get_equity_retrofit_dates() { get_equity_retrofit_dates() {
# get_equity_retrofit_dates cvtt@cloud21.cvtt.vpn /works/cvtt/md_archive/equity/alpaca_md/*/N/NVDA/*.alpaca_1m_bars.db.gz 50
local host=${1:-} local host=${1:-}
local root_dir=${2:-} local filename_glob=${2:-}
local filename_glob=${3:-} local day_count=${3:-}
local day_count=${4:-}
if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then if [[ -z "$host" || -z "$filename_glob" || -z "$day_count" ]]; then
echo "usage: get_equity_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2 echo "usage: get_equity_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
return 1 return 1
fi fi
@ -118,13 +118,14 @@ get_equity_retrofit_dates() {
declare -A existing_dates=() declare -A existing_dates=()
local remote_cmd remote_output entry local remote_cmd remote_output entry
printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1 %s" "$root_dir" "$filename_glob" remote_cmd="ls ${filename_glob}"
remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true) remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
if [[ -n "$remote_output" ]]; then if [[ -n "$remote_output" ]]; then
while IFS= read -r entry; do while IFS= read -r entry; do
[[ -z "$entry" ]] && continue [[ -z "$entry" ]] && continue
if [[ "$entry" =~ ^([0-9]{8}) ]]; then fname=$(basename $entry)
if [[ "$fname" =~ ^([0-9]{8}) ]]; then
existing_dates["${BASH_REMATCH[1]}"]=1 existing_dates["${BASH_REMATCH[1]}"]=1
fi fi
done <<<"$remote_output" done <<<"$remote_output"