From 95056f252a05205c001179e47663008b22a7c0e7 Mon Sep 17 00:00:00 2001 From: Cryptoval Trading Technologies Date: Sun, 16 Nov 2025 20:34:00 +0000 Subject: [PATCH] progress --- .env | 6 +++--- docker-compose.yml | 8 ++------ retrofit/crypto_missing_dates.sh | 2 -- retrofit/crypto_sim_missing_dates.sh | 2 +- retrofit/readme.md | 3 +++ retrofit/retrofit_crypto.sh | 2 +- retrofit/retrofit_equity.sh | 2 +- retrofit/run_retrofit.sh | 2 +- retrofit/sim_crypto_retrofit.sh | 2 +- retrofit/sim_equity_retrofit.sh | 2 +- run.sh | 12 +++--------- 11 files changed, 17 insertions(+), 26 deletions(-) create mode 100644 retrofit/readme.md diff --git a/.env b/.env index bfa3afa..252c415 100644 --- a/.env +++ b/.env @@ -1,7 +1,7 @@ CREDS_FILE=/config/.creds # ==================================== daily_crypto_md -CRYPTO_IMAGE_VERSION=0.0.3 +CRYPTO_IMAGE_VERSION=0.0.4 CRYPTO_CONFIG_FILE=http://cloud16.cvtt.vpn:6789/apps/utils/mktdata/daily_md_crypto CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md CRYPTO_DB_SCHEMAS=coinbase,bnbspot @@ -33,14 +33,14 @@ EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/c # EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test" # ==================================== daily_sim_crypto -SIM_CRYPTO_IMAGE_VERSION=0.0.2 +SIM_CRYPTO_IMAGE_VERSION=0.0.3 DAILY_SIM_CRYPTO_DATE= #DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn #DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto -DAILY_SIM_CRYPTO_DB_SOURCE=cloud29 +DAILY_SIM_CRYPTO_DB_SOURCES=cloud28,cloud29 DAILY_SIM_CRYPTO_RSYNC_TARGETS= DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim" DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim" diff --git a/docker-compose.yml b/docker-compose.yml index 156c0d3..308d570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,8 +119,8 @@ services: SCRIPT: /prepare_crypto_simdata.sh DATE: ${DAILY_SIM_CRYPTO_DATE:-} SOURCE_HOST: ${DAILY_SIM_CRYPTO_SOURCE_HOST:-cloud21.cvtt.vpn} - SOURCE_ROOT_DIR: ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto} - DB_SOURCE: ${DAILY_SIM_CRYPTO_DB_SOURCE:-cloud28} + SOURCE_ROOT_DIR: ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/works/cvtt/md_archive/crypto} + DB_SOURCES: ${DAILY_SIM_CRYPTO_DB_SOURCES:-cloud28,cloud29} OUTPUT_DIR: /tmp RSYNC_TARGETS: ${DAILY_SIM_CRYPTO_RSYNC_TARGETS} @@ -152,7 +152,3 @@ services: - ./logs:/logs - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro -networks: - default: - name: daily_mktdata_default - external: true diff --git a/retrofit/crypto_missing_dates.sh b/retrofit/crypto_missing_dates.sh index 195f9ea..60cf446 100755 --- a/retrofit/crypto_missing_dates.sh +++ b/retrofit/crypto_missing_dates.sh @@ -1,7 +1,5 @@ #!/bin/bash -# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 - SourceHost=${1} if [ "${SourceHost}" == "" ]; then echo "Usage: ${0} " diff --git a/retrofit/crypto_sim_missing_dates.sh b/retrofit/crypto_sim_missing_dates.sh index 3b31be3..ce26a3e 100755 --- a/retrofit/crypto_sim_missing_dates.sh +++ b/retrofit/crypto_sim_missing_dates.sh @@ -9,7 +9,7 @@ DEFAULT_FGLOB=*.crypto_sim_md.db.gz Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn} FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}} -BackDaysCount=${BACKDAYS_COUNT:-50} +BackDaysCount=${BACKDAYS_COUNT:-30} Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}" echo $Cmd diff --git a/retrofit/readme.md b/retrofit/readme.md new file mode 100644 index 0000000..521b4e1 --- /dev/null +++ b/retrofit/readme.md @@ -0,0 +1,3 @@ + ./run_retrofit.sh retrofit_crypto.sh cloud28 + + ./run_retrofit.sh retrofit_crypto.sh cloud29 diff --git a/retrofit/retrofit_crypto.sh b/retrofit/retrofit_crypto.sh index 6656673..beafb3b 100755 --- a/retrofit/retrofit_crypto.sh +++ b/retrofit/retrofit_crypto.sh @@ -26,7 +26,7 @@ for dt in ${DATES}; do echo $dt Cmd="CRYPTO_MD_DATE=${dt}" Cmd+=" /usr/bin/docker compose" - Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" + Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" Cmd+=" daily_crypto_${SourceHost}" echo ${Cmd} diff --git a/retrofit/retrofit_equity.sh b/retrofit/retrofit_equity.sh index 3f0b2b0..6ad773b 100755 --- a/retrofit/retrofit_equity.sh +++ b/retrofit/retrofit_equity.sh @@ -19,7 +19,7 @@ for dt in ${DATES}; do echo $dt Cmd="EQUITY_HBAR_DATE=${dt}" Cmd+=" /usr/bin/docker compose" - Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" + Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" Cmd+=" daily_equity_hbar" diff --git a/retrofit/run_retrofit.sh b/retrofit/run_retrofit.sh index e00b19c..f68d90d 100755 --- a/retrofit/run_retrofit.sh +++ b/retrofit/run_retrofit.sh @@ -5,7 +5,7 @@ shift Params=${*} ScriptDir=$(realpath $(dirname ${0})) -LogDir=${ScriptDir}/log +LogDir=${ScriptDir}/logs mkdir -p ${LogDir} Cmd="nohup ${ScriptDir}/${Script} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &" diff --git a/retrofit/sim_crypto_retrofit.sh b/retrofit/sim_crypto_retrofit.sh index f5cb0cc..7c42ac0 100755 --- a/retrofit/sim_crypto_retrofit.sh +++ b/retrofit/sim_crypto_retrofit.sh @@ -19,7 +19,7 @@ DATES=$($Cmd) for dt in ${DATES}; do Cmd="DAILY_SIM_CRYPTO_DATE=${dt}" Cmd+=" /usr/bin/docker compose" - Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" + Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" Cmd+=" daily_sim_crypto" echo ${Cmd} diff --git a/retrofit/sim_equity_retrofit.sh b/retrofit/sim_equity_retrofit.sh index f98037b..cda58ab 100755 --- a/retrofit/sim_equity_retrofit.sh +++ b/retrofit/sim_equity_retrofit.sh @@ -20,7 +20,7 @@ DATES=$($Cmd) for dt in ${DATES}; do Cmd="DAILY_SIM_EQUITY_DATE=${dt}" Cmd+=" /usr/bin/docker compose" - Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" + Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" Cmd+=" daily_sim_equity" echo ${Cmd} diff --git a/run.sh b/run.sh index e802d18..ed3ad07 100755 --- a/run.sh +++ b/run.sh @@ -6,19 +6,13 @@ COMPOSE_FILE="/works/docker/daily_mktdata/docker-compose.yml" PROJECT_DIR="/works/docker/daily_mktdata" PROJECT_NAME="daily_mktdata" NETWORK_NAME="daily_mktdata_default" -SERVICE="${1:?usage: run_mktdata.sh }" +SERVICE="${1:?usage: $0 }" -# 1) Make sure the external network exists -docker network inspect "${NETWORK_NAME}" >/dev/null 2>&1 || docker network create "${NETWORK_NAME}" - -# 2) Show quick status (optional but useful in logs) -docker network inspect "${NETWORK_NAME}" --format '{{.Name}} {{.Id}} {{.Driver}} {{.Attachable}}' - -# 3) Run exactly one service without touching others +# Run the service as a one-off job and auto-remove container exec docker compose \ --project-name "${PROJECT_NAME}" \ --project-directory "${PROJECT_DIR}" \ -f "${COMPOSE_FILE}" \ - up --no-deps --force-recreate \ + run --rm --no-deps \ "${SERVICE}"