progress

.env

@@ -1,7 +1,7 @@
 CREDS_FILE=/config/.creds
 
 # ==================================== daily_crypto_md
-CRYPTO_IMAGE_VERSION=0.0.3
+CRYPTO_IMAGE_VERSION=0.0.4
 CRYPTO_CONFIG_FILE=http://cloud16.cvtt.vpn:6789/apps/utils/mktdata/daily_md_crypto
 CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
 CRYPTO_DB_SCHEMAS=coinbase,bnbspot
@@ -33,14 +33,14 @@ EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/c
 # EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"
 
 # ==================================== daily_sim_crypto
-SIM_CRYPTO_IMAGE_VERSION=0.0.2
+SIM_CRYPTO_IMAGE_VERSION=0.0.3
 DAILY_SIM_CRYPTO_DATE=
 
 #DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn
 #DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto
 DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn
 DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto
-DAILY_SIM_CRYPTO_DB_SOURCE=cloud29
+DAILY_SIM_CRYPTO_DB_SOURCES=cloud28,cloud29
 DAILY_SIM_CRYPTO_RSYNC_TARGETS=
 DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim"
 DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim"

docker-compose.yml

@@ -119,8 +119,8 @@ services:
           SCRIPT:            /prepare_crypto_simdata.sh
           DATE:              ${DAILY_SIM_CRYPTO_DATE:-}
           SOURCE_HOST:       ${DAILY_SIM_CRYPTO_SOURCE_HOST:-cloud21.cvtt.vpn}
-          SOURCE_ROOT_DIR:   ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto}
+          SOURCE_ROOT_DIR:   ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/works/cvtt/md_archive/crypto}
-          DB_SOURCE:         ${DAILY_SIM_CRYPTO_DB_SOURCE:-cloud28}
+          DB_SOURCES:        ${DAILY_SIM_CRYPTO_DB_SOURCES:-cloud28,cloud29}
           OUTPUT_DIR:        /tmp
 
           RSYNC_TARGETS:      ${DAILY_SIM_CRYPTO_RSYNC_TARGETS}
@@ -152,7 +152,3 @@ services:
           - ./logs:/logs
           - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
           - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
-networks:
-    default:
-        name: daily_mktdata_default
-        external: true

retrofit/crypto_missing_dates.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 
-
 SourceHost=${1}
 if [ "${SourceHost}" == "" ]; then
    echo "Usage: ${0} <source_host>"

retrofit/crypto_sim_missing_dates.sh

@@ -9,7 +9,7 @@ DEFAULT_FGLOB=*.crypto_sim_md.db.gz
 
 Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
 FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
-BackDaysCount=${BACKDAYS_COUNT:-50}
+BackDaysCount=${BACKDAYS_COUNT:-30}
 
 Cmd="get_retrofit_dates  ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
 echo $Cmd

retrofit/readme.md

@@ -0,0 +1,3 @@
+ ./run_retrofit.sh retrofit_crypto.sh cloud28
+
+ ./run_retrofit.sh retrofit_crypto.sh cloud29

retrofit/retrofit_crypto.sh

@@ -26,7 +26,7 @@ for dt in ${DATES}; do
    echo $dt
    Cmd="CRYPTO_MD_DATE=${dt}"
    Cmd+=" /usr/bin/docker compose" 
-   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" 
+   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" 
    Cmd+=" daily_crypto_${SourceHost}"
 
    echo ${Cmd}

retrofit/retrofit_equity.sh

@@ -19,7 +19,7 @@ for dt in ${DATES}; do
    echo $dt
    Cmd="EQUITY_HBAR_DATE=${dt}"
    Cmd+=" /usr/bin/docker compose" 
-   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" 
+   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" 
    Cmd+=" daily_equity_hbar"
 
 

retrofit/run_retrofit.sh

@@ -5,7 +5,7 @@ shift
 Params=${*}
 
 ScriptDir=$(realpath $(dirname ${0}))
-LogDir=${ScriptDir}/log
+LogDir=${ScriptDir}/logs
 mkdir -p ${LogDir}
 
 Cmd="nohup ${ScriptDir}/${Script} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &"

retrofit/sim_crypto_retrofit.sh

@@ -19,7 +19,7 @@ DATES=$($Cmd)
 for dt in ${DATES}; do
    Cmd="DAILY_SIM_CRYPTO_DATE=${dt}"
    Cmd+=" /usr/bin/docker compose" 
-   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" 
+   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" 
    Cmd+=" daily_sim_crypto"
 
    echo ${Cmd}

retrofit/sim_equity_retrofit.sh

@@ -20,7 +20,7 @@ DATES=$($Cmd)
 for dt in ${DATES}; do
    Cmd="DAILY_SIM_EQUITY_DATE=${dt}"
    Cmd+=" /usr/bin/docker compose" 
-   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" 
+   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm" 
    Cmd+=" daily_sim_equity"
 
    echo ${Cmd}

run.sh

@@ -6,19 +6,13 @@ COMPOSE_FILE="/works/docker/daily_mktdata/docker-compose.yml"
 PROJECT_DIR="/works/docker/daily_mktdata"
 PROJECT_NAME="daily_mktdata"
 NETWORK_NAME="daily_mktdata_default"
-SERVICE="${1:?usage: run_mktdata.sh <service-name>}"
+SERVICE="${1:?usage: $0 <service-name>}"
 
-# 1) Make sure the external network exists
+# Run the service as a one-off job and auto-remove container
-docker network inspect "${NETWORK_NAME}" >/dev/null 2>&1 || docker network create "${NETWORK_NAME}"
-
-# 2) Show quick status (optional but useful in logs)
-docker network inspect "${NETWORK_NAME}" --format '{{.Name}} {{.Id}} {{.Driver}} {{.Attachable}}'
-
-# 3) Run exactly one service without touching others
 exec docker compose \
   --project-name "${PROJECT_NAME}" \
   --project-directory "${PROJECT_DIR}" \
   -f "${COMPOSE_FILE}" \
-  up --no-deps --force-recreate \
+  run --rm --no-deps \
   "${SERVICE}"