prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Cryptoval Trading Technologies 2025-11-16 20:34:00 +00:00
parent f2d7992e9a
commit 95056f252a
11 changed files with 17 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.env
View File

@ -1,7 +1,7 @@
CREDS_FILE=/config/.creds
# ==================================== daily_crypto_md
CRYPTO_IMAGE_VERSION=0.0.3
CRYPTO_IMAGE_VERSION=0.0.4
CRYPTO_CONFIG_FILE=http://cloud16.cvtt.vpn:6789/apps/utils/mktdata/daily_md_crypto
CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
CRYPTO_DB_SCHEMAS=coinbase,bnbspot
@ -33,14 +33,14 @@ EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/c
# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"
# ==================================== daily_sim_crypto
SIM_CRYPTO_IMAGE_VERSION=0.0.2
SIM_CRYPTO_IMAGE_VERSION=0.0.3
DAILY_SIM_CRYPTO_DATE=
#DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn
#DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto
DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn
DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto
DAILY_SIM_CRYPTO_DB_SOURCE=cloud29
DAILY_SIM_CRYPTO_DB_SOURCES=cloud28,cloud29
DAILY_SIM_CRYPTO_RSYNC_TARGETS=
DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim"
DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim"

8
docker-compose.yml
View File

@ -119,8 +119,8 @@ services:
SCRIPT: /prepare_crypto_simdata.sh
DATE: ${DAILY_SIM_CRYPTO_DATE:-}
SOURCE_HOST: ${DAILY_SIM_CRYPTO_SOURCE_HOST:-cloud21.cvtt.vpn}
SOURCE_ROOT_DIR: ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto}
DB_SOURCE: ${DAILY_SIM_CRYPTO_DB_SOURCE:-cloud28}
SOURCE_ROOT_DIR: ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/works/cvtt/md_archive/crypto}
DB_SOURCES: ${DAILY_SIM_CRYPTO_DB_SOURCES:-cloud28,cloud29}
OUTPUT_DIR: /tmp
RSYNC_TARGETS: ${DAILY_SIM_CRYPTO_RSYNC_TARGETS}
@ -152,7 +152,3 @@ services:
- ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
networks:
default:
name: daily_mktdata_default
external: true

2
retrofit/crypto_missing_dates.sh
View File

@ -1,7 +1,5 @@
#!/bin/bash
# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29
SourceHost=${1}
if [ "${SourceHost}" == "" ]; then
echo "Usage: ${0} <source_host>"

2
retrofit/crypto_sim_missing_dates.sh
View File

@ -9,7 +9,7 @@ DEFAULT_FGLOB=*.crypto_sim_md.db.gz
Source=cvtt@${MD_HOST:-cloud21.cvtt.vpn}
FileGlob=${FILE_GLOB:-${DEFAULT_FGLOB}}
BackDaysCount=${BACKDAYS_COUNT:-50}
BackDaysCount=${BACKDAYS_COUNT:-30}
Cmd="get_retrofit_dates ${Source} /works/cvtt/md_archive/crypto/sim ${FileGlob} ${BackDaysCount}"
echo $Cmd

3
retrofit/readme.md Normal file
View File

@ -0,0 +1,3 @@
./run_retrofit.sh retrofit_crypto.sh cloud28
./run_retrofit.sh retrofit_crypto.sh cloud29

2
retrofit/retrofit_crypto.sh
View File

@ -26,7 +26,7 @@ for dt in ${DATES}; do
echo $dt
Cmd="CRYPTO_MD_DATE=${dt}"
Cmd+=" /usr/bin/docker compose"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm"
Cmd+=" daily_crypto_${SourceHost}"
echo ${Cmd}

2
retrofit/retrofit_equity.sh
View File

@ -19,7 +19,7 @@ for dt in ${DATES}; do
echo $dt
Cmd="EQUITY_HBAR_DATE=${dt}"
Cmd+=" /usr/bin/docker compose"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm"
Cmd+=" daily_equity_hbar"

2
retrofit/run_retrofit.sh
View File

@ -5,7 +5,7 @@ shift
Params=${*}
ScriptDir=$(realpath $(dirname ${0}))
LogDir=${ScriptDir}/log
LogDir=${ScriptDir}/logs
mkdir -p ${LogDir}
Cmd="nohup ${ScriptDir}/${Script} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &"

2
retrofit/sim_crypto_retrofit.sh
View File

@ -19,7 +19,7 @@ DATES=$($Cmd)
for dt in ${DATES}; do
Cmd="DAILY_SIM_CRYPTO_DATE=${dt}"
Cmd+=" /usr/bin/docker compose"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm"
Cmd+=" daily_sim_crypto"
echo ${Cmd}

2
retrofit/sim_equity_retrofit.sh
View File

@ -20,7 +20,7 @@ DATES=$($Cmd)
for dt in ${DATES}; do
Cmd="DAILY_SIM_EQUITY_DATE=${dt}"
Cmd+=" /usr/bin/docker compose"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml run --rm"
Cmd+=" daily_sim_equity"
echo ${Cmd}

12
run.sh
View File

@ -6,19 +6,13 @@ COMPOSE_FILE="/works/docker/daily_mktdata/docker-compose.yml"
PROJECT_DIR="/works/docker/daily_mktdata"
PROJECT_NAME="daily_mktdata"
NETWORK_NAME="daily_mktdata_default"
SERVICE="${1:?usage: run_mktdata.sh <service-name>}"
SERVICE="${1:?usage: $0 <service-name>}"
# 1) Make sure the external network exists
docker network inspect "${NETWORK_NAME}" >/dev/null 2>&1 || docker network create "${NETWORK_NAME}"
# 2) Show quick status (optional but useful in logs)
docker network inspect "${NETWORK_NAME}" --format '{{.Name}} {{.Id}} {{.Driver}} {{.Attachable}}'
# 3) Run exactly one service without touching others
# Run the service as a one-off job and auto-remove container
exec docker compose \
--project-name "${PROJECT_NAME}" \
--project-directory "${PROJECT_DIR}" \
-f "${COMPOSE_FILE}" \
up --no-deps --force-recreate \
run --rm --no-deps \
"${SERVICE}"