From a82b125edb01af88fae6d8a96701c05f88f1af60 Mon Sep 17 00:00:00 2001 From: Cryptoval Trading Technologies Date: Wed, 23 Jul 2025 23:11:11 +0000 Subject: [PATCH] progress --- .env | 37 +++++++++++++++++++++----- config/.creds | 35 +++++-------------------- docker-compose.yml | 65 ++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 94 insertions(+), 43 deletions(-) diff --git a/.env b/.env index 9b4306b..222818e 100644 --- a/.env +++ b/.env @@ -1,9 +1,10 @@ MKTDATA_IMAGE_VERSION=0.0.1 - -CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md -CONFIG_FILE="http://cloud16.cvtt.vpn:6789/apps/md_recorder" +CONFIG_FILE=/config/daily_md.cfg CREDS_FILE=/config/.creds +# ==================================== daily_crypto_md + +CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md CRYPTO_DB_SCHEMAS=coinbase,bnbspot # Date format YYYYMMDD CRYPTO_MD_DATE= @@ -12,21 +13,43 @@ CRYPTO_RSYNC_TARGETS= CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto" CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto" -# CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/test" - -CRYPTO_ADD_ARGS= +CRYPTO_ADD_ARGS="--log_stdout" CRYPTO_PYTHON_DEBUG=N CRYPTO_PYTHON_DEBUG_WAIT=Y CRYPTO_PYTHON_DEBUG_PORT=45678 +# ==================================== daily_equity_md EQUITY_CONTAINER_NAME_PFX=daily_equity_md EQUITY_HBAR_NUM_SYMBOLS= -EQUITY_QAT_SYMBOLS=CAN,COIN,GBTC,HOOD,MSTR,PYPL,XYZ +EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA EQUITY_RSYNC_TARGETS= EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity" EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" # EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test" +# ==================================== daily_sim_crypto +DAILY_SIM_CRYPTO_DATE= + +#DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn +#DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto +DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn +DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto +DAILY_SIM_CRYPTO_RSYNC_TARGETS= +DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/" +DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/" + + +# ==================================== daily_sim_equity +DAILY_SIM_EQUITY_DATE= + +DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn +DAILY_SIM_EQUITY_SOURCE_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md +DAILY_SIM_EQUITY_SYMBOL_LIST=COIN,CRCL,GBTC,HOOD,MSTR +DAILY_SIM_EQUITY_RSYNC_TARGETS= +DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/" +DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/" + + diff --git a/config/.creds b/config/.creds index 648d8e5..a6c040e 100644 --- a/config/.creds +++ b/config/.creds @@ -1,38 +1,17 @@ { "__DUMMY__": null, - // "INFISICAL": { - // "url": "https://infisical.cvtt.net" - // , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e" - // , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c" - // , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c" - // , "environment": "dev" - // }, + "INFISICAL": { + "url": "https://infisical.cvtt.net" + , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e" + , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c" + , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c" + , "environment": "dev" + }, "MATTERMOST": { "url": "https://mattermost.cryptovaltrading.com" , "team": "CVTT" , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" }, - "TSDB_MD_CLOUD28_RO": { - "host": "cloud28.cvtt.vpn" - , "port": 55432 - , "user": "cvtt_ro" - , "database": "cvtt_md" - , "password": "c63eb05adc99b17a71fe4fc810fe1b98" - }, - "TSDB_MD_CLOUD24_RO": { - "host": "cloud24.cvtt.vpn" - , "port": 55432 - , "user": "cvtt_ro" - , "database": "cvtt_md" - , "password": "afa3151120e945a817d1d4ec0a1d5b81" - }, - "TSDB_MD_CLOUD29_RO": { - "host": "cloud29.cvtt.vpn" - , "port": 55432 - , "user": "cvtt_ro" - , "database": "cvtt_md" - , "password": "2a7967cf2cbaab7f5afe97aa3ab83768" - }, "ALPACA_SANDBOX": { "api_key": "PKM1ASU8ULKSGO45ZZBV", "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9" diff --git a/docker-compose.yml b/docker-compose.yml index 4757599..b96fadd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,7 @@ +#================================= +# Create network +# docker network create --driver bridge --attachable daily_mktdata_default +#================================= # CVTT Microservices services: daily_crypto_cloud28: @@ -7,7 +11,6 @@ services: pull_policy: always restart: no environment: - CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} CREDS_FILE: ${CREDS_FILE} @@ -31,25 +34,24 @@ services: ports: - 45678:5678 - daily_crypto_cloud24: + daily_crypto_cloud29: image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION} - container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud24 + container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud29 user: ${CVTT_USER:-1001:1001} pull_policy: always restart: no environment: - CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} CREDS_FILE: ${CREDS_FILE} - DB_HOST: cloud24 - DB_CRED_KEY: TSDB_MD_CLOUD24_RO + DB_HOST: cloud29 + DB_CRED_KEY: TSDB_MD_CLOUD29_RO DB_SCHEMAS: ${CRYPTO_DB_SCHEMAS} MD_DATE: ${CRYPTO_MD_DATE} RSYNC_TARGETS: ${CRYPTO_RSYNC_TARGETS} - LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud24.log + LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud29.log PYTHON_DEBUG: ${CRYPTO_PYTHON_DEBUG} PYTHON_DEBUG_WAIT: ${CRYPTO_PYTHON_DEBUG_WAIT} @@ -62,7 +64,6 @@ services: - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro ports: - 45679:5678 - daily_equity_hbar: image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION} container_name: alpaca_md_hbar_loader @@ -104,3 +105,51 @@ services: - ./logs:/logs - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro + + daily_sim_crypto: + image: cloud21.cvtt.vpn:5500/daily_sim:${MKTDATA_IMAGE_VERSION} + container_name: daily_sim_crypto + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: no + environment: + SCRIPT: /prepare_crypto_simdata.sh + DATE: ${DAILY_SIM_CRYPTO_DATE:-} + SOURCE_HOST: ${DAILY_SIM_CRYPTO_SOURCE_HOST:-cloud21.cvtt.vpn} + SOURCE_ROOT_DIR: ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto} + DB_SOURCE: cloud28 + OUTPUT_DIR: /tmp + + RSYNC_TARGETS: ${DAILY_SIM_CRYPTO_RSYNC_TARGETS} + LOG_FILE: daily_sim_crypto.log + + volumes: + - ./logs:/logs + - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro + - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro + + daily_sim_equity: + image: cloud21.cvtt.vpn:5500/daily_sim:${MKTDATA_IMAGE_VERSION} + container_name: daily_sim_equity + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: no + environment: + SCRIPT: /prepare_eqt_simdata.sh + DATE: ${DAILY_SIM_EQUITY_DATE:-} + SOURCE_HOST: ${DAILY_SIM_EQUITY_SOURCE_HOST:-cloud21.cvtt.vpn} + SOURCE_ROOT_DIR: ${DAILY_SIM_EQUITY_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto} + SYMBOL_LIST: ${DAILY_SIM_EQUITY_SYMBOL_LIST} + OUTPUT_DIR: /tmp + + RSYNC_TARGETS: ${DAILY_SIM_EQUITY_RSYNC_TARGETS} + LOG_FILE: daily_sim_equity.log + + volumes: + - ./logs:/logs + - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro + - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro +networks: + default: + external: true + name: daily_mktdata_default