prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Cryptoval Trading Technologies 2025-07-23 23:11:11 +00:00
parent b3ff960188
commit a82b125edb
3 changed files with 94 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
.env
View File

@ -1,9 +1,10 @@
MKTDATA_IMAGE_VERSION=0.0.1 MKTDATA_IMAGE_VERSION=0.0.1
CONFIG_FILE=/config/daily_md.cfg
CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
CONFIG_FILE="http://cloud16.cvtt.vpn:6789/apps/md_recorder"
CREDS_FILE=/config/.creds CREDS_FILE=/config/.creds
# ==================================== daily_crypto_md
CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
CRYPTO_DB_SCHEMAS=coinbase,bnbspot CRYPTO_DB_SCHEMAS=coinbase,bnbspot
# Date format YYYYMMDD # Date format YYYYMMDD
CRYPTO_MD_DATE= CRYPTO_MD_DATE=
@ -12,21 +13,43 @@ CRYPTO_RSYNC_TARGETS=
CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto" CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto" CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
# CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/test" CRYPTO_ADD_ARGS="--log_stdout"
CRYPTO_ADD_ARGS=
CRYPTO_PYTHON_DEBUG=N CRYPTO_PYTHON_DEBUG=N
CRYPTO_PYTHON_DEBUG_WAIT=Y CRYPTO_PYTHON_DEBUG_WAIT=Y
CRYPTO_PYTHON_DEBUG_PORT=45678 CRYPTO_PYTHON_DEBUG_PORT=45678
# ==================================== daily_equity_md
EQUITY_CONTAINER_NAME_PFX=daily_equity_md EQUITY_CONTAINER_NAME_PFX=daily_equity_md
EQUITY_HBAR_NUM_SYMBOLS= EQUITY_HBAR_NUM_SYMBOLS=
EQUITY_QAT_SYMBOLS=CAN,COIN,GBTC,HOOD,MSTR,PYPL,XYZ EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA
EQUITY_RSYNC_TARGETS= EQUITY_RSYNC_TARGETS=
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity" EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test" # EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"
# ==================================== daily_sim_crypto
DAILY_SIM_CRYPTO_DATE=
#DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn
#DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto
DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn
DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto
DAILY_SIM_CRYPTO_RSYNC_TARGETS=
DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
# ==================================== daily_sim_equity
DAILY_SIM_EQUITY_DATE=
DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn
DAILY_SIM_EQUITY_SOURCE_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
DAILY_SIM_EQUITY_SYMBOL_LIST=COIN,CRCL,GBTC,HOOD,MSTR
DAILY_SIM_EQUITY_RSYNC_TARGETS=
DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"

35
config/.creds
View File

@ -1,38 +1,17 @@
{ {
"__DUMMY__": null, "__DUMMY__": null,
// "INFISICAL": { "INFISICAL": {
// "url": "https://infisical.cvtt.net" "url": "https://infisical.cvtt.net"
// , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e" , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e"
// , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c" , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c"
// , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c" , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c"
// , "environment": "dev" , "environment": "dev"
// }, },
"MATTERMOST": { "MATTERMOST": {
"url": "https://mattermost.cryptovaltrading.com" "url": "https://mattermost.cryptovaltrading.com"
, "team": "CVTT" , "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
}, },
"TSDB_MD_CLOUD28_RO": {
"host": "cloud28.cvtt.vpn"
, "port": 55432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "c63eb05adc99b17a71fe4fc810fe1b98"
},
"TSDB_MD_CLOUD24_RO": {
"host": "cloud24.cvtt.vpn"
, "port": 55432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "afa3151120e945a817d1d4ec0a1d5b81"
},
"TSDB_MD_CLOUD29_RO": {
"host": "cloud29.cvtt.vpn"
, "port": 55432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "2a7967cf2cbaab7f5afe97aa3ab83768"
},
"ALPACA_SANDBOX": { "ALPACA_SANDBOX": {
"api_key": "PKM1ASU8ULKSGO45ZZBV", "api_key": "PKM1ASU8ULKSGO45ZZBV",
"secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9" "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9"

65
docker-compose.yml
View File

@ -1,3 +1,7 @@
#=================================
# Create network
# docker network create --driver bridge --attachable daily_mktdata_default
#=================================
# CVTT Microservices # CVTT Microservices
services: services:
daily_crypto_cloud28: daily_crypto_cloud28:
@ -7,7 +11,6 @@ services:
pull_policy: always pull_policy: always
restart: no restart: no
environment: environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE} CREDS_FILE: ${CREDS_FILE}
@ -31,25 +34,24 @@ services:
ports: ports:
- 45678:5678 - 45678:5678
daily_crypto_cloud24: daily_crypto_cloud29:
image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION} image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION}
container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud24 container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud29
user: ${CVTT_USER:-1001:1001} user: ${CVTT_USER:-1001:1001}
pull_policy: always pull_policy: always
restart: no restart: no
environment: environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE} CREDS_FILE: ${CREDS_FILE}
DB_HOST: cloud24 DB_HOST: cloud29
DB_CRED_KEY: TSDB_MD_CLOUD24_RO DB_CRED_KEY: TSDB_MD_CLOUD29_RO
DB_SCHEMAS: ${CRYPTO_DB_SCHEMAS} DB_SCHEMAS: ${CRYPTO_DB_SCHEMAS}
MD_DATE: ${CRYPTO_MD_DATE} MD_DATE: ${CRYPTO_MD_DATE}
RSYNC_TARGETS: ${CRYPTO_RSYNC_TARGETS} RSYNC_TARGETS: ${CRYPTO_RSYNC_TARGETS}
LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud24.log LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud29.log
PYTHON_DEBUG: ${CRYPTO_PYTHON_DEBUG} PYTHON_DEBUG: ${CRYPTO_PYTHON_DEBUG}
PYTHON_DEBUG_WAIT: ${CRYPTO_PYTHON_DEBUG_WAIT} PYTHON_DEBUG_WAIT: ${CRYPTO_PYTHON_DEBUG_WAIT}
@ -62,7 +64,6 @@ services:
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
ports: ports:
- 45679:5678 - 45679:5678
daily_equity_hbar: daily_equity_hbar:
image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION} image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION}
container_name: alpaca_md_hbar_loader container_name: alpaca_md_hbar_loader
@ -104,3 +105,51 @@ services:
- ./logs:/logs - ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
daily_sim_crypto:
image: cloud21.cvtt.vpn:5500/daily_sim:${MKTDATA_IMAGE_VERSION}
container_name: daily_sim_crypto
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: no
environment:
SCRIPT: /prepare_crypto_simdata.sh
DATE: ${DAILY_SIM_CRYPTO_DATE:-}
SOURCE_HOST: ${DAILY_SIM_CRYPTO_SOURCE_HOST:-cloud21.cvtt.vpn}
SOURCE_ROOT_DIR: ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto}
DB_SOURCE: cloud28
OUTPUT_DIR: /tmp
RSYNC_TARGETS: ${DAILY_SIM_CRYPTO_RSYNC_TARGETS}
LOG_FILE: daily_sim_crypto.log
volumes:
- ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
daily_sim_equity:
image: cloud21.cvtt.vpn:5500/daily_sim:${MKTDATA_IMAGE_VERSION}
container_name: daily_sim_equity
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: no
environment:
SCRIPT: /prepare_eqt_simdata.sh
DATE: ${DAILY_SIM_EQUITY_DATE:-}
SOURCE_HOST: ${DAILY_SIM_EQUITY_SOURCE_HOST:-cloud21.cvtt.vpn}
SOURCE_ROOT_DIR: ${DAILY_SIM_EQUITY_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto}
SYMBOL_LIST: ${DAILY_SIM_EQUITY_SYMBOL_LIST}
OUTPUT_DIR: /tmp
RSYNC_TARGETS: ${DAILY_SIM_EQUITY_RSYNC_TARGETS}
LOG_FILE: daily_sim_equity.log
volumes:
- ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
networks:
default:
external: true
name: daily_mktdata_default