prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial

Browse Source
This commit is contained in:
Cryptoval Trading Technologies 2025-05-18 02:55:01 +00:00
commit 68c06b9a7c
6 changed files with 229 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
.env Normal file
View File

@ -0,0 +1,32 @@
MKTDATA_IMAGE_VERSION=0.0.1
CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
CONFIG_FILE="http://cloud16.cvtt.vpn:6789/apps/md_recorder"
CREDS_FILE=/config/.creds
CRYPTO_DB_SCHEMAS=coinbase,bnbspot
# Date format YYYYMMDD
CRYPTO_MD_DATE=
CRYPTO_RSYNC_TARGETS=
CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
# CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/test"
CRYPTO_ADD_ARGS=
CRYPTO_PYTHON_DEBUG=N
CRYPTO_PYTHON_DEBUG_WAIT=Y
CRYPTO_PYTHON_DEBUG_PORT=45678
EQUITY_CONTAINER_NAME_PFX=daily_equity_md
EQUITY_HBAR_NUM_SYMBOLS=
EQUITY_QAT_SYMBOLS=CAN,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
EQUITY_RSYNC_TARGETS=
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
logs/

40
config/.creds Normal file
View File

@ -0,0 +1,40 @@
{
"__DUMMY__": null,
"INFISICAL": {
"url": "https://infisical.cvtt.net"
, "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e"
, "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c"
, "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c"
, "environment": "dev"
},
"MATTERMOST": {
"url": "https://mattermost.cryptovaltrading.com"
, "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
},
"TSDB_MD_CLD21": {
"host": "cloud21.cvtt.vpn"
, "port": 5432
, "user": "cvtt"
, "database": "cvtt_md"
, "password": "ICdIh0JnMM7vM7Pf"
},
"TSDB_MD_CLOUD28_RO": {
"host": "cloud28.cvtt.vpn"
, "port": 55432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "c63eb05adc99b17a71fe4fc810fe1b98"
},
"TSDB_MD_CLOUD24_RO": {
"host": "cloud24.cvtt.vpn"
, "port": 55432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "afa3151120e945a817d1d4ec0a1d5b81"
},
"ALPACA_SANDBOX": {
"api_key": "PKM1ASU8ULKSGO45ZZBV",
"secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9"
}
}

49
config/identity/id_rsa Normal file
View File

@ -0,0 +1,49 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
NhAAAAAwEAAQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMNo1DzPcnqFWgBUki5Sy/F
1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZkf2AUhRMhYlroXp7emmI7
XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOypqCQeoHpI2imhe2Fr7rg
fEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEtkmsONS0XzFEaVwx/wBTl
kLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7HVwU+LkONJmyy4YJdbET
SxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1sFqEfiYFnGpXeMZvjnAp
/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xVyagoxfwMqMjgUgZ4ecoK
ga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y7bu1uWfmWKVcSRw0729J
7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yiWvLaDWAtKDcickkzqj8Q
uUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1miCVSVpOFBmEihUFnRvRjj
kAAAdYoVe5GqFXuRoAAAAHc3NoLXJzYQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMN
o1DzPcnqFWgBUki5Sy/F1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZk
f2AUhRMhYlroXp7emmI7XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOy
pqCQeoHpI2imhe2Fr7rgfEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEt
kmsONS0XzFEaVwx/wBTlkLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7
HVwU+LkONJmyy4YJdbETSxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1
sFqEfiYFnGpXeMZvjnAp/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xV
yagoxfwMqMjgUgZ4ecoKga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y
7bu1uWfmWKVcSRw0729J7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yi
WvLaDWAtKDcickkzqj8QuUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1mi
CVSVpOFBmEihUFnRvRjjkAAAADAQABAAACABKTOpWsGDzST4iL8qQsdF6uDqpwnVMZU9ja
Sq++FVwZM3dvzB5uu1amT8cCkYSxt6/94GHaiGV0JOrd5lG2Hlvy4ZoftHoirlspD1SwnM
rUi6I9SU22E5dx24PsUMKlv/luPavAWT06BXw7QrXf/tJDithwUOPq1u7J6g6z00zrjTLm
3CWdyTnm2NEn3UMuqp7IgKFqBv+YiZgGk+z+GkxEKX00CwhD60dEyezJ/VV8+TJUDIM8r2
nn3tYNIQUrgsK8qoYOlTBHeLX1/fKl2Oqa6Lc//3XGvo3hyKFLIAHLuXLeQSwhTwUbmzWL
+XUc6LFTpk1w0s8MXyoT7tpILJkPnP+uPtc8vGL0xFWzlUtbeipWWowIvzEfbZPisy3OmE
QfRmHf7KicQH0d2p43lHNwbdTKegdiZDOkbukkkm3G+0Ft/I51swA2AuZ5CAtU+i77aGB6
kQdLKOe3udwEKyNGEi9RIlMlTC/XFbuTkM3iywZoxHYqRfdkf8S+3XIRlik8Mc1UsdKg8Y
r1g/vD7z9lT2gahdY0n2uRcpZKqFTuJwIR70WaAbgh8u8N2ciEZDx9Pnvk+eh7nsRVth7d
7Zf/dUjVhXVZPrHPmlUm0OFpOlObGs7yBUX4btz7vFH7ucpmWNdvZGN9p/cu0LGr70TCgp
EobaPxG5Yy/bLt/npBAAABADcSbg3sIjdsiH1yY8mXnpdh1zewJ62CijZ4jPO6huZ86hyy
Mqfbnoj4WOKuDg9h5JZf2JmWXStF9NX4IZAMWmm7fCuq/k/vPJH/COQicSSHPOCVlJyCpv
qJC6POpoYStyy4THiipzjD56zZCBy5NoZCI+jtr8QYb/oLK1Mh5XTlSFs1zvPn8ui2YXko
+r1OXxMW/stOlpjdqWaPYPpBB+lnvRgsNWx1JVQpbY6tZoQE8cobrB/QbCo5dKvO7HL2c9
ZeFBoHs9trxwm4AQaK4uvX3xPgxXwQqRK/iy94NQ2Nqn+1wc8p7lpqXn8sBRylXNs7kcFU
8anqhKUWozQX4ksAAAEBAMGDmpi626H8GUS9UYEtWCpptBFn8svEFE4G8K7Cs0Q824l23L
SI/fRdWMMeipdTTlFX/9YQxmTOgKzbsAyYuojZmMkg4+pZ0Q75iEzwTn8nX1W4N4MLkkUT
n0eB6fZh21qOoORyIa1bj6sdCnMVHxuDWZBGC90jnar1J6PhqZeKzmxyn9UqExYuUpPj2T
Sk7do4lR2h8oDq0DAJXKcQrHcW3PqddvDRSFYTpImudr4TQzI6qWzpi8KudrdKriDwJLCA
BA+l9VBzxUba6/LGHxDN+2L/NpPCmcDerC2HIpMs/grlsjTbjUpP6vzwb5vElPwb/FkKSS
282UsIo9WGenkAAAEBAOlCFBlt6FeFu/ywvD5lNQ89C5bLlLauzXyx0XoVoXEjpS7v2DsA
uPdYSuQ7ftXMumT89bfs1MynohuQN9BX3VIQh7c60BP2NXlsGs/jphGWlz8++zAj55awG1
Jb0NycqaKMWZ8/oRGl4A/2YpjWaKl+wKqeWJD7H+iFBnR5NQ5jPqnuqRJvCwDTfksMdsxw
R1y/+geHAq8ZBnXlkqphbSyTRQqxSmYUwvVFycj5mI84k+T3xTD5jHHthW5CGx6o9nIDJS
KAjsydR2GAMAGhQRM1a3c+VabC8ZqjhUQl+lga1Od3rvvDio/pH1yrrWQteWAtiqZAficv
Q8qlPhFqwcEAAAAgY3Z0dF9kb2NrZXJAY3J5cHRvdmFsdHJhZGluZy5jb20BAgM=
-----END OPENSSH PRIVATE KEY-----

1
config/identity/id_rsa.pub Normal file
View File

@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwUrrUXnTE0ZEU0I80ZrYS0wWRARlY0w2jUPM9yeoVaAFSSLlLL8XWNTvfLdquH7BqxayboxtsnIHCZS/ebCKUNJ4M9hlUGNfCKsxmR/YBSFEyFiWuhent6aYjtdkmGD6+rc26vvLc4bWeezi48MgW5DMAeV9otN3/mIGKce97I7KmoJB6gekjaKaF7YWvuuB8Qdl/1QDaTjwsOTFn93uHhyZItFNF4PT9Yj59oXYvS/fozCcS2Saw41LRfMURpXDH/AFOWQu1EZbQFlF/BXnpCD0mTmJVzL2qS2RTU5xrr9M67l/8FmIGdzsdXBT4uQ40mbLLhgl1sRNLE0PGZfkNuJYhPi5Aq6P/2ymAkOAm2aBKPKc2aNcJnpYVF9BrWwWoR+JgWcald4xm+OcCn/sgf6/hkQGEf2tRucNkeNLjGE4fMa4a7PswfjtuWwPibzazfFXJqCjF/AyoyOBSBnh5ygqBr1P6o25TKqefufmcAlpC5fifBi2F1KsqjewCbR1JxFa+9dn9jtu7W5Z+ZYpVxJHDTvb0nt9wNjW/IPgYEnbm9xiE074XeypgoMRpSAM4e7abOL3LwPMjXKJa8toNYC0oNyJySTOqPxC5Q7HFBzT06iKOUAbHfAoaqZkw41GUnL+V14b974hS6SOfqHWaIJVJWk4UGYSKFQWdG9GOOQ== cvtt_docker@cryptovaltrading.com

106
docker-compose.yml Normal file
View File

@ -0,0 +1,106 @@
# CVTT Microservices
services:
daily_crypto_cloud28:
image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION}
container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud28
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: no
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
DB_HOST: cloud28
DB_CRED_KEY: TSDB_MD_CLOUD28_RO
DB_SCHEMAS: ${CRYPTO_DB_SCHEMAS}
MD_DATE: ${CRYPTO_MD_DATE}
RSYNC_TARGETS: ${CRYPTO_RSYNC_TARGETS}
LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud28.log
PYTHON_DEBUG: ${CRYPTO_PYTHON_DEBUG}
PYTHON_DEBUG_WAIT: ${CRYPTO_PYTHON_DEBUG_WAIT}
ADD_ARGS: ${CRYPTO_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
ports:
- 45678:5678
daily_crypto_cloud24:
image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION}
container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud24
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: no
environment:
CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg}
CREDS_FILE: ${CREDS_FILE}
DB_HOST: cloud24
DB_CRED_KEY: TSDB_MD_CLOUD24_RO
DB_SCHEMAS: ${CRYPTO_DB_SCHEMAS}
MD_DATE: ${CRYPTO_MD_DATE}
RSYNC_TARGETS: ${CRYPTO_RSYNC_TARGETS}
LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud24.log
PYTHON_DEBUG: ${CRYPTO_PYTHON_DEBUG}
PYTHON_DEBUG_WAIT: ${CRYPTO_PYTHON_DEBUG_WAIT}
ADD_ARGS: ${CRYPTO_ADD_ARGS:-}
volumes:
- ./config:/config
- ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
ports:
- 45679:5678
daily_equity_hbar:
image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION}
container_name: alpaca_md_hbar_loader
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: no
environment:
PROGRAM: /alpaca_md_hbar_loader
TRD_CALENDAR_URL: https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
CREDS_FILE: ${CREDS_FILE}
NUM_SYMBOLS: ${EQUITY_HBAR_NUM_SYMBOLS:-}
RSYNC_TARGETS: ${EQUITY_RSYNC_TARGETS}
LOG_FILE_NAME: alpaca_md_hbar_loader
volumes:
- ./config:/config
- ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
daily_equity_qat:
image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION}
container_name: alpaca_md_qat_loader
user: ${CVTT_USER:-1001:1001}
pull_policy: always
restart: no
environment:
PROGRAM: /alpaca_md_qat_loader
TRD_CALENDAR_URL: https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
CREDS_FILE: ${CREDS_FILE}
SYMBOLS: ${EQUITY_QAT_SYMBOLS:-}
RSYNC_TARGETS: ${EQUITY_RSYNC_TARGETS}
LOG_FILE_NAME: alpaca_md_qat_loader
volumes:
- ./config:/config
- ./logs:/logs
- ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
- ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro