From 68c06b9a7cdbe831c07d05370d0bbf42a4b22210 Mon Sep 17 00:00:00 2001 From: Cryptoval Trading Technologies Date: Sun, 18 May 2025 02:55:01 +0000 Subject: [PATCH] initial --- .env | 32 +++++++++++ .gitignore | 1 + config/.creds | 40 ++++++++++++++ config/identity/id_rsa | 49 +++++++++++++++++ config/identity/id_rsa.pub | 1 + docker-compose.yml | 106 +++++++++++++++++++++++++++++++++++++ 6 files changed, 229 insertions(+) create mode 100644 .env create mode 100644 .gitignore create mode 100644 config/.creds create mode 100644 config/identity/id_rsa create mode 100644 config/identity/id_rsa.pub create mode 100644 docker-compose.yml diff --git a/.env b/.env new file mode 100644 index 0000000..9b4306b --- /dev/null +++ b/.env @@ -0,0 +1,32 @@ +MKTDATA_IMAGE_VERSION=0.0.1 + +CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md +CONFIG_FILE="http://cloud16.cvtt.vpn:6789/apps/md_recorder" +CREDS_FILE=/config/.creds + +CRYPTO_DB_SCHEMAS=coinbase,bnbspot +# Date format YYYYMMDD +CRYPTO_MD_DATE= + +CRYPTO_RSYNC_TARGETS= +CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto" +CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto" + +# CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/test" + +CRYPTO_ADD_ARGS= + + +CRYPTO_PYTHON_DEBUG=N +CRYPTO_PYTHON_DEBUG_WAIT=Y +CRYPTO_PYTHON_DEBUG_PORT=45678 + +EQUITY_CONTAINER_NAME_PFX=daily_equity_md +EQUITY_HBAR_NUM_SYMBOLS= +EQUITY_QAT_SYMBOLS=CAN,COIN,GBTC,HOOD,MSTR,PYPL,XYZ + +EQUITY_RSYNC_TARGETS= +EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity" +EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" +# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test" + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..333c1e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +logs/ diff --git a/config/.creds b/config/.creds new file mode 100644 index 0000000..8524570 --- /dev/null +++ b/config/.creds @@ -0,0 +1,40 @@ +{ + "__DUMMY__": null, + "INFISICAL": { + "url": "https://infisical.cvtt.net" + , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e" + , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c" + , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c" + , "environment": "dev" + }, + "MATTERMOST": { + "url": "https://mattermost.cryptovaltrading.com" + , "team": "CVTT" + , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" + }, + "TSDB_MD_CLD21": { + "host": "cloud21.cvtt.vpn" + , "port": 5432 + , "user": "cvtt" + , "database": "cvtt_md" + , "password": "ICdIh0JnMM7vM7Pf" + }, + "TSDB_MD_CLOUD28_RO": { + "host": "cloud28.cvtt.vpn" + , "port": 55432 + , "user": "cvtt_ro" + , "database": "cvtt_md" + , "password": "c63eb05adc99b17a71fe4fc810fe1b98" + }, + "TSDB_MD_CLOUD24_RO": { + "host": "cloud24.cvtt.vpn" + , "port": 55432 + , "user": "cvtt_ro" + , "database": "cvtt_md" + , "password": "afa3151120e945a817d1d4ec0a1d5b81" + }, + "ALPACA_SANDBOX": { + "api_key": "PKM1ASU8ULKSGO45ZZBV", + "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9" + } +} diff --git a/config/identity/id_rsa b/config/identity/id_rsa new file mode 100644 index 0000000..e5d2191 --- /dev/null +++ b/config/identity/id_rsa @@ -0,0 +1,49 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMNo1DzPcnqFWgBUki5Sy/F +1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZkf2AUhRMhYlroXp7emmI7 +XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOypqCQeoHpI2imhe2Fr7rg +fEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEtkmsONS0XzFEaVwx/wBTl +kLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7HVwU+LkONJmyy4YJdbET +SxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1sFqEfiYFnGpXeMZvjnAp +/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xVyagoxfwMqMjgUgZ4ecoK +ga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y7bu1uWfmWKVcSRw0729J +7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yiWvLaDWAtKDcickkzqj8Q +uUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1miCVSVpOFBmEihUFnRvRjj +kAAAdYoVe5GqFXuRoAAAAHc3NoLXJzYQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMN +o1DzPcnqFWgBUki5Sy/F1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZk +f2AUhRMhYlroXp7emmI7XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOy +pqCQeoHpI2imhe2Fr7rgfEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEt +kmsONS0XzFEaVwx/wBTlkLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7 +HVwU+LkONJmyy4YJdbETSxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1 +sFqEfiYFnGpXeMZvjnAp/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xV +yagoxfwMqMjgUgZ4ecoKga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y +7bu1uWfmWKVcSRw0729J7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yi +WvLaDWAtKDcickkzqj8QuUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1mi +CVSVpOFBmEihUFnRvRjjkAAAADAQABAAACABKTOpWsGDzST4iL8qQsdF6uDqpwnVMZU9ja +Sq++FVwZM3dvzB5uu1amT8cCkYSxt6/94GHaiGV0JOrd5lG2Hlvy4ZoftHoirlspD1SwnM +rUi6I9SU22E5dx24PsUMKlv/luPavAWT06BXw7QrXf/tJDithwUOPq1u7J6g6z00zrjTLm +3CWdyTnm2NEn3UMuqp7IgKFqBv+YiZgGk+z+GkxEKX00CwhD60dEyezJ/VV8+TJUDIM8r2 +nn3tYNIQUrgsK8qoYOlTBHeLX1/fKl2Oqa6Lc//3XGvo3hyKFLIAHLuXLeQSwhTwUbmzWL ++XUc6LFTpk1w0s8MXyoT7tpILJkPnP+uPtc8vGL0xFWzlUtbeipWWowIvzEfbZPisy3OmE +QfRmHf7KicQH0d2p43lHNwbdTKegdiZDOkbukkkm3G+0Ft/I51swA2AuZ5CAtU+i77aGB6 +kQdLKOe3udwEKyNGEi9RIlMlTC/XFbuTkM3iywZoxHYqRfdkf8S+3XIRlik8Mc1UsdKg8Y +r1g/vD7z9lT2gahdY0n2uRcpZKqFTuJwIR70WaAbgh8u8N2ciEZDx9Pnvk+eh7nsRVth7d +7Zf/dUjVhXVZPrHPmlUm0OFpOlObGs7yBUX4btz7vFH7ucpmWNdvZGN9p/cu0LGr70TCgp +EobaPxG5Yy/bLt/npBAAABADcSbg3sIjdsiH1yY8mXnpdh1zewJ62CijZ4jPO6huZ86hyy +Mqfbnoj4WOKuDg9h5JZf2JmWXStF9NX4IZAMWmm7fCuq/k/vPJH/COQicSSHPOCVlJyCpv +qJC6POpoYStyy4THiipzjD56zZCBy5NoZCI+jtr8QYb/oLK1Mh5XTlSFs1zvPn8ui2YXko ++r1OXxMW/stOlpjdqWaPYPpBB+lnvRgsNWx1JVQpbY6tZoQE8cobrB/QbCo5dKvO7HL2c9 +ZeFBoHs9trxwm4AQaK4uvX3xPgxXwQqRK/iy94NQ2Nqn+1wc8p7lpqXn8sBRylXNs7kcFU +8anqhKUWozQX4ksAAAEBAMGDmpi626H8GUS9UYEtWCpptBFn8svEFE4G8K7Cs0Q824l23L +SI/fRdWMMeipdTTlFX/9YQxmTOgKzbsAyYuojZmMkg4+pZ0Q75iEzwTn8nX1W4N4MLkkUT +n0eB6fZh21qOoORyIa1bj6sdCnMVHxuDWZBGC90jnar1J6PhqZeKzmxyn9UqExYuUpPj2T +Sk7do4lR2h8oDq0DAJXKcQrHcW3PqddvDRSFYTpImudr4TQzI6qWzpi8KudrdKriDwJLCA +BA+l9VBzxUba6/LGHxDN+2L/NpPCmcDerC2HIpMs/grlsjTbjUpP6vzwb5vElPwb/FkKSS +282UsIo9WGenkAAAEBAOlCFBlt6FeFu/ywvD5lNQ89C5bLlLauzXyx0XoVoXEjpS7v2DsA +uPdYSuQ7ftXMumT89bfs1MynohuQN9BX3VIQh7c60BP2NXlsGs/jphGWlz8++zAj55awG1 +Jb0NycqaKMWZ8/oRGl4A/2YpjWaKl+wKqeWJD7H+iFBnR5NQ5jPqnuqRJvCwDTfksMdsxw +R1y/+geHAq8ZBnXlkqphbSyTRQqxSmYUwvVFycj5mI84k+T3xTD5jHHthW5CGx6o9nIDJS +KAjsydR2GAMAGhQRM1a3c+VabC8ZqjhUQl+lga1Od3rvvDio/pH1yrrWQteWAtiqZAficv +Q8qlPhFqwcEAAAAgY3Z0dF9kb2NrZXJAY3J5cHRvdmFsdHJhZGluZy5jb20BAgM= +-----END OPENSSH PRIVATE KEY----- diff --git a/config/identity/id_rsa.pub b/config/identity/id_rsa.pub new file mode 100644 index 0000000..f301f8f --- /dev/null +++ b/config/identity/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwUrrUXnTE0ZEU0I80ZrYS0wWRARlY0w2jUPM9yeoVaAFSSLlLL8XWNTvfLdquH7BqxayboxtsnIHCZS/ebCKUNJ4M9hlUGNfCKsxmR/YBSFEyFiWuhent6aYjtdkmGD6+rc26vvLc4bWeezi48MgW5DMAeV9otN3/mIGKce97I7KmoJB6gekjaKaF7YWvuuB8Qdl/1QDaTjwsOTFn93uHhyZItFNF4PT9Yj59oXYvS/fozCcS2Saw41LRfMURpXDH/AFOWQu1EZbQFlF/BXnpCD0mTmJVzL2qS2RTU5xrr9M67l/8FmIGdzsdXBT4uQ40mbLLhgl1sRNLE0PGZfkNuJYhPi5Aq6P/2ymAkOAm2aBKPKc2aNcJnpYVF9BrWwWoR+JgWcald4xm+OcCn/sgf6/hkQGEf2tRucNkeNLjGE4fMa4a7PswfjtuWwPibzazfFXJqCjF/AyoyOBSBnh5ygqBr1P6o25TKqefufmcAlpC5fifBi2F1KsqjewCbR1JxFa+9dn9jtu7W5Z+ZYpVxJHDTvb0nt9wNjW/IPgYEnbm9xiE074XeypgoMRpSAM4e7abOL3LwPMjXKJa8toNYC0oNyJySTOqPxC5Q7HFBzT06iKOUAbHfAoaqZkw41GUnL+V14b974hS6SOfqHWaIJVJWk4UGYSKFQWdG9GOOQ== cvtt_docker@cryptovaltrading.com diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4757599 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,106 @@ +# CVTT Microservices +services: + daily_crypto_cloud28: + image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION} + container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud28 + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: no + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + + DB_HOST: cloud28 + DB_CRED_KEY: TSDB_MD_CLOUD28_RO + + DB_SCHEMAS: ${CRYPTO_DB_SCHEMAS} + MD_DATE: ${CRYPTO_MD_DATE} + RSYNC_TARGETS: ${CRYPTO_RSYNC_TARGETS} + + LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud28.log + PYTHON_DEBUG: ${CRYPTO_PYTHON_DEBUG} + PYTHON_DEBUG_WAIT: ${CRYPTO_PYTHON_DEBUG_WAIT} + + ADD_ARGS: ${CRYPTO_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro + - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro + ports: + - 45678:5678 + + daily_crypto_cloud24: + image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION} + container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud24 + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: no + environment: + CONFIG_SERVICE: ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789} + CONFIG_FILE: ${CONFIG_FILE:-/config/md_recorder.cfg} + CREDS_FILE: ${CREDS_FILE} + + DB_HOST: cloud24 + DB_CRED_KEY: TSDB_MD_CLOUD24_RO + + DB_SCHEMAS: ${CRYPTO_DB_SCHEMAS} + MD_DATE: ${CRYPTO_MD_DATE} + RSYNC_TARGETS: ${CRYPTO_RSYNC_TARGETS} + + LOG_FILE: /logs/%D.daily_mktdata_crypto.cloud24.log + + PYTHON_DEBUG: ${CRYPTO_PYTHON_DEBUG} + PYTHON_DEBUG_WAIT: ${CRYPTO_PYTHON_DEBUG_WAIT} + + ADD_ARGS: ${CRYPTO_ADD_ARGS:-} + volumes: + - ./config:/config + - ./logs:/logs + - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro + - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro + ports: + - 45679:5678 + + daily_equity_hbar: + image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION} + container_name: alpaca_md_hbar_loader + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: no + environment: + PROGRAM: /alpaca_md_hbar_loader + TRD_CALENDAR_URL: https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS + CREDS_FILE: ${CREDS_FILE} + NUM_SYMBOLS: ${EQUITY_HBAR_NUM_SYMBOLS:-} + + RSYNC_TARGETS: ${EQUITY_RSYNC_TARGETS} + LOG_FILE_NAME: alpaca_md_hbar_loader + + volumes: + - ./config:/config + - ./logs:/logs + - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro + - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro + + daily_equity_qat: + image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION} + container_name: alpaca_md_qat_loader + user: ${CVTT_USER:-1001:1001} + pull_policy: always + restart: no + environment: + PROGRAM: /alpaca_md_qat_loader + TRD_CALENDAR_URL: https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS + CREDS_FILE: ${CREDS_FILE} + SYMBOLS: ${EQUITY_QAT_SYMBOLS:-} + + RSYNC_TARGETS: ${EQUITY_RSYNC_TARGETS} + LOG_FILE_NAME: alpaca_md_qat_loader + + volumes: + - ./config:/config + - ./logs:/logs + - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro + - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro