initial

.env

@@ -0,0 +1,32 @@
+MKTDATA_IMAGE_VERSION=0.0.1
+
+CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
+CONFIG_FILE="http://cloud16.cvtt.vpn:6789/apps/md_recorder"
+CREDS_FILE=/config/.creds
+
+CRYPTO_DB_SCHEMAS=coinbase,bnbspot
+# Date format YYYYMMDD
+CRYPTO_MD_DATE=
+
+CRYPTO_RSYNC_TARGETS=
+CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
+CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"
+
+# CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/test"
+
+CRYPTO_ADD_ARGS=
+
+
+CRYPTO_PYTHON_DEBUG=N
+CRYPTO_PYTHON_DEBUG_WAIT=Y
+CRYPTO_PYTHON_DEBUG_PORT=45678
+
+EQUITY_CONTAINER_NAME_PFX=daily_equity_md
+EQUITY_HBAR_NUM_SYMBOLS=
+EQUITY_QAT_SYMBOLS=CAN,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
+
+EQUITY_RSYNC_TARGETS=
+EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
+EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
+# EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"
+

.gitignore

@@ -0,0 +1 @@
+logs/

config/.creds

@@ -0,0 +1,40 @@
+{
+    "__DUMMY__": null,
+    "INFISICAL": {
+        "url": "https://infisical.cvtt.net"
+        , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e"
+        , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c"
+        , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c"
+        , "environment": "dev"
+    },
+    "MATTERMOST": {
+         "url": "https://mattermost.cryptovaltrading.com"
+         , "team": "CVTT"
+         , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
+    },
+    "TSDB_MD_CLD21": {
+        "host": "cloud21.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    },
+    "TSDB_MD_CLOUD28_RO": {
+        "host": "cloud28.cvtt.vpn"
+        , "port": 55432
+        , "user": "cvtt_ro"
+        , "database": "cvtt_md"
+        , "password": "c63eb05adc99b17a71fe4fc810fe1b98"
+    },
+    "TSDB_MD_CLOUD24_RO": {
+        "host": "cloud24.cvtt.vpn"
+        , "port": 55432
+        , "user": "cvtt_ro"
+        , "database": "cvtt_md"
+        , "password": "afa3151120e945a817d1d4ec0a1d5b81"
+    },
+    "ALPACA_SANDBOX": {
+       "api_key": "PKM1ASU8ULKSGO45ZZBV",
+        "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9"
+    }
+}

config/identity/id_rsa

@@ -0,0 +1,49 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMNo1DzPcnqFWgBUki5Sy/F
+1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZkf2AUhRMhYlroXp7emmI7
+XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOypqCQeoHpI2imhe2Fr7rg
+fEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEtkmsONS0XzFEaVwx/wBTl
+kLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7HVwU+LkONJmyy4YJdbET
+SxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1sFqEfiYFnGpXeMZvjnAp
+/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xVyagoxfwMqMjgUgZ4ecoK
+ga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y7bu1uWfmWKVcSRw0729J
+7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yiWvLaDWAtKDcickkzqj8Q
+uUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1miCVSVpOFBmEihUFnRvRjj
+kAAAdYoVe5GqFXuRoAAAAHc3NoLXJzYQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMN
+o1DzPcnqFWgBUki5Sy/F1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZk
+f2AUhRMhYlroXp7emmI7XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOy
+pqCQeoHpI2imhe2Fr7rgfEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEt
+kmsONS0XzFEaVwx/wBTlkLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7
+HVwU+LkONJmyy4YJdbETSxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1
+sFqEfiYFnGpXeMZvjnAp/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xV
+yagoxfwMqMjgUgZ4ecoKga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y
+7bu1uWfmWKVcSRw0729J7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yi
+WvLaDWAtKDcickkzqj8QuUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1mi
+CVSVpOFBmEihUFnRvRjjkAAAADAQABAAACABKTOpWsGDzST4iL8qQsdF6uDqpwnVMZU9ja
+Sq++FVwZM3dvzB5uu1amT8cCkYSxt6/94GHaiGV0JOrd5lG2Hlvy4ZoftHoirlspD1SwnM
+rUi6I9SU22E5dx24PsUMKlv/luPavAWT06BXw7QrXf/tJDithwUOPq1u7J6g6z00zrjTLm
+3CWdyTnm2NEn3UMuqp7IgKFqBv+YiZgGk+z+GkxEKX00CwhD60dEyezJ/VV8+TJUDIM8r2
+nn3tYNIQUrgsK8qoYOlTBHeLX1/fKl2Oqa6Lc//3XGvo3hyKFLIAHLuXLeQSwhTwUbmzWL
++XUc6LFTpk1w0s8MXyoT7tpILJkPnP+uPtc8vGL0xFWzlUtbeipWWowIvzEfbZPisy3OmE
+QfRmHf7KicQH0d2p43lHNwbdTKegdiZDOkbukkkm3G+0Ft/I51swA2AuZ5CAtU+i77aGB6
+kQdLKOe3udwEKyNGEi9RIlMlTC/XFbuTkM3iywZoxHYqRfdkf8S+3XIRlik8Mc1UsdKg8Y
+r1g/vD7z9lT2gahdY0n2uRcpZKqFTuJwIR70WaAbgh8u8N2ciEZDx9Pnvk+eh7nsRVth7d
+7Zf/dUjVhXVZPrHPmlUm0OFpOlObGs7yBUX4btz7vFH7ucpmWNdvZGN9p/cu0LGr70TCgp
+EobaPxG5Yy/bLt/npBAAABADcSbg3sIjdsiH1yY8mXnpdh1zewJ62CijZ4jPO6huZ86hyy
+Mqfbnoj4WOKuDg9h5JZf2JmWXStF9NX4IZAMWmm7fCuq/k/vPJH/COQicSSHPOCVlJyCpv
+qJC6POpoYStyy4THiipzjD56zZCBy5NoZCI+jtr8QYb/oLK1Mh5XTlSFs1zvPn8ui2YXko
++r1OXxMW/stOlpjdqWaPYPpBB+lnvRgsNWx1JVQpbY6tZoQE8cobrB/QbCo5dKvO7HL2c9
+ZeFBoHs9trxwm4AQaK4uvX3xPgxXwQqRK/iy94NQ2Nqn+1wc8p7lpqXn8sBRylXNs7kcFU
+8anqhKUWozQX4ksAAAEBAMGDmpi626H8GUS9UYEtWCpptBFn8svEFE4G8K7Cs0Q824l23L
+SI/fRdWMMeipdTTlFX/9YQxmTOgKzbsAyYuojZmMkg4+pZ0Q75iEzwTn8nX1W4N4MLkkUT
+n0eB6fZh21qOoORyIa1bj6sdCnMVHxuDWZBGC90jnar1J6PhqZeKzmxyn9UqExYuUpPj2T
+Sk7do4lR2h8oDq0DAJXKcQrHcW3PqddvDRSFYTpImudr4TQzI6qWzpi8KudrdKriDwJLCA
+BA+l9VBzxUba6/LGHxDN+2L/NpPCmcDerC2HIpMs/grlsjTbjUpP6vzwb5vElPwb/FkKSS
+282UsIo9WGenkAAAEBAOlCFBlt6FeFu/ywvD5lNQ89C5bLlLauzXyx0XoVoXEjpS7v2DsA
+uPdYSuQ7ftXMumT89bfs1MynohuQN9BX3VIQh7c60BP2NXlsGs/jphGWlz8++zAj55awG1
+Jb0NycqaKMWZ8/oRGl4A/2YpjWaKl+wKqeWJD7H+iFBnR5NQ5jPqnuqRJvCwDTfksMdsxw
+R1y/+geHAq8ZBnXlkqphbSyTRQqxSmYUwvVFycj5mI84k+T3xTD5jHHthW5CGx6o9nIDJS
+KAjsydR2GAMAGhQRM1a3c+VabC8ZqjhUQl+lga1Od3rvvDio/pH1yrrWQteWAtiqZAficv
+Q8qlPhFqwcEAAAAgY3Z0dF9kb2NrZXJAY3J5cHRvdmFsdHJhZGluZy5jb20BAgM=
+-----END OPENSSH PRIVATE KEY-----

config/identity/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwUrrUXnTE0ZEU0I80ZrYS0wWRARlY0w2jUPM9yeoVaAFSSLlLL8XWNTvfLdquH7BqxayboxtsnIHCZS/ebCKUNJ4M9hlUGNfCKsxmR/YBSFEyFiWuhent6aYjtdkmGD6+rc26vvLc4bWeezi48MgW5DMAeV9otN3/mIGKce97I7KmoJB6gekjaKaF7YWvuuB8Qdl/1QDaTjwsOTFn93uHhyZItFNF4PT9Yj59oXYvS/fozCcS2Saw41LRfMURpXDH/AFOWQu1EZbQFlF/BXnpCD0mTmJVzL2qS2RTU5xrr9M67l/8FmIGdzsdXBT4uQ40mbLLhgl1sRNLE0PGZfkNuJYhPi5Aq6P/2ymAkOAm2aBKPKc2aNcJnpYVF9BrWwWoR+JgWcald4xm+OcCn/sgf6/hkQGEf2tRucNkeNLjGE4fMa4a7PswfjtuWwPibzazfFXJqCjF/AyoyOBSBnh5ygqBr1P6o25TKqefufmcAlpC5fifBi2F1KsqjewCbR1JxFa+9dn9jtu7W5Z+ZYpVxJHDTvb0nt9wNjW/IPgYEnbm9xiE074XeypgoMRpSAM4e7abOL3LwPMjXKJa8toNYC0oNyJySTOqPxC5Q7HFBzT06iKOUAbHfAoaqZkw41GUnL+V14b974hS6SOfqHWaIJVJWk4UGYSKFQWdG9GOOQ== cvtt_docker@cryptovaltrading.com

docker-compose.yml

@@ -0,0 +1,106 @@
+# CVTT Microservices
+services:
+    daily_crypto_cloud28:
+        image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION}
+        container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud28
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: no
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+
+          DB_HOST:            cloud28
+          DB_CRED_KEY:        TSDB_MD_CLOUD28_RO
+
+          DB_SCHEMAS:         ${CRYPTO_DB_SCHEMAS}
+          MD_DATE:            ${CRYPTO_MD_DATE}
+          RSYNC_TARGETS:      ${CRYPTO_RSYNC_TARGETS}
+
+          LOG_FILE:           /logs/%D.daily_mktdata_crypto.cloud28.log
+          PYTHON_DEBUG:       ${CRYPTO_PYTHON_DEBUG}
+          PYTHON_DEBUG_WAIT:  ${CRYPTO_PYTHON_DEBUG_WAIT}
+
+          ADD_ARGS:           ${CRYPTO_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+          - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
+          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
+        ports:
+          - 45678:5678
+
+    daily_crypto_cloud24:
+        image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION}
+        container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud24
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: no
+        environment:
+          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
+          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
+          CREDS_FILE:         ${CREDS_FILE}
+
+          DB_HOST:            cloud24
+          DB_CRED_KEY:        TSDB_MD_CLOUD24_RO
+
+          DB_SCHEMAS:         ${CRYPTO_DB_SCHEMAS}
+          MD_DATE:            ${CRYPTO_MD_DATE}
+          RSYNC_TARGETS:      ${CRYPTO_RSYNC_TARGETS}
+
+          LOG_FILE:           /logs/%D.daily_mktdata_crypto.cloud24.log
+
+          PYTHON_DEBUG:       ${CRYPTO_PYTHON_DEBUG}
+          PYTHON_DEBUG_WAIT:  ${CRYPTO_PYTHON_DEBUG_WAIT}
+
+          ADD_ARGS:           ${CRYPTO_ADD_ARGS:-}
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+          - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
+          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
+        ports:
+          - 45679:5678
+
+    daily_equity_hbar:
+        image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION}
+        container_name: alpaca_md_hbar_loader
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: no
+        environment:
+          PROGRAM:            /alpaca_md_hbar_loader
+          TRD_CALENDAR_URL:   https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
+          CREDS_FILE:         ${CREDS_FILE}
+          NUM_SYMBOLS:        ${EQUITY_HBAR_NUM_SYMBOLS:-}
+
+          RSYNC_TARGETS:      ${EQUITY_RSYNC_TARGETS}
+          LOG_FILE_NAME:      alpaca_md_hbar_loader
+
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+          - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
+          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
+
+    daily_equity_qat:
+        image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION}
+        container_name: alpaca_md_qat_loader
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: no
+        environment:
+          PROGRAM:            /alpaca_md_qat_loader
+          TRD_CALENDAR_URL:   https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
+          CREDS_FILE:         ${CREDS_FILE}
+          SYMBOLS:            ${EQUITY_QAT_SYMBOLS:-}
+
+          RSYNC_TARGETS:      ${EQUITY_RSYNC_TARGETS}
+          LOG_FILE_NAME:      alpaca_md_qat_loader
+
+        volumes:
+          - ./config:/config
+          - ./logs:/logs
+          - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
+          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro