progress

2025-07-23 23:11:11 +00:00 · 2025-07-23 23:11:11 +00:00 · a82b125edb
commit a82b125edb
parent b3ff960188
3 changed files with 94 additions and 43 deletions
--- a/.env
+++ b/.env
@ -1,9 +1,10 @@
 MKTDATA_IMAGE_VERSION=0.0.1
-
-CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
-CONFIG_FILE="http://cloud16.cvtt.vpn:6789/apps/md_recorder"
+CONFIG_FILE=/config/daily_md.cfg
 CREDS_FILE=/config/.creds

+# ==================================== daily_crypto_md
+
+CRYPTO_CONTAINER_NAME_PFX=daily_crypto_md
 CRYPTO_DB_SCHEMAS=coinbase,bnbspot
 # Date format YYYYMMDD
 CRYPTO_MD_DATE=
@ -12,21 +13,43 @@ CRYPTO_RSYNC_TARGETS=
 CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
 CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto"

-# CRYPTO_RSYNC_TARGETS="${CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/test"
-
-CRYPTO_ADD_ARGS=
+CRYPTO_ADD_ARGS="--log_stdout"


 CRYPTO_PYTHON_DEBUG=N
 CRYPTO_PYTHON_DEBUG_WAIT=Y
 CRYPTO_PYTHON_DEBUG_PORT=45678

+# ==================================== daily_equity_md
 EQUITY_CONTAINER_NAME_PFX=daily_equity_md
 EQUITY_HBAR_NUM_SYMBOLS=
-EQUITY_QAT_SYMBOLS=CAN,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
+EQUITY_QAT_SYMBOLS=COIN,GBTC,HOOD,MSTR,PYPL,QQQ,NVDA

 EQUITY_RSYNC_TARGETS=
 EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity"
 EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
 # EQUITY_RSYNC_TARGETS="${EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test"

+# ==================================== daily_sim_crypto
+DAILY_SIM_CRYPTO_DATE=
+
+#DAILY_SIM_CRYPTO_SOURCE_HOST=cloud21.cvtt.vpn
+#DAILY_SIM_CRYPTO_SOURCE_DIR=/opt/store/cvtt/md_archive/crypto
+DAILY_SIM_CRYPTO_SOURCE_HOST=hs01.cvtt.vpn
+DAILY_SIM_CRYPTO_SOURCE_DIR=/works/cvtt/md_archive/crypto
+DAILY_SIM_CRYPTO_RSYNC_TARGETS=
+DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
+DAILY_SIM_CRYPTO_RSYNC_TARGETS="${DAILY_SIM_CRYPTO_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
+
+
+# ==================================== daily_sim_equity
+DAILY_SIM_EQUITY_DATE=
+
+DAILY_SIM_EQUITY_SOURCE_HOST=cloud21.cvtt.vpn
+DAILY_SIM_EQUITY_SOURCE_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
+DAILY_SIM_EQUITY_SYMBOL_LIST=COIN,CRCL,GBTC,HOOD,MSTR
+DAILY_SIM_EQUITY_RSYNC_TARGETS=
+DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
+DAILY_SIM_EQUITY_RSYNC_TARGETS="${DAILY_SIM_EQUITY_RSYNC_TARGETS} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
+
+
--- a/config/.creds
+++ b/config/.creds
@ -1,38 +1,17 @@
 {
    "__DUMMY__": null,
-    // "INFISICAL": {
-    //     "url": "https://infisical.cvtt.net"
-    //     , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e"
-    //     , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c"
-    //     , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c"
-    //     , "environment": "dev"
-    // },
+    "INFISICAL": {
+        "url": "https://infisical.cvtt.net"
+        , "client_id": "e330a1c4-08dc-452f-ac02-5b37fff4ab3e"
+        , "client_secret": "190554828c3c342f63517320ac029f822b6358aa6187aec266465d0bde4d6b3c"
+        , "workspace_id": "6923c950-8473-4cec-818f-10df41d8b39c"
+        , "environment": "dev"
+    },
    "MATTERMOST": {
         "url": "https://mattermost.cryptovaltrading.com"
         , "team": "CVTT"
         , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
    },
-    "TSDB_MD_CLOUD28_RO": {
-        "host": "cloud28.cvtt.vpn"
-        , "port": 55432
-        , "user": "cvtt_ro"
-        , "database": "cvtt_md"
-        , "password": "c63eb05adc99b17a71fe4fc810fe1b98"
-    },
-    "TSDB_MD_CLOUD24_RO": {
-        "host": "cloud24.cvtt.vpn"
-        , "port": 55432
-        , "user": "cvtt_ro"
-        , "database": "cvtt_md"
-        , "password": "afa3151120e945a817d1d4ec0a1d5b81"
-    },
-    "TSDB_MD_CLOUD29_RO": {
-        "host": "cloud29.cvtt.vpn"
-        , "port": 55432
-        , "user": "cvtt_ro"
-        , "database": "cvtt_md"
-        , "password": "2a7967cf2cbaab7f5afe97aa3ab83768"
-    },
    "ALPACA_SANDBOX": {
       "api_key": "PKM1ASU8ULKSGO45ZZBV",
        "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,3 +1,7 @@
+#=================================
+# Create network 
+#    docker network create  --driver bridge --attachable daily_mktdata_default
+#=================================
 # CVTT Microservices
 services:
    daily_crypto_cloud28:
@ -7,7 +11,6 @@ services:
        pull_policy: always
        restart: no
        environment:
-          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
          CREDS_FILE:         ${CREDS_FILE}

@ -31,25 +34,24 @@ services:
        ports:
          - 45678:5678

-    daily_crypto_cloud24:
+    daily_crypto_cloud29:
        image: cloud21.cvtt.vpn:5500/daily_crypto_md:${MKTDATA_IMAGE_VERSION}
-        container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud24
+        container_name: ${CRYPTO_CONTAINER_NAME_PFX}.cloud29
        user: ${CVTT_USER:-1001:1001}
        pull_policy: always
        restart: no
        environment:
-          CONFIG_SERVICE:     ${CONFIG_SERVICE:-cloud16.cvtt.vpn:6789}
          CONFIG_FILE:        ${CONFIG_FILE:-/config/md_recorder.cfg}
          CREDS_FILE:         ${CREDS_FILE}

-          DB_HOST:            cloud24
-          DB_CRED_KEY:        TSDB_MD_CLOUD24_RO
+          DB_HOST:            cloud29
+          DB_CRED_KEY:        TSDB_MD_CLOUD29_RO

          DB_SCHEMAS:         ${CRYPTO_DB_SCHEMAS}
          MD_DATE:            ${CRYPTO_MD_DATE}
          RSYNC_TARGETS:      ${CRYPTO_RSYNC_TARGETS}

-          LOG_FILE:           /logs/%D.daily_mktdata_crypto.cloud24.log
+          LOG_FILE:           /logs/%D.daily_mktdata_crypto.cloud29.log

          PYTHON_DEBUG:       ${CRYPTO_PYTHON_DEBUG}
          PYTHON_DEBUG_WAIT:  ${CRYPTO_PYTHON_DEBUG_WAIT}
@ -62,7 +64,6 @@ services:
          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
        ports:
          - 45679:5678
-
    daily_equity_hbar:
        image: cloud21.cvtt.vpn:5500/daily_equity_md:${MKTDATA_IMAGE_VERSION}
        container_name: alpaca_md_hbar_loader
@ -104,3 +105,51 @@ services:
          - ./logs:/logs
          - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
+
+    daily_sim_crypto:
+        image: cloud21.cvtt.vpn:5500/daily_sim:${MKTDATA_IMAGE_VERSION}
+        container_name: daily_sim_crypto
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: no
+        environment:
+          SCRIPT:            /prepare_crypto_simdata.sh
+          DATE:              ${DAILY_SIM_CRYPTO_DATE:-}
+          SOURCE_HOST:       ${DAILY_SIM_CRYPTO_SOURCE_HOST:-cloud21.cvtt.vpn}
+          SOURCE_ROOT_DIR:   ${DAILY_SIM_CRYPTO_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto}
+          DB_SOURCE:         cloud28
+          OUTPUT_DIR:        /tmp
+
+          RSYNC_TARGETS:      ${DAILY_SIM_CRYPTO_RSYNC_TARGETS}
+          LOG_FILE:           daily_sim_crypto.log
+
+        volumes:
+          - ./logs:/logs
+          - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
+          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
+
+    daily_sim_equity:
+        image: cloud21.cvtt.vpn:5500/daily_sim:${MKTDATA_IMAGE_VERSION}
+        container_name: daily_sim_equity
+        user: ${CVTT_USER:-1001:1001}
+        pull_policy: always
+        restart: no
+        environment:
+          SCRIPT:            /prepare_eqt_simdata.sh
+          DATE:              ${DAILY_SIM_EQUITY_DATE:-}
+          SOURCE_HOST:       ${DAILY_SIM_EQUITY_SOURCE_HOST:-cloud21.cvtt.vpn}
+          SOURCE_ROOT_DIR:   ${DAILY_SIM_EQUITY_SOURCE_DIR:-/opt/store/cvtt/md_archive/crypto}
+          SYMBOL_LIST:       ${DAILY_SIM_EQUITY_SYMBOL_LIST}
+          OUTPUT_DIR:        /tmp
+
+          RSYNC_TARGETS:      ${DAILY_SIM_EQUITY_RSYNC_TARGETS}
+          LOG_FILE:           daily_sim_equity.log
+
+        volumes:
+          - ./logs:/logs
+          - ./config/identity/id_rsa:/home/cvtt/.ssh/id_rsa:ro
+          - ./config/identity/id_rsa.pub:/home/cvtt/.ssh/id_rsa.pub:ro
+networks:
+    default:
+        external: true
+        name: daily_mktdata_default