merging

VERSION

@@ -1 +1 @@
-0.2.4,exch_acct_cfgname fixes
+0.2.7,new md_recorder build

config_service/.dockerignore

@@ -1,3 +0,0 @@
-.git
-**/__pycache__
-.pipenv

config_service/Dockerfile

@@ -1,22 +0,0 @@
-FROM python:3.12-slim
-
-WORKDIR /
-COPY docker_dev/config_service/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-
-# Shared Volumes
-RUN mkdir -p /app/data
-RUN mkdir /logs
-
-ENV PYTHONPATH=/
-CMD [                                        \
-    "python3.12",                            \
-    "cvttpy/apps/utils/config_server.py",    \
-    "--port=6789",                           \
-    "--root=/app/data",                      \
-    "--log_file=/logs/%T.config_service.log" \
-]
-

config_service/HOWTO.md

@@ -1,42 +0,0 @@
-# Build/Dev Host
-## Build
-```bash
-TgtHostname=cloud16.cvtt.vpn
-Version=$(cat cvttpy/VERSION)
-
-cd /home/oleg/develop/cvtt2
-docker build  -t config_service -t config_service:${Version} -f cvttpy/utils/docker/config_service/Dockerfile .
-
-## Deploy
-
-docker save -o /tmp/cvtt_config_service.img.tar config_service
-scp /tmp/cvtt_config_service.img.tar cvtt@${TgtHostname}:/tmp/
-
-ssh cvtt@${TgtHostname}
-```
-
-## On target Machine make sure user can use docker
-```bash
-sudo usermod -a -G docker cvtt
-```
-**Re-login as cvtt after adding cvtt to the group**
-
-```bash
-docker rm -f cvtt_config_service
-docker load -i /tmp/cvtt_config_service.img.tar
-docker run -d --name=cvtt_config_service -p 6789:6789 -v /home/cvtt/prod/config_service/data:/app/data -v /home/cvtt/prod/logs:/logs config_service
-```
-
-## Restarting
-```bash
-docker restart cvtt_config_service
-```
-
-# Useful Commands
-```bash
-docker images
-docker ps -a
-docker image rm <image>
-docker rm <container>
-docker exec -it <container> /bin/bash
-```

config_service/requirements.txt

@@ -1,2 +0,0 @@
-aiohttp>=3.7.4.post0
-nest-asyncio>=1.5.5

dckr_bld_functions.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+get_project_version() {
+    Project=${1}
+    Version=${2}
+
+    host="cloud21.cvtt.vpn"
+    port="22"
+    user="cvttdist"
+    rel_dir="/home/cvttdist/software/cvtt2"
+
+    if [ "${Version}" == "latest" ]; then
+        echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
+        Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
+        echo "Latest version is ${Version}"
+    fi
+    echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
+    if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
+    then
+        echo "Getting Version \"${Version}\"..."
+        rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
+        rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}/"
+        rsync_cmd="${rsync_cmd} ./"
+        echo ${rsync_cmd}
+        eval ${rsync_cmd}
+        status=$?
+        if [ ${status} -eq 0 ]
+        then
+            echo "Loading successful..."
+            return 0
+        else
+            return 1
+        fi
+    else
+        echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
+        return 1
+    fi
+}
+
+build_docker_image() {
+    ImageName=${1}
+    RegistryService=${2}
+    Version=${3}
+    Project=${4}
+    ProjectVersion=${5}
+
+    Cmd="docker build"
+    if [ "${ProjectVersion}" != "" ]; then
+        Cmd+=" --label ${Project}=\"${ProjectVersion}\""
+    fi
+    Cmd+=" -t ${ImageName}"
+    Cmd+=" -t ${ImageName}:latest"
+    Cmd+=" -t ${ImageName}:${Version}"
+    Cmd+=" -f Dockerfile"
+    Cmd+=" ."
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker push ${RegistryService}/${ImageName}:latest"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+
+    Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
+    echo ${Cmd}
+    eval ${Cmd} || exit
+}
+
+

market_data/alpaca_hbar/Dockerfile

@@ -24,7 +24,6 @@ COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
 RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
 
 
-
 # Shared Volumes
 RUN mkdir -p /data
 RUN mkdir /logs

market_data/md_recorder/.dockerignore

@@ -1,3 +0,0 @@
-.git
-**/__pycache__
-.pipenv

market_data/md_recorder/Dockerfile

@@ -1,27 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=docker_dev/market_data/md_recorder
-
-RUN apt-get update && apt-get install -y \
-    apt-utils \
-    libpq-dev \
-    build-essential
-
-COPY ${FROM_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh
-
-
-# Shared Volumes
-RUN mkdir -p /app/data
-RUN mkdir /logs
-RUN chmod +x /run_md_recorder.sh
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_md_recorder.sh" ]
-

market_data/md_recorder/deployment/cloud24/config/md_recorder.cfg

@@ -0,0 +1,206 @@
+{
+    "refdata": {
+        "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets
+        , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments
+        , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments
+        , "dynamic_instrument_exchanges": ["ALPACA"]
+        , "exchanges": {
+            "COINBASE_AT": {
+                "exchange_id":          "COINBASE"
+                , "name":               "Coinbase Advanced Trade"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # ****** NEW VAlUE
+                    # "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed"
+                    "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed"
+                    # ****** NEW VAlUE
+
+                    , "reconnect_secs": 2
+                    , "max_connections": 10
+                    , "price_throttle": true
+
+                    , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"]
+                }
+            }
+            , "BNBSPOT": {
+                "exchange_id":          "BNBSPOT"
+                , "name":               "Binance Spot"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed"
+                    "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+
+                    , "ws_url": ["WSS", "wss://stream.binance.com/ws"]
+                    , "REST": {
+                        "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"],
+                    }
+                }
+            }
+            , "BNBFUT": {
+                "exchange_id":          "BNBFUT"
+                , "name":               "Binance Futures USD-M"
+                , "type":               "FUTURES"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed"
+                    "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+                    , "ws_url": ["WSS", "wss://fstream.binance.com/ws"]
+
+                    , "REST": {
+                        # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000
+                        "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"],
+                    }
+                }
+            }
+        }
+    }
+    , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm
+
+    , "md_recorder": {
+        "instrument_groups": {
+            "BNBFUT_CLD21": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            "BNBFUT_CVTTDATA": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            , "BNBSPOT_CLD21": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "BNBSPOT_CVTTDATA": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "COINBASE_CLD21": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+            , "COINBASE_CVTTDATA": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+        }
+        , "db": {
+            "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient"
+        }
+        , "flush_interval_sec": 0.5
+
+        , "collections": {
+            "__dummy__": null
+            , "ALPACA": {
+                # dynamic instruments names
+                "tables": {
+                    "trades": "alpaca_iex.md_trades"
+                    , "booktops": "alpaca_iex.md_booktops"
+                    , "booksnaps": "alpaca_iex.md_booksnaps"
+                }
+            }
+            , "BNBSPOT": {
+                "instruments": {
+                    "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbspot.md_trades"
+                    , "booktops": "bnbspot.md_booktops"
+                    , "booksnaps": "bnbspot.md_booksnaps"
+                }
+            }
+            , "BNBFUT": {
+                "instruments": {
+                    "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbfut.md_trades"
+                    , "booktops": "bnbfut.md_booktops"
+                    , "booksnaps": "bnbfut.md_booksnaps"
+                }
+            }
+            , "COINBASE": {
+                "instruments": {
+                    "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "coinbase.md_trades"
+                    , "booktops": "coinbase.md_booktops"
+                    , "booksnaps": "coinbase.md_booksnaps"
+                }
+            }
+        }
+    }
+
+    # ----------------------------------
+    , "exchanges_settings": {
+        "*": {
+            "order_entry": {
+                "enabled": false
+                , "streaming": false
+            },
+            "mkt_data": {
+                "enabled": true
+                , "streaming": true
+            },
+            "accounting": {
+                "enabled": false
+                , "streaming": false
+            }
+        }
+    }
+}
+

market_data/md_recorder/deployment/cvtt-md/config/.creds

@@ -0,0 +1,17 @@
+{
+    "__dummy__": null
+    , "TSDB_MD_CLD21": {
+        "host": "cloud21.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+    , "TSDB_MD_CVTTDATA": {
+        "host": "cvttdata.cvtt.vpn"
+        , "port": 5432
+        , "user": "cvtt"
+        , "database": "cvtt_md"
+        , "password": "ICdIh0JnMM7vM7Pf"
+    }
+}

market_data/md_recorder/deployment/cvtt-md/config/md_recorder.cfg

@@ -0,0 +1,206 @@
+{
+    "refdata": {
+        "assets": @inc=http://@env{CONFIG_SERVICE}/refdata/assets
+        , "instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/instruments
+        , "exchange_instruments": @inc=http://@env{CONFIG_SERVICE}/refdata/exchange_instruments
+        , "dynamic_instrument_exchanges": ["ALPACA"]
+        , "exchanges": {
+            "COINBASE_AT": {
+                "exchange_id":          "COINBASE"
+                , "name":               "Coinbase Advanced Trade"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # ****** NEW VAlUE
+                    # "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMultiWsFeed"
+                    "class":  "cvttpy.exchanges.coinbase.spot.mkt_data.CoinbaseMdFeed"
+                    # ****** NEW VAlUE
+
+                    , "reconnect_secs": 2
+                    , "max_connections": 10
+                    , "price_throttle": true
+
+                    , "ws_url": ["WSS", "wss://ws-feed.exchange.coinbase.com"]
+                }
+            }
+            , "BNBSPOT": {
+                "exchange_id":          "BNBSPOT"
+                , "name":               "Binance Spot"
+                , "type":               "SPOT"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMultiWsFeed"
+                    "class": "cvttpy.exchanges.binance.spot.mkt_data.BnbSpotMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+
+                    , "ws_url": ["WSS", "wss://stream.binance.com/ws"]
+                    , "REST": {
+                        "depth_snapshot_url": ["GET", "https://api.binance.com/api/v3/depth"],
+                    }
+                }
+            }
+            , "BNBFUT": {
+                "exchange_id":          "BNBFUT"
+                , "name":               "Binance Futures USD-M"
+                , "type":               "FUTURES"
+
+                , "mktdata" : {
+                    # -------- NEW VALUE
+                    # "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMultiWsFeed"
+                    "class":           "cvttpy.exchanges.binance.futures.mkt_data.BnbFutMktDataFeed"
+                    # -------- NEW VALUE
+                    , "max_connections": 10
+                    , "reconnect_secs": 2.0
+
+                    , "ws_url": ["WSS", "wss://fstream.binance.com/ws"]
+
+                    , "REST": {
+                        # https://fapi.binance.com/fapi/v1/depth?symbol=BNBBTC&limit=1000
+                        "depth_snapshot_url": ["GET", "https://fapi.binance.com/fapi/v1/depth"],
+                    }
+                }
+            }
+        }
+    }
+    , "comm": @inc=http://@env{CONFIG_SERVICE}/apps/common/comm
+
+    , "md_recorder": {
+        "instrument_groups": {
+            "BNBFUT_CLD21": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            "BNBFUT_CVTTDATA": [
+                "BNBFUT:PERP-BTC-USDT"
+                , "BNBFUT:PERP-ETH-USDT"
+            ]
+            , "BNBSPOT_CLD21": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "BNBSPOT_CVTTDATA": [
+                "BNBSPOT:PAIR-ADA-USDT"
+                , "BNBSPOT:PAIR-BCH-USDT"
+                , "BNBSPOT:PAIR-BTC-USDT"
+                , "BNBSPOT:PAIR-DOT-USDT"
+                , "BNBSPOT:PAIR-ETH-USDT"
+                , "BNBSPOT:PAIR-LTC-USDT"
+                , "BNBSPOT:PAIR-SOL-USDT"
+                , "BNBSPOT:PAIR-USDC-USDT"
+                , "BNBSPOT:PAIR-XRP-USDT"
+            ]
+            , "COINBASE_CLD21": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+            , "COINBASE_CVTTDATA": [
+                "COINBASE:PAIR-ADA-USD"
+                , "COINBASE:PAIR-BCH-USD"
+                , "COINBASE:PAIR-BTC-USD"
+                , "COINBASE:PAIR-DOT-USD"
+                , "COINBASE:PAIR-ETH-USD"
+                , "COINBASE:PAIR-LTC-USD"
+                , "COINBASE:PAIR-SOL-USD"
+                , "COINBASE:PAIR-XRP-USD"
+            ]
+        }
+        , "db": {
+            "class": "cvttpy.tools.db.timescale.tsdb_client.TimescaleDbClient"
+        }
+        , "flush_interval_sec": 0.5
+
+        , "collections": {
+            "__dummy__": null
+            , "ALPACA": {
+                # dynamic instruments names
+                "tables": {
+                    "trades": "alpaca_iex.md_trades"
+                    , "booktops": "alpaca_iex.md_booktops"
+                    , "booksnaps": "alpaca_iex.md_booksnaps"
+                }
+            }
+            , "BNBSPOT": {
+                "instruments": {
+                    "PAIR-ADA-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-USDC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbspot.md_trades"
+                    , "booktops": "bnbspot.md_booktops"
+                    , "booksnaps": "bnbspot.md_booksnaps"
+                }
+            }
+            , "BNBFUT": {
+                "instruments": {
+                    "PERP-BTC-USDT": ["TRADES", "BOOK_TOP"]
+                    , "PERP-ETH-USDT": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "bnbfut.md_trades"
+                    , "booktops": "bnbfut.md_booktops"
+                    , "booksnaps": "bnbfut.md_booksnaps"
+                }
+            }
+            , "COINBASE": {
+                "instruments": {
+                    "PAIR-ADA-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BCH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-BTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-DOT-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-ETH-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-LTC-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-SOL-USD": ["TRADES", "BOOK_TOP"]
+                    , "PAIR-XRP-USD": ["TRADES", "BOOK_TOP"]
+                }
+                , "tables": {
+                    "trades": "coinbase.md_trades"
+                    , "booktops": "coinbase.md_booktops"
+                    , "booksnaps": "coinbase.md_booksnaps"
+                }
+            }
+        }
+    }
+
+    # ----------------------------------
+    , "exchanges_settings": {
+        "*": {
+            "order_entry": {
+                "enabled": false
+                , "streaming": false
+            },
+            "mkt_data": {
+                "enabled": true
+                , "streaming": true
+            },
+            "accounting": {
+                "enabled": false
+                , "streaming": false
+            }
+        }
+    }
+}
+

market_data/md_recorder/requirements.txt

@@ -1,7 +0,0 @@
-aiohttp>=3.7.4.post0
-nest-asyncio>=1.5.5
-psycopg>=3.2.1
-hjson>=3.1.0
-pandas>=1.5.3
-sortedcontainers>=2.4.0
-redis>=5.0.8

market_data/md_recorder/run_md_recorder.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-ValidJobs=('BNBFUT_CLD21' 'BNBSPOT_CLD21' 'COINBASE_CLD21' 'BNBFUT_CVTTDATA' 'BNBSPOT_CVTTDATA' 'COINBASE_CVTTDATA')
-# runs on host to start container
-usage() {
-  echo "Usage: $0 <job_name; one of (${ValidJobs[@]})>"
-  exit 1
-}
-
-is_valid() {
-  local job=$1
-  for valid_job in "${ValidJobs[@]}";
-  do
-    # echo "job=$job valid_job=$valid_job"
-    if [[ "${job}" == "${valid_job}" ]]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-job=${1}
-if  ! is_valid "${job}"; then
-    usage
-fi
-
-ConfigServer=cloud16.cvtt.vpn:6789
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
-Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --compress_log"
-Cmd="${Cmd} --instrument_group=${job}"
-Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
-if [ "${job}" == "BNBFUT_CLD21" ] ; then
-    Cmd="${Cmd} --exch_acct_cfgname=BNBFUT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
-    Cmd="${Cmd} --admin_port=7201"
-elif [ "${job}" == "COINBASE_CLD21" ] ; then
-    Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT"
-    Cmd="${Cmd} --exch_acct_cfgname=TSDB_MD_CLD21"
-    Cmd="${Cmd} --admin_port=7202"
-elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
-    Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
-    Cmd="${Cmd} --admin_port=7203"
-elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
-    Cmd="${Cmd} --exch_acct_cfgname=BNBSPOT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
-    Cmd="${Cmd} --admin_port=7204"
-elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
-    Cmd="${Cmd} --exch_acct_cfgname=BNBFUT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
-    Cmd="${Cmd} --admin_port=7205"
-elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
-    Cmd="${Cmd} --exch_acct_cfgname=COINBASE_AT"
-    Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
-    Cmd="${Cmd} --admin_port=7206"
-else
-    echo "Unrecognized JOB: ${job}"
-    exit 1
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/docker-compose.yml

@@ -1,52 +0,0 @@
-version: "3.9"
-
-services:
-    md_gateway:
-        image: cloud21.cvtt.vpn:5500/md_gateway:latest
-        container_name: md_gateway
-        environment:
-          - REDIS_HOST=redis
-        volumes:
-          - ./.creds:/.creds
-          - ./config:/config
-          - ./logs:/logs
-          - ./data:/data
-        depends_on:
-          - redis
-
-    tester:
-        image: cloud21.cvtt.vpn:5500/tester:latest
-        container_name: tester
-        environment:
-          - REDIS_HOST=redis
-        volumes:
-          - ./config:/config
-          - ./logs:/logs
-          - ./data:/data
-        depends_on:
-          - redis
-          - md_gateway
-
-    redis:
-        image: redis:latest
-        container_name: redis
-        ports:
-          - "6379:6379"
-        volumes:
-          - ./data/redis:/data
-
-
-#   test_program:
-#     image: your_test_program_image:latest
-#     container_name: test_program
-#     environment:
-#       - REDIS_HOST=redis
-#     volumes:
-#       - ./config:/shared/config
-#       - ./logs:/shared/logs
-#     depends_on:
-#       - market_data_gateway
-#       - redis
-
-# volumes:
-#   redis_data:

microservices/md_gateway/.dockerignore

@@ -1,3 +0,0 @@
-.git
-**/__pycache__
-.pipenv

microservices/md_gateway/Dockerfile

@@ -1,25 +0,0 @@
-FROM python:3.12-slim
-
-ARG ROOT_MCRSVC_DIR=docker_dev/microservices
-ARG FROM_DIR=${ROOT_MCRSVC_DIR}/md_gateway
-
-COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-# COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh
-RUN chmod +x /run_mdgw.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_mdgw.sh" ]
-

microservices/md_gateway/run_mdgw.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg)>]"
-    echo -n " [-a <admin_port (7220)>]"
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7220
-COMPRESS_LOG=false
-
-
-while getopts "c:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/md_gateway.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/oe_gateway/.dockerignore

@@ -1,3 +0,0 @@
-.git
-**/__pycache__
-.pipenv

microservices/oe_gateway/Dockerfile

@@ -1,25 +0,0 @@
-FROM python:3.12-slim
-
-ARG ROOT_MCRSVC_DIR=docker_dev/microservices
-ARG FROM_DIR=${ROOT_MCRSVC_DIR}/oe_gateway
-
-COPY ${ROOT_MCRSVC_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-COPY ${ROOT_MCRSVC_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_oegw.sh /run_oegw.sh
-RUN chmod +x /run_oegw.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_oegw.sh" ]
-

microservices/oe_gateway/run_oegw.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg)>]"
-    echo -n " [-a <admin_port (7220)>]"
-    echo -n " [-z (compress log)"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7220
-COMPRESS_LOG=false
-
-
-while getopts "c:a:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/market_gateways/order_entry_gateway.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --log_file=/logs/%T.md_gateway.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

microservices/requirements.txt

@@ -1,7 +0,0 @@
-aiohttp>=3.7.4.post0
-nest-asyncio>=1.5.5
-hjson>=3.1.0
-sortedcontainers>=2.4.0
-redis>=5.0.8
-python-dateutil>=2.8.2
-types-python-dateutil>=2.8.19.6

microservices/tester/.dockerignore

@@ -1,3 +0,0 @@
-.git
-**/__pycache__
-.pipenv

microservices/tester/Dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.12-slim
-
-ARG ROOT_MUSVC_DIR=docker_dev/microservices
-ARG FROM_DIR=${ROOT_MUSVC_DIR}/tester
-
-COPY ${ROOT_MUSVC_DIR}/requirements.txt /
-RUN pip install --upgrade pip --root-user-action=ignore
-RUN pip install -r /requirements.txt --root-user-action=ignore
-
-COPY cvttpy /cvttpy
-# COPY ${ROOT_MUSVC_DIR}/.creds  /.creds
-
-
-COPY ${FROM_DIR}/run_tester.sh /run_tester.sh
-RUN chmod +x /run_tester.sh
-
-RUN mkdir /logs
-RUN mkdir -p /config
-RUN mkdir -p /data
-
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/run_tester.sh" ]
-

microservices/tester/run_tester.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# runs in container
-
-# runs on host to start container
-usage() {
-    echo -n "Usage: ${0}"
-    echo -n " [-c <config (/config/cvtt.cfg) >]"
-    echo -n " [-i <instruments (COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT) >]"
-    echo -n " [-d <data types (TRADES) >]"
-    echo -n " [-a <admin_port (7221) >]"
-    echo -n " [-z (compress log (false))"]
-    echo
-    exit 1
-}
-
-Config=/config/cvtt.cfg
-AdminPort=7221
-COMPRESS_LOG=false
-Instruments="COINBASE:PAIR-BTC-USD,BNBFUT:PERP-BTC-USDT"
-DataTypes="TRADES"
-
-while getopts "a:c:d:i:z" opt; do
-    case ${opt} in
-        c ) Config=$OPTARG  ;;
-        a ) AdminPort=$OPTARG ;;
-        i ) Instruments=$OPTARG ;;
-        d ) DataTypes=$OPTARG ;;
-        z ) COMPRESS_LOG=true ;;
-        \? )
-            echo "Invalid option: -$OPTARG" >&2
-            usage
-            ;;
-        : )
-            echo "Option -$OPTARG requires an argument." >&2
-            usage
-            ;;
-    esac
-done
-
-Cmd="python3.12"
-Cmd="${Cmd} cvttpy/apps/microservices/tester/musvc_tester.py"
-Cmd="${Cmd} --config=${Config}"
-Cmd="${Cmd} --credentials_file=/.creds"
-Cmd="${Cmd} --admin_port=${AdminPort}"
-Cmd="${Cmd} --data_types=${DataTypes}"
-Cmd="${Cmd} --instruments=${Instruments}"
-Cmd="${Cmd} --log_file=/logs/%T.tester.log"
-if ${COMPRESS_LOG} ; then
-    Cmd="${Cmd} --compress_log"
-fi
-echo ${Cmd}
-eval ${Cmd}
-
-

test/rsync_test/Dockerfile

@@ -1,21 +0,0 @@
-FROM python:3.12-slim
-
-ARG FROM_DIR=docker_dev/test/rsync_test
-
-COPY ${FROM_DIR}/test.sh /test.sh
-
-RUN apt-get update && apt-get install -y rsync openssh-client
-
-COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
-COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
-
-RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
-
-# Shared Volumes
-RUN chmod +x /test.sh
-
-WORKDIR /
-SHELL ["/bin/bash", "-c"]
-ENV PYTHONPATH=/
-ENTRYPOINT [ "/test.sh" ]
-

test/rsync_test/test.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-date > /testfile.txt
-
-Source=/testfile.txt
-Targets=
-Targets="${Targets} cvtt@cloud21.cvtt.vpn:/tmp/"
-Targets="${Targets} cvtt@hs01.cvtt.vpn:/tmp/"
-
-
-for tgt in ${Targets}
-do
-    Cmd="/usr/bin/rsync -ahv"
-    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
-    Cmd+=" ${Source} ${tgt}"
-    echo $Cmd
-    eval $Cmd
-done
-