diff --git a/market_data/_OLD_/alpaca_md_day/.creds b/market_data/_OLD_/alpaca_md_day/.creds deleted file mode 100644 index 0a3b012..0000000 --- a/market_data/_OLD_/alpaca_md_day/.creds +++ /dev/null @@ -1,7 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKLZSLFZMFMN1R28K9HK" - , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" - } -} diff --git a/market_data/_OLD_/alpaca_md_day/.dockerignore b/market_data/_OLD_/alpaca_md_day/.dockerignore deleted file mode 100644 index a4f7c14..0000000 --- a/market_data/_OLD_/alpaca_md_day/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.git -**/__pycache__ -.pipenv \ No newline at end of file diff --git a/market_data/_OLD_/alpaca_md_day/Dockerfile b/market_data/_OLD_/alpaca_md_day/Dockerfile deleted file mode 100644 index 8af44d6..0000000 --- a/market_data/_OLD_/alpaca_md_day/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -FROM python:3.12-slim - -ARG FROM_DIR=docker_dev/market_data/alpaca_md_day - -# Update the package list and install required packages -RUN apt-get update && apt-get install -y \ - apt-utils \ - libpq-dev \ - build-essential -RUN apt-get update && apt-get install -y rsync openssh-client - -RUN apt-get update && apt-get install -y --no-install-recommends \ - software-properties-common \ - curl \ - parallel \ - jq - -RUN pip install --upgrade pip - - -COPY ${FROM_DIR}/requirements.txt / -RUN pip install -r /requirements.txt - -COPY cvttpy /cvttpy -COPY ${FROM_DIR}/.creds /.creds -COPY ${FROM_DIR}/.creds /root/.creds -COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh - -COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa -COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub -COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts - -RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts - - -# Shared Volumes -RUN mkdir -p /app/data -RUN mkdir /logs -RUN chmod +x /alpaca_md_day.sh - -WORKDIR / -SHELL ["/bin/bash", "-c"] -ENV PYTHONPATH=/ -ENTRYPOINT [ "/alpaca_md_day.sh" ] - -# CMD [ "echo", "alpaca_md_day"] diff --git a/market_data/_OLD_/alpaca_md_day/HOWTO.md b/market_data/_OLD_/alpaca_md_day/HOWTO.md deleted file mode 100644 index 611d30e..0000000 --- a/market_data/_OLD_/alpaca_md_day/HOWTO.md +++ /dev/null @@ -1,116 +0,0 @@ -# Build/Dev Host -## Build -```bash -Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt) - -cd /home/oleg/develop/cvtt2 -docker build -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile . - -## Deploy - -/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day - -``` - -## On target Machine make sure user can use docker -```bash -sudo usermod -a -G docker cvtt -``` -**Re-login as cvtt after adding cvtt to the group** - -### Clean -```bash -docker rm -f crypto_md_day -docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f -``` -### Test run locally -``` -docker load -i /home/oleg/docker_images/crypto_md_day.img.tar -mkdir -p /tmp/data -mkdir -p /tmp/logs -docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day -``` - -## Restarting -```bash -docker restart crypto_md_day -``` - -# Useful Commands -```bash -docker images -docker ps -a -docker image rm -docker rm -docker exec -it /bin/bash -``` - ---- -```bash -docker rm -f crypto_md_day -docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f - -/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn -docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest -docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest - -# Local test -docker load -i /home/oleg/docker_images/crypto_md_day.img.tar && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase -``` - -## Local (cloud21) docker registry -### Run registry container (cloud21) - - -```bash -mkdir /opt/store/cvtt/docker_registry/data -vi /opt/store/cvtt/docker_registry/config.yml -``` - -```yaml -version: 0.1 -log: - fields: - service: registry -storage: - filesystem: - rootdirectory: /var/lib/registry -http: - addr: :5500 - -``` -```bash -docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2 -``` - -### Building -#### /etc/docker/daemon.json -```json -{ - "insecure-registries": ["cloud21.cvtt.vpn:5500"] -} -``` -```bash -sudo systemctl restart docker -``` -```bash -/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn -docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest -docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest -``` - -### Loading and Running (cryptoval4) -#### /etc/docker/daemon.json -```json -{ - "insecure-registries": ["cloud21.cvtt.vpn:5500"] -} -``` -```bash -sudo systemctl restart docker -``` -``` -docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest -docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase -``` - diff --git a/market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh b/market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh deleted file mode 100755 index ac08a54..0000000 --- a/market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash - -# --- Settings -export PYTHONPATH=/ -export Python=python3.12 -export Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md -export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py - -export OutputDir=/app/data/alpaca_md # Local -export LogDir=/logs - -DEFAULT_NUM_JOBS=10 -SLEEP_SEC_AFTER_PROC=2 -DEFAULT_SLICE_SIZE=500 -# --- Settings - -mkdir -p ${OutputDir} -mkdir -p ${LogDir} - - -usage() { - echo "Usage: $0 [ (30) ] [] " - exit 1 -} - -echo "CommandLine: ${*}" - -Start=${1} -NumJobs=${2} - -export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS - -is_business_day() { - dt=${1} - - open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time') - if [ -n "${open_time}" ]; then - return 0 - else - return 1 - fi - -} -export -f is_business_day - - -if [ -z ${Start} ] ; then - echo "start is not specified, getting yesterday..." - Start=$(date -d "yesterday" "+%Y-%m-%d") - echo "Start=${Start}" -fi - -while true; do - if is_business_day ${Start}; then - break - fi - echo "${Start} is not business day in US" - Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d") -done - -End=${Start} # only 1 day is practical - -if [ "${NumJobs}" == "" ] ; then - NumJobs=${DEFAULT_NUM_JOBS} -fi - -echo "Start=${Start} End=${End} NumJobs=${NumJobs}" - - -run_proc() { - Inst=${1} - Start=${2} - End=${3} - echo "Running for $Inst" - - Cmd="${Python} ${PyScript}" - Cmd="${Cmd} --config=${Config}" - Cmd="${Cmd} --output_dir=${OutputDir}" - Cmd="${Cmd} --instruments=ALPACA:${Inst}" - Cmd="${Cmd} --start=${Start}" - if [ "${End}" != "" ]; then - Cmd="${Cmd} --end=${End}" - fi - Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log" - Cmd="${Cmd} --log_level=WARNING" - echo ${Cmd} - eval ${Cmd} - - sleep 2 #${SLEEP_SEC_AFTER_PROC} -} - -export -f run_proc - -key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds) -secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds) - -Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'" -Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'" -Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'" -Cmd="${Cmd} --header 'accept: application/json'" -Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'" -Cmd="${Cmd} | sed 's/\"//g'" -Cmd="${Cmd} | sed 's/^/STOCK-/'" -# split string into array -Instruments=() -for Inst in $(eval ${Cmd}) -do - Instruments+=("$Inst") -done - -slice_size=${DEFAULT_SLICE_SIZE} # 10K symbols parallel cannot handle -for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do - InstSlice=("${Instruments[@]:ii:slice_size}") - - parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}" -done - -echo "Compressing" -for file in $(find ${OutputDir} -type f -name '*db' -print ) -do - echo "Compressing ${file}" - gzip ${file} -done - - -Source=/app/data/ -Targets= -Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity" -Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" - - -for tgt in ${Targets} -do - Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}" - echo $Cmd - eval $Cmd -done - diff --git a/market_data/_OLD_/alpaca_md_day/requirements.txt b/market_data/_OLD_/alpaca_md_day/requirements.txt deleted file mode 100644 index 3242ddd..0000000 --- a/market_data/_OLD_/alpaca_md_day/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -aiohttp>=3.7.4.post0 -nest-asyncio>=1.5.5 -psycopg>=3.2.1 -hjson>=3.1.0 -pandas>=1.5.3 -sortedcontainers>=2.4.0 -redis>=5.0.8 \ No newline at end of file diff --git a/market_data/alpaca_hbar/.creds b/market_data/alpaca_hbar/.creds deleted file mode 100644 index 8e41e10..0000000 --- a/market_data/alpaca_hbar/.creds +++ /dev/null @@ -1,7 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKM1ASU8ULKSGO45ZZBV" - , "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9" - } -} diff --git a/market_data/alpaca_hbar/Dockerfile b/market_data/alpaca_hbar/Dockerfile index 56d672a..345510e 100644 --- a/market_data/alpaca_hbar/Dockerfile +++ b/market_data/alpaca_hbar/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y \ # must be from disted version COPY cvtt-rust/bin/alpaca_md_hbar_loader / -COPY ${FROM_DIR}/.creds /.creds +# COPY ${FROM_DIR}/.creds /.creds COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa diff --git a/market_data/alpaca_hbar/alpaca_hbar.sh b/market_data/alpaca_hbar/alpaca_hbar.sh index 2009283..2cefb2b 100755 --- a/market_data/alpaca_hbar/alpaca_hbar.sh +++ b/market_data/alpaca_hbar/alpaca_hbar.sh @@ -43,10 +43,6 @@ while getopts ":d:N:" opt; do esac done -# if [ -z ${Symbols} ] ; then -# echo "symbols are not specified" -# usage -# fi mkdir -p ${OutputDir} mkdir -p ${LogDir} diff --git a/market_data/alpaca_qat/.creds b/market_data/alpaca_qat/.creds deleted file mode 100644 index 8e41e10..0000000 --- a/market_data/alpaca_qat/.creds +++ /dev/null @@ -1,7 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKM1ASU8ULKSGO45ZZBV" - , "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9" - } -} diff --git a/market_data/alpaca_qat/Dockerfile b/market_data/alpaca_qat/Dockerfile index e79ecbb..2d7bc2e 100644 --- a/market_data/alpaca_qat/Dockerfile +++ b/market_data/alpaca_qat/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y \ # must be from disted version COPY cvtt-rust/bin/alpaca_md_qat_loader / -COPY ${FROM_DIR}/.creds /.creds +# COPY ${FROM_DIR}/.creds /.creds COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa diff --git a/market_data/crypto_md_day/.creds b/market_data/crypto_md_day/.creds deleted file mode 100644 index 0eef25c..0000000 --- a/market_data/crypto_md_day/.creds +++ /dev/null @@ -1,17 +0,0 @@ -{ - "__dummy__": null - , "TSDB_MD_CLD21_RO": { - "host": "cloud21.cvtt.vpn" - , "port": 5432 - , "user": "cvtt_ro" - , "database": "cvtt_md" - , "password": "3Gkotqa0ZSmZLXSH3RkTUm" - } - , "TSDB_MD_CVTTDATA_RO": { - "host": "cvttdata.cvtt.vpn" - , "port": 5432 - , "user": "cvtt_ro" - , "database": "cvtt_md" - , "password": "3Gkotqa0ZSmZLXSH3RkTUm" - } -} diff --git a/market_data/md_recorder/Dockerfile b/market_data/md_recorder/Dockerfile index 4e77fa9..8561e66 100644 --- a/market_data/md_recorder/Dockerfile +++ b/market_data/md_recorder/Dockerfile @@ -12,7 +12,7 @@ RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY ${FROM_DIR}/.creds /.creds +# COPY ${FROM_DIR}/.creds /.creds COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh diff --git a/market_data/md_recorder_monitor/Dockerfile b/market_data/md_recorder_monitor/Dockerfile index 2ce5e80..0237f41 100644 --- a/market_data/md_recorder_monitor/Dockerfile +++ b/market_data/md_recorder_monitor/Dockerfile @@ -13,7 +13,7 @@ RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY ${FROM_DIR}/.creds /.creds +# COPY ${FROM_DIR}/.creds /.creds COPY ${FROM_DIR}/run_mdr_monitor.sh /run_mdr_monitor.sh diff --git a/microservices/.creds b/microservices/.creds deleted file mode 100644 index 0a3b012..0000000 --- a/microservices/.creds +++ /dev/null @@ -1,7 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKLZSLFZMFMN1R28K9HK" - , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" - } -} diff --git a/microservices/docker-compose.yml b/microservices/docker-compose.yml index ef17e00..dd779c1 100644 --- a/microservices/docker-compose.yml +++ b/microservices/docker-compose.yml @@ -7,6 +7,7 @@ services: environment: - REDIS_HOST=redis volumes: + - ./.creds:/.creds - ./config:/config - ./logs:/logs - ./data:/data diff --git a/microservices/md_gateway/Dockerfile b/microservices/md_gateway/Dockerfile index 9edc896..5a421a3 100644 --- a/microservices/md_gateway/Dockerfile +++ b/microservices/md_gateway/Dockerfile @@ -8,7 +8,7 @@ RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY ${ROOT_MCRSVC_DIR}/.creds /.creds +# COPY ${ROOT_MCRSVC_DIR}/.creds /.creds COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh diff --git a/microservices/tester/Dockerfile b/microservices/tester/Dockerfile index 0a3439e..4f8e8ed 100644 --- a/microservices/tester/Dockerfile +++ b/microservices/tester/Dockerfile @@ -8,7 +8,7 @@ RUN pip install --upgrade pip --root-user-action=ignore RUN pip install -r /requirements.txt --root-user-action=ignore COPY cvttpy /cvttpy -COPY ${ROOT_MUSVC_DIR}/.creds /.creds +# COPY ${ROOT_MUSVC_DIR}/.creds /.creds COPY ${FROM_DIR}/run_tester.sh /run_tester.sh diff --git a/release_version.txt b/release_version.txt index 86030f5..4d04336 100644 --- a/release_version.txt +++ b/release_version.txt @@ -1 +1 @@ -0.0.6,musvcs progress \ No newline at end of file +0.0.7,creds to be taken from volumes \ No newline at end of file diff --git a/trading/executor/.creds b/trading/executor/.creds deleted file mode 100644 index 4b66d91..0000000 --- a/trading/executor/.creds +++ /dev/null @@ -1,12 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKLZSLFZMFMN1R28K9HK" - , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" - } - , "MATTERMOST": { - "url": "https://mattermost.cryptovaltrading.com" - , "team": "CVTT" - , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" - } -} diff --git a/trading/md_portal/.creds b/trading/md_portal/.creds deleted file mode 100644 index 0a3b012..0000000 --- a/trading/md_portal/.creds +++ /dev/null @@ -1,7 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKLZSLFZMFMN1R28K9HK" - , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" - } -} diff --git a/trading/quant/.creds b/trading/quant/.creds deleted file mode 100644 index 137def0..0000000 --- a/trading/quant/.creds +++ /dev/null @@ -1,21 +0,0 @@ -{ - "TSDB_MD_CLD19_RO": { - "host": "cloud19.cvtt.vpn" - , "port": 5432 - , "user": "cvtt_ro" - , "database": "cvtt_md" - , "password": "3Gkotqa0ZSmZLXSH3RkTUm" - } - , "TSDB_MD_CVTTDATA_RO": { - "host": "cvttdata.cvtt.vpn" - , "port": 5432 - , "user": "cvtt_ro" - , "database": "cvtt_md" - , "password": "3Gkotqa0ZSmZLXSH3RkTUm" - } - , "MATTERMOST": { - "url": "https://cloud11.cryptovaltrading.com" - , "team": "CVTT" - , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" - } -} diff --git a/trading/risk_mgr/.creds b/trading/risk_mgr/.creds deleted file mode 100644 index 40f7f12..0000000 --- a/trading/risk_mgr/.creds +++ /dev/null @@ -1,20 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKLZSLFZMFMN1R28K9HK" - , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" - } - , "TSDB_TRD_CVTTDATA": { - "host": "cvttdata.cvtt.vpn" - , "port": "5432" - , "database": "cvtt_trading" - , "user": "cvtt" - , "password": "ICdIh0JnMM7vM7Pf" - } - , "COINBASE_ADV_RO": { - "api_key": "Ip4QpRtRHBmWNpc3" - , "secret_key": "N6ZEMdIdUYUldkJRTUcCRKUFot4Fp2j0" - , "account_uuid": "???" - , "book_id": "CVTT_BK01" - } -} diff --git a/trading/service_checker/.creds b/trading/service_checker/.creds deleted file mode 100644 index aae4c4d..0000000 --- a/trading/service_checker/.creds +++ /dev/null @@ -1,8 +0,0 @@ -{ - "__dummy__": null - , "MATTERMOST": { - "url": "https://mattermost.cryptovaltrading.com" - , "team": "CVTT" - , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" - } -} diff --git a/trading/trader/.creds b/trading/trader/.creds deleted file mode 100644 index 4b66d91..0000000 --- a/trading/trader/.creds +++ /dev/null @@ -1,12 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKLZSLFZMFMN1R28K9HK" - , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" - } - , "MATTERMOST": { - "url": "https://mattermost.cryptovaltrading.com" - , "team": "CVTT" - , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" - } -} diff --git a/trading/trading_recorder/.creds b/trading/trading_recorder/.creds deleted file mode 100644 index 4b66d91..0000000 --- a/trading/trading_recorder/.creds +++ /dev/null @@ -1,12 +0,0 @@ -{ - "__dummy__": null - , "ALPACA_SANDBOX": { - "api_key": "PKLZSLFZMFMN1R28K9HK" - , "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA" - } - , "MATTERMOST": { - "url": "https://mattermost.cryptovaltrading.com" - , "team": "CVTT" - , "bearer": "5ysaaxjeijrwjbmhuzcuos9ano" - } -}