archive/docker_dev
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

creds to be taken from volumes

Browse Source
This commit is contained in:
Oleg Sheynin 2024-12-16 18:07:34 -05:00
parent 7165d611c6
commit 63831e16b1
26 changed files with 8 additions and 458 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
market_data/_OLD_/alpaca_md_day/.creds
View File

@ -1,7 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
}

3
market_data/_OLD_/alpaca_md_day/.dockerignore
View File

@ -1,3 +0,0 @@
.git
**/__pycache__
.pipenv

46
market_data/_OLD_/alpaca_md_day/Dockerfile
View File

@ -1,46 +0,0 @@
FROM python:3.12-slim
ARG FROM_DIR=docker_dev/market_data/alpaca_md_day
# Update the package list and install required packages
RUN apt-get update && apt-get install -y \
apt-utils \
libpq-dev \
build-essential
RUN apt-get update && apt-get install -y rsync openssh-client
RUN apt-get update && apt-get install -y --no-install-recommends \
software-properties-common \
curl \
parallel \
jq
RUN pip install --upgrade pip
COPY ${FROM_DIR}/requirements.txt /
RUN pip install -r /requirements.txt
COPY cvttpy /cvttpy
COPY ${FROM_DIR}/.creds /.creds
COPY ${FROM_DIR}/.creds /root/.creds
COPY ${FROM_DIR}/alpaca_md_day.sh /alpaca_md_day.sh
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
# Shared Volumes
RUN mkdir -p /app/data
RUN mkdir /logs
RUN chmod +x /alpaca_md_day.sh
WORKDIR /
SHELL ["/bin/bash", "-c"]
ENV PYTHONPATH=/
ENTRYPOINT [ "/alpaca_md_day.sh" ]
# CMD [ "echo", "alpaca_md_day"]

116
market_data/_OLD_/alpaca_md_day/HOWTO.md
View File

@ -1,116 +0,0 @@
# Build/Dev Host
## Build
```bash
Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
cd /home/oleg/develop/cvtt2
docker build -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
## Deploy
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
```
## On target Machine make sure user can use docker
```bash
sudo usermod -a -G docker cvtt
```
**Re-login as cvtt after adding cvtt to the group**
### Clean
```bash
docker rm -f crypto_md_day
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
```
### Test run locally
```
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
mkdir -p /tmp/data
mkdir -p /tmp/logs
docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
```
## Restarting
```bash
docker restart crypto_md_day
```
# Useful Commands
```bash
docker images
docker ps -a
docker image rm <image>
docker rm <container>
docker exec -it <container> /bin/bash
```
---
```bash
docker rm -f crypto_md_day
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
# Local test
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
```
## Local (cloud21) docker registry
### Run registry container (cloud21)
```bash
mkdir /opt/store/cvtt/docker_registry/data
vi /opt/store/cvtt/docker_registry/config.yml
```
```yaml
version: 0.1
log:
fields:
service: registry
storage:
filesystem:
rootdirectory: /var/lib/registry
http:
addr: :5500
```
```bash
docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
```
### Building
#### /etc/docker/daemon.json
```json
{
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
}
```
```bash
sudo systemctl restart docker
```
```bash
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
```
### Loading and Running (cryptoval4)
#### /etc/docker/daemon.json
```json
{
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
}
```
```bash
sudo systemctl restart docker
```
```
docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
```

138
market_data/_OLD_/alpaca_md_day/alpaca_md_day.sh
View File

@ -1,138 +0,0 @@
#!/bin/bash
# --- Settings
export PYTHONPATH=/
export Python=python3.12
export Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md
export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
export OutputDir=/app/data/alpaca_md # Local
export LogDir=/logs
DEFAULT_NUM_JOBS=10
SLEEP_SEC_AFTER_PROC=2
DEFAULT_SLICE_SIZE=500
# --- Settings
mkdir -p ${OutputDir}
mkdir -p ${LogDir}
usage() {
echo "Usage: $0 <date (YYYY-MM-DD)> [<num-jobs> (30) ] [<instrument_list_file>] "
exit 1
}
echo "CommandLine: ${*}"
Start=${1}
NumJobs=${2}
export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
is_business_day() {
dt=${1}
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
if [ -n "${open_time}" ]; then
return 0
else
return 1
fi
}
export -f is_business_day
if [ -z ${Start} ] ; then
echo "start is not specified, getting yesterday..."
Start=$(date -d "yesterday" "+%Y-%m-%d")
echo "Start=${Start}"
fi
while true; do
if is_business_day ${Start}; then
break
fi
echo "${Start} is not business day in US"
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
done
End=${Start} # only 1 day is practical
if [ "${NumJobs}" == "" ] ; then
NumJobs=${DEFAULT_NUM_JOBS}
fi
echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
run_proc() {
Inst=${1}
Start=${2}
End=${3}
echo "Running for $Inst"
Cmd="${Python} ${PyScript}"
Cmd="${Cmd} --config=${Config}"
Cmd="${Cmd} --output_dir=${OutputDir}"
Cmd="${Cmd} --instruments=ALPACA:${Inst}"
Cmd="${Cmd} --start=${Start}"
if [ "${End}" != "" ]; then
Cmd="${Cmd} --end=${End}"
fi
Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
Cmd="${Cmd} --log_level=WARNING"
echo ${Cmd}
eval ${Cmd}
sleep 2 #${SLEEP_SEC_AFTER_PROC}
}
export -f run_proc
key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
Cmd="${Cmd} --header 'accept: application/json'"
Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
Cmd="${Cmd} | sed 's/\"//g'"
Cmd="${Cmd} | sed 's/^/STOCK-/'"
# split string into array
Instruments=()
for Inst in $(eval ${Cmd})
do
Instruments+=("$Inst")
done
slice_size=${DEFAULT_SLICE_SIZE} # 10K symbols parallel cannot handle
for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
InstSlice=("${Instruments[@]:ii:slice_size}")
parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
done
echo "Compressing"
for file in $(find ${OutputDir} -type f -name '*db' -print )
do
echo "Compressing ${file}"
gzip ${file}
done
Source=/app/data/
Targets=
Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
for tgt in ${Targets}
do
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
echo $Cmd
eval $Cmd
done

7
market_data/_OLD_/alpaca_md_day/requirements.txt
View File

@ -1,7 +0,0 @@
aiohttp>=3.7.4.post0
nest-asyncio>=1.5.5
psycopg>=3.2.1
hjson>=3.1.0
pandas>=1.5.3
sortedcontainers>=2.4.0
redis>=5.0.8

7
market_data/alpaca_hbar/.creds
View File

@ -1,7 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKM1ASU8ULKSGO45ZZBV"
, "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9"
}
}

2
market_data/alpaca_hbar/Dockerfile
View File

@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y \
# must be from disted version
COPY cvtt-rust/bin/alpaca_md_hbar_loader /
COPY ${FROM_DIR}/.creds /.creds
# COPY ${FROM_DIR}/.creds /.creds
COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa

4
market_data/alpaca_hbar/alpaca_hbar.sh
View File

@ -43,10 +43,6 @@ while getopts ":d:N:" opt; do
esac
done
# if [ -z ${Symbols} ] ; then
# echo "symbols are not specified"
# usage
# fi
mkdir -p ${OutputDir}
mkdir -p ${LogDir}

7
market_data/alpaca_qat/.creds
View File

@ -1,7 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKM1ASU8ULKSGO45ZZBV"
, "secret_key": "XDBrrkOsxl9rMDs4GAFvG1WYiC3oLba04WbEZjR9"
}
}

2
market_data/alpaca_qat/Dockerfile
View File

@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y \
# must be from disted version
COPY cvtt-rust/bin/alpaca_md_qat_loader /
COPY ${FROM_DIR}/.creds /.creds
# COPY ${FROM_DIR}/.creds /.creds
COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa

17
market_data/crypto_md_day/.creds
View File

@ -1,17 +0,0 @@
{
"__dummy__": null
, "TSDB_MD_CLD21_RO": {
"host": "cloud21.cvtt.vpn"
, "port": 5432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "3Gkotqa0ZSmZLXSH3RkTUm"
}
, "TSDB_MD_CVTTDATA_RO": {
"host": "cvttdata.cvtt.vpn"
, "port": 5432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "3Gkotqa0ZSmZLXSH3RkTUm"
}
}

2
market_data/md_recorder/Dockerfile
View File

@ -12,7 +12,7 @@ RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${FROM_DIR}/.creds /.creds
# COPY ${FROM_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_md_recorder.sh /run_md_recorder.sh

2
market_data/md_recorder_monitor/Dockerfile
View File

@ -13,7 +13,7 @@ RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${FROM_DIR}/.creds /.creds
# COPY ${FROM_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_mdr_monitor.sh /run_mdr_monitor.sh

7
microservices/.creds
View File

@ -1,7 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
}

1
microservices/docker-compose.yml
View File

@ -7,6 +7,7 @@ services:
environment:
- REDIS_HOST=redis
volumes:
- ./.creds:/.creds
- ./config:/config
- ./logs:/logs
- ./data:/data

2
microservices/md_gateway/Dockerfile
View File

@ -8,7 +8,7 @@ RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${ROOT_MCRSVC_DIR}/.creds /.creds
# COPY ${ROOT_MCRSVC_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_mdgw.sh /run_mdgw.sh

2
microservices/tester/Dockerfile
View File

@ -8,7 +8,7 @@ RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY ${ROOT_MUSVC_DIR}/.creds /.creds
# COPY ${ROOT_MUSVC_DIR}/.creds /.creds
COPY ${FROM_DIR}/run_tester.sh /run_tester.sh

2
release_version.txt
View File

@ -1 +1 @@
0.0.6,musvcs progress
0.0.7,creds to be taken from volumes

12
trading/executor/.creds
View File

@ -1,12 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
, "MATTERMOST": {
"url": "https://mattermost.cryptovaltrading.com"
, "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
}
}

7
trading/md_portal/.creds
View File

@ -1,7 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
}

21
trading/quant/.creds
View File

@ -1,21 +0,0 @@
{
"TSDB_MD_CLD19_RO": {
"host": "cloud19.cvtt.vpn"
, "port": 5432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "3Gkotqa0ZSmZLXSH3RkTUm"
}
, "TSDB_MD_CVTTDATA_RO": {
"host": "cvttdata.cvtt.vpn"
, "port": 5432
, "user": "cvtt_ro"
, "database": "cvtt_md"
, "password": "3Gkotqa0ZSmZLXSH3RkTUm"
}
, "MATTERMOST": {
"url": "https://cloud11.cryptovaltrading.com"
, "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
}
}

20
trading/risk_mgr/.creds
View File

@ -1,20 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
, "TSDB_TRD_CVTTDATA": {
"host": "cvttdata.cvtt.vpn"
, "port": "5432"
, "database": "cvtt_trading"
, "user": "cvtt"
, "password": "ICdIh0JnMM7vM7Pf"
}
, "COINBASE_ADV_RO": {
"api_key": "Ip4QpRtRHBmWNpc3"
, "secret_key": "N6ZEMdIdUYUldkJRTUcCRKUFot4Fp2j0"
, "account_uuid": "???"
, "book_id": "CVTT_BK01"
}
}

8
trading/service_checker/.creds
View File

@ -1,8 +0,0 @@
{
"__dummy__": null
, "MATTERMOST": {
"url": "https://mattermost.cryptovaltrading.com"
, "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
}
}

12
trading/trader/.creds
View File

@ -1,12 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
, "MATTERMOST": {
"url": "https://mattermost.cryptovaltrading.com"
, "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
}
}

12
trading/trading_recorder/.creds
View File

@ -1,12 +0,0 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
, "MATTERMOST": {
"url": "https://mattermost.cryptovaltrading.com"
, "team": "CVTT"
, "bearer": "5ysaaxjeijrwjbmhuzcuos9ano"
}
}