This commit is contained in:
Oleg Sheynin 2024-07-22 12:59:53 -04:00
parent 4d5806f3eb
commit 0b437a1d60
11 changed files with 304 additions and 16 deletions

7
alpaca_md_day/.creds Normal file
View File

@ -0,0 +1,7 @@
{
"__dummy__": null
, "ALPACA_SANDBOX": {
"api_key": "PKLZSLFZMFMN1R28K9HK"
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
}
}

32
alpaca_md_day/Dockerfile Normal file
View File

@ -0,0 +1,32 @@
FROM python:3.10-slim
WORKDIR /
SHELL ["/bin/bash", "-c"]
COPY docker_dev/alpaca_md_day/requirements.txt /
RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy
COPY docker_dev/alpaca_md_day/.creds /.creds
COPY docker_dev/alpaca_md_day/alpaca_md_day.sh /alpaca_md_day.sh
# RUN apt install openssh-client -y
RUN apt-get update && apt-get install -y --no-install-recommends rsync openssh-client && rm -rf /var/lib/apt/lists/*
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
# Shared Volumes
RUN mkdir -p /app/data
RUN mkdir /logs
RUN chmod +x /alpaca_md_day.sh
ENTRYPOINT [ "/alpaca_md_day.sh" ]
ENV PYTHONPATH=/
CMD [ "echo", "alpaca_md_day"]

116
alpaca_md_day/HOWTO.md Normal file
View File

@ -0,0 +1,116 @@
# Build/Dev Host
## Build
```bash
Version=$(cat /home/oleg/develop/cvtt2/cvttpy/release_version.txt)
cd /home/oleg/develop/cvtt2
docker build -t crypto_md_day -t crypto_md_day:${Version} -f docker_dev/crypto_md_day/Dockerfile .
## Deploy
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day
```
## On target Machine make sure user can use docker
```bash
sudo usermod -a -G docker cvtt
```
**Re-login as cvtt after adding cvtt to the group**
### Clean
```bash
docker rm -f crypto_md_day
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
```
### Test run locally
```
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar
mkdir -p /tmp/data
mkdir -p /tmp/logs
docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data -v /tmp/logs:/logs crypto_md_day
```
## Restarting
```bash
docker restart crypto_md_day
```
# Useful Commands
```bash
docker images
docker ps -a
docker image rm <image>
docker rm <container>
docker exec -it <container> /bin/bash
```
---
```bash
docker rm -f crypto_md_day
docker images --filter=reference=crypto_md_day --format="{{.ID}}" | uniq | xargs docker rmi -f
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
# Local test
docker load -i /home/oleg/docker_images/crypto_md_day.img.tar && docker run --network="host" --name=crypto_md_day -v /tmp/data:/app/data crypto_md_day -h cvttdata -d 20240717 -s coinbase
```
## Local (cloud21) docker registry
### Run registry container (cloud21)
```bash
mkdir /opt/store/cvtt/docker_registry/data
vi /opt/store/cvtt/docker_registry/config.yml
```
```yaml
version: 0.1
log:
fields:
service: registry
storage:
filesystem:
rootdirectory: /var/lib/registry
http:
addr: :5500
```
```bash
docker run -d -p 5500:5500 --name registry -v /opt/store/cvtt/docker_registry/config.yml:/etc/docker/registry/config.yml -v /opt/store/cvtt/docker_registry/data:/var/lib/registry registry:2
```
### Building
#### /etc/docker/daemon.json
```json
{
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
}
```
```bash
sudo systemctl restart docker
```
```bash
/home/oleg/develop/cvtt2/docker_dev/build.sh crypto_md_day cryptoval4.cvtt.vpn
docker tag crypto_md_day:latest cloud21.cvtt.vpn:5500/crypto_md_day:latest
docker push cloud21.cvtt.vpn:5500/crypto_md_day:latest
```
### Loading and Running (cryptoval4)
#### /etc/docker/daemon.json
```json
{
"insecure-registries": ["cloud21.cvtt.vpn:5500"]
}
```
```bash
sudo systemctl restart docker
```
```
docker pull cloud21.cvtt.vpn:5500/crypto_md_day:latest
docker run --network="host" --name=cloud21.cvtt.vpn:5500/crypto_md_day crypto_md_day -h cvttdata -d 20240717 -s coinbase
```

81
alpaca_md_day/crypto_md_day.sh Executable file
View File

@ -0,0 +1,81 @@
#!/bin/bash
usage() {
echo "Usage: $0 -h <db host> -d <YYYYMMDD Date> -s <schemas|coinbase,bnbspot,bnbfut>"
exit 1
}
DbHost=""
Date=""
Schemas=coinbase,bnbspot,bnbfut
while getopts ":h:d:s:" opt; do
case ${opt} in
h )
DbHost=$OPTARG
;;
d )
Date=$OPTARG
;;
s )
Schemas=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ -z "${DbHost}" ] || [ -z "$Date" ]; then
echo "DbHost=${DbHost} Date=${Date}. Cmdline: ${*}"
usage
fi
CredKey=""
if [ "${DbHost}" == "cvttdata" ] ; then
CredKey=TSDB_MD_CVTTDATA_RO
elif [ "${DbHost}" == "cloud21" ] ; then
CredKey=TSDB_MD_CLD21_RO
else
echo "host ${DbHost} is not valid source"
usage
fi
export PYTHONPATH=/
Cmd="python3.10"
Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
Cmd="${Cmd} --db_credentials_key=${CredKey}"
Cmd="${Cmd} --credentials_file=/.creds"
Cmd="${Cmd} --date=${Date}"
Cmd="${Cmd} --schemas=${Schemas}"
Cmd="${Cmd} --root_dir=/app/data/${DbHost}"
Cmd="${Cmd} --format=SQLite"
Cmd="${Cmd} --compress"
# Cmd="${Cmd} --log_file=/logs/%T.crypto_md_day.log"
echo ${Cmd}
eval ${Cmd}
echo "Data Sync"
Source=/app/data/
Targets=
Targets="${Targets} oleg@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto"
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto"
for tgt in ${Targets}
do
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
echo $Cmd
eval $Cmd
done

View File

@ -0,0 +1,6 @@
aiohttp>=3.7.4.post0
nest-asyncio>=1.5.5
asyncpg>=0.27.0
hjson>=3.1.0
pandas>=1.5.3
sortedcontainers>=2.4.0

View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
function usage { function usage {
echo "Usage: ${0} <app_name> <target_host>" echo "Usage: ${0} <app_name>"
exit exit
} }
@ -11,25 +11,16 @@ if [ "${AppName}" == "" ]
then then
usage usage
fi fi
shift
TgtHostname=${1}
if [ "${TgtHostname}" == "" ]
then
usage
fi
# --- Settings # --- Settings
DockerDir=$(realpath $(dirname ${0})/${AppName}) DockerDir=$(realpath $(dirname ${0})/${AppName})
RootDir=$(realpath ${DockerDir}/../..) RootDir=$(realpath ${DockerDir}/../..)
RegistryService=cloud21.cvtt.vpn:5500
ImageName=${AppName} ImageName=${AppName}
ImageDir=${HOME}/docker_images ImageDir=${HOME}/docker_images
mkdir -p ${ImageDir} mkdir -p ${ImageDir}
RegistryService=cloud21.cvtt.vpn:5500
Version=$(cat ${RootDir}/cvttpy/release_version.txt) Version=$(cat ${RootDir}/cvttpy/release_version.txt)

View File

@ -1,9 +1,9 @@
FROM python:3.10-slim FROM python:3.10-slim
WORKDIR / WORKDIR /
COPY cvttpy/utils/docker/config_service/requirements.txt / COPY docker_dev/config_service/requirements.txt /
RUN pip install --upgrade pip RUN pip install --upgrade pip --root-user-action=ignore
RUN pip install -r /requirements.txt RUN pip install -r /requirements.txt --root-user-action=ignore
COPY cvttpy /cvttpy COPY cvttpy /cvttpy

View File

@ -6,8 +6,6 @@ usage() {
exit 1 exit 1
} }
echo "SHELL=${SHELL}"
DbHost="" DbHost=""
Date="" Date=""
Schemas=coinbase,bnbspot,bnbfut Schemas=coinbase,bnbspot,bnbfut
@ -35,6 +33,7 @@ while getopts ":h:d:s:" opt; do
done done
if [ -z "${DbHost}" ] || [ -z "$Date" ]; then if [ -z "${DbHost}" ] || [ -z "$Date" ]; then
echo "DbHost=${DbHost} Date=${Date}. Cmdline: ${*}"
usage usage
fi fi

49
shared/id_rsa Normal file
View File

@ -0,0 +1,49 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
NhAAAAAwEAAQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMNo1DzPcnqFWgBUki5Sy/F
1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZkf2AUhRMhYlroXp7emmI7
XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOypqCQeoHpI2imhe2Fr7rg
fEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEtkmsONS0XzFEaVwx/wBTl
kLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7HVwU+LkONJmyy4YJdbET
SxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1sFqEfiYFnGpXeMZvjnAp
/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xVyagoxfwMqMjgUgZ4ecoK
ga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y7bu1uWfmWKVcSRw0729J
7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yiWvLaDWAtKDcickkzqj8Q
uUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1miCVSVpOFBmEihUFnRvRjj
kAAAdYoVe5GqFXuRoAAAAHc3NoLXJzYQAAAgEAsFK61F50xNGRFNCPNGa2EtMFkQEZWNMN
o1DzPcnqFWgBUki5Sy/F1jU73y3arh+wasWsm6MbbJyBwmUv3mwilDSeDPYZVBjXwirMZk
f2AUhRMhYlroXp7emmI7XZJhg+vq3Nur7y3OG1nns4uPDIFuQzAHlfaLTd/5iBinHveyOy
pqCQeoHpI2imhe2Fr7rgfEHZf9UA2k48LDkxZ/d7h4cmSLRTReD0/WI+faF2L0v36MwnEt
kmsONS0XzFEaVwx/wBTlkLtRGW0BZRfwV56Qg9Jk5iVcy9qktkU1Oca6/TOu5f/BZiBnc7
HVwU+LkONJmyy4YJdbETSxNDxmX5DbiWIT4uQKuj/9spgJDgJtmgSjynNmjXCZ6WFRfQa1
sFqEfiYFnGpXeMZvjnAp/7IH+v4ZEBhH9rUbnDZHjS4xhOHzGuGuz7MH47blsD4m82s3xV
yagoxfwMqMjgUgZ4ecoKga9T+qNuUyqnn7n5nAJaQuX4nwYthdSrKo3sAm0dScRWvvXZ/Y
7bu1uWfmWKVcSRw0729J7fcDY1vyD4GBJ25vcYhNO+F3sqYKDEaUgDOHu2mzi9y8DzI1yi
WvLaDWAtKDcickkzqj8QuUOxxQc09OoijlAGx3wKGqmZMONRlJy/ldeG/e+IUukjn6h1mi
CVSVpOFBmEihUFnRvRjjkAAAADAQABAAACABKTOpWsGDzST4iL8qQsdF6uDqpwnVMZU9ja
Sq++FVwZM3dvzB5uu1amT8cCkYSxt6/94GHaiGV0JOrd5lG2Hlvy4ZoftHoirlspD1SwnM
rUi6I9SU22E5dx24PsUMKlv/luPavAWT06BXw7QrXf/tJDithwUOPq1u7J6g6z00zrjTLm
3CWdyTnm2NEn3UMuqp7IgKFqBv+YiZgGk+z+GkxEKX00CwhD60dEyezJ/VV8+TJUDIM8r2
nn3tYNIQUrgsK8qoYOlTBHeLX1/fKl2Oqa6Lc//3XGvo3hyKFLIAHLuXLeQSwhTwUbmzWL
+XUc6LFTpk1w0s8MXyoT7tpILJkPnP+uPtc8vGL0xFWzlUtbeipWWowIvzEfbZPisy3OmE
QfRmHf7KicQH0d2p43lHNwbdTKegdiZDOkbukkkm3G+0Ft/I51swA2AuZ5CAtU+i77aGB6
kQdLKOe3udwEKyNGEi9RIlMlTC/XFbuTkM3iywZoxHYqRfdkf8S+3XIRlik8Mc1UsdKg8Y
r1g/vD7z9lT2gahdY0n2uRcpZKqFTuJwIR70WaAbgh8u8N2ciEZDx9Pnvk+eh7nsRVth7d
7Zf/dUjVhXVZPrHPmlUm0OFpOlObGs7yBUX4btz7vFH7ucpmWNdvZGN9p/cu0LGr70TCgp
EobaPxG5Yy/bLt/npBAAABADcSbg3sIjdsiH1yY8mXnpdh1zewJ62CijZ4jPO6huZ86hyy
Mqfbnoj4WOKuDg9h5JZf2JmWXStF9NX4IZAMWmm7fCuq/k/vPJH/COQicSSHPOCVlJyCpv
qJC6POpoYStyy4THiipzjD56zZCBy5NoZCI+jtr8QYb/oLK1Mh5XTlSFs1zvPn8ui2YXko
+r1OXxMW/stOlpjdqWaPYPpBB+lnvRgsNWx1JVQpbY6tZoQE8cobrB/QbCo5dKvO7HL2c9
ZeFBoHs9trxwm4AQaK4uvX3xPgxXwQqRK/iy94NQ2Nqn+1wc8p7lpqXn8sBRylXNs7kcFU
8anqhKUWozQX4ksAAAEBAMGDmpi626H8GUS9UYEtWCpptBFn8svEFE4G8K7Cs0Q824l23L
SI/fRdWMMeipdTTlFX/9YQxmTOgKzbsAyYuojZmMkg4+pZ0Q75iEzwTn8nX1W4N4MLkkUT
n0eB6fZh21qOoORyIa1bj6sdCnMVHxuDWZBGC90jnar1J6PhqZeKzmxyn9UqExYuUpPj2T
Sk7do4lR2h8oDq0DAJXKcQrHcW3PqddvDRSFYTpImudr4TQzI6qWzpi8KudrdKriDwJLCA
BA+l9VBzxUba6/LGHxDN+2L/NpPCmcDerC2HIpMs/grlsjTbjUpP6vzwb5vElPwb/FkKSS
282UsIo9WGenkAAAEBAOlCFBlt6FeFu/ywvD5lNQ89C5bLlLauzXyx0XoVoXEjpS7v2DsA
uPdYSuQ7ftXMumT89bfs1MynohuQN9BX3VIQh7c60BP2NXlsGs/jphGWlz8++zAj55awG1
Jb0NycqaKMWZ8/oRGl4A/2YpjWaKl+wKqeWJD7H+iFBnR5NQ5jPqnuqRJvCwDTfksMdsxw
R1y/+geHAq8ZBnXlkqphbSyTRQqxSmYUwvVFycj5mI84k+T3xTD5jHHthW5CGx6o9nIDJS
KAjsydR2GAMAGhQRM1a3c+VabC8ZqjhUQl+lga1Od3rvvDio/pH1yrrWQteWAtiqZAficv
Q8qlPhFqwcEAAAAgY3Z0dF9kb2NrZXJAY3J5cHRvdmFsdHJhZGluZy5jb20BAgM=
-----END OPENSSH PRIVATE KEY-----

1
shared/id_rsa.pub Normal file
View File

@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwUrrUXnTE0ZEU0I80ZrYS0wWRARlY0w2jUPM9yeoVaAFSSLlLL8XWNTvfLdquH7BqxayboxtsnIHCZS/ebCKUNJ4M9hlUGNfCKsxmR/YBSFEyFiWuhent6aYjtdkmGD6+rc26vvLc4bWeezi48MgW5DMAeV9otN3/mIGKce97I7KmoJB6gekjaKaF7YWvuuB8Qdl/1QDaTjwsOTFn93uHhyZItFNF4PT9Yj59oXYvS/fozCcS2Saw41LRfMURpXDH/AFOWQu1EZbQFlF/BXnpCD0mTmJVzL2qS2RTU5xrr9M67l/8FmIGdzsdXBT4uQ40mbLLhgl1sRNLE0PGZfkNuJYhPi5Aq6P/2ymAkOAm2aBKPKc2aNcJnpYVF9BrWwWoR+JgWcald4xm+OcCn/sgf6/hkQGEf2tRucNkeNLjGE4fMa4a7PswfjtuWwPibzazfFXJqCjF/AyoyOBSBnh5ygqBr1P6o25TKqefufmcAlpC5fifBi2F1KsqjewCbR1JxFa+9dn9jtu7W5Z+ZYpVxJHDTvb0nt9wNjW/IPgYEnbm9xiE074XeypgoMRpSAM4e7abOL3LwPMjXKJa8toNYC0oNyJySTOqPxC5Q7HFBzT06iKOUAbHfAoaqZkw41GUnL+V14b974hS6SOfqHWaIJVJWk4UGYSKFQWdG9GOOQ== cvtt_docker@cryptovaltrading.com

6
shared/known_hosts Normal file
View File

@ -0,0 +1,6 @@
|1|yT6WmkWOPxXSQeNECVJIX7M3upw=|wQPqbw6mXxwfO83sWVH9IRapSe0= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNIOY8dNaWBcUu31F2OOuXUFGam2sx3Admn51SZR37MOMDEeKzpX+w0OnDGME9uOfuBToDW5F8ZoC+/p492uywZb4ZvUqVE8ic3uTaMro7zB9sykt4AO2rifM/0k7Fo1WoNWSaQptS3qyf7Umgl+qD5H8PxEM1OqKQl38BI5hfKUGx/WSYawgGIKkyO1eFwdjmSJ4RgQpN6/p5+1wJqbqbzkk0cEvZ06BbUbY+nUSUYY5WKa8ICBI0GKDdgRg9FXTV18r0YPT62HhShIyScbmg2h9hDKb7q9lLNLs2qg11jpSZCsIYTR2yms60uqXXq+isReLZ+W4z0t/8YVoID4Q82UlJ8R/LEqiJd8VcjZHaRVqtr7oVQjSurg1ChgCnOHHjvjZLVUg7QRMbCVT2/6PMAjjTyHUx8TKscYjz21ctPMn63pH/6lYzncQrt7ZT6PromVtnBs7p8dHaCyv18z9EBvGuE2rYOsfsCuR7FNlno7ej/gbe6CoaT2KwcNU8vpc=
|1|QgYdyhwM1uy97fUH/l9jXfRnYlE=|P575Q9H/V0Hoszk+G9As1GozQmE= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHlFA8BawdNbgYsFPPUJiG7xfuWI6dwNIFkuUwqco+TPXanMG4CvZ0TjRgb2fwdMXAGyoNAmwcr8eR5HU8wTg8g=
|1|vSzhghnKpUIGuOOgELHWGrbMxd8=|GY2Acsr6iqo4Y/dDS8V5glZ7tmA= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKz0QyrgFpp9NsPvZVwkjoua57SztFOMUnEzOLfW7kPr
|1|LJg/h/d7KSfFYzdtrrEFpcpnF/4=|xRkJEoqiptYGTTt5EuQo4f3zo6c= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKnC41A86Q2mVQWyRXHKzht6t8J1JJsRbYrQRpcMzh6VkVmOPDdAtl4P5iOH/GUaHr+HNhS7Y/QZS0yyjv8u1yOjb6ovfKvIuysj7f87wI1UOtQaUgX/PG5wupk/SgiUcfkpG+hg5wUACbsbMjp+HbE/OB7Nk5ZzUHNZcwQ55mRis8AzLW3IuFEboE+tJCxqTz8mr95ecqEWXdG4cJdDp7BMuCzsPCsOWgpcJiH+kgimok9n4qJPgoFSTVvHKBj0uJlm33rc8ns/Uu8knq6QfeSlzlN7uW0Xue6irW4Uaz69NbSUcFCeG83WNGO7tceYc7tmVM+9jjAtyhh9x1kGyk6thBMimRE5taa+5+bZP5tMctFsze6LVUGOothYW2MVgllJ+ovNFV/lin1y1591eAl340uSs4E4AJH6F/2Tta9OVGvNaXGMTYzKOLBYwo9sXJRHFuPjFxEs0Ya4pFUoCa8B53reNiOafrtwRU8/8G6zMgrF5z4FhCYBcIzoJdWCE=
|1|9oemW7STLSSF7uf+SBwD7JWe0K8=|YCblOLOrxHLzbHkMuZfiUP8nkTU= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPEpfGoofmKWz3/t3Svaahh5ieq4LJ6/n+y3M0rOK/RgndUCESaTZXbw701NOF8BPcJczTQTuQPO2AJwrwQYcK0=
|1|EC/isVVKdC1w/IthdEuhT29iU+w=|VClak/HfuZVXh4zlhhtx9euaTT0= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA2qvtcS1snKAdWKJefMCuKKgVlz0haOT0sElceM06qV