From 8eb52d718f747ef401084aa68f8c395986e1decc Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Wed, 17 Jul 2024 10:47:52 -0400 Subject: [PATCH] progress --- doc/crypto_md.txt | 21 ++++++++----- doc/equity_md.txt | 46 +++++++++++++++++++++++++++++ release_version.txt | 2 +- scripts/alpaca_hist_md/alpaca_md.sh | 3 +- scripts/legacy_eqt_md/process_md.sh | 33 +++++++++++++++++++++ 5 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 doc/equity_md.txt create mode 100644 scripts/legacy_eqt_md/process_md.sh diff --git a/doc/crypto_md.txt b/doc/crypto_md.txt index fab8844..85fc214 100644 --- a/doc/crypto_md.txt +++ b/doc/crypto_md.txt @@ -1,9 +1,12 @@ ------------------------------------- -C R Y P T O M A R K E T D A T A +C R Y P T O M A R K E T D A T A ------------------------------------- ============ Exchanges ============ + | Coinbase + | Binance Spot + | Binance Futures | v ============ @@ -16,19 +19,23 @@ C R Y P T O M A R K E T D A T A Daily Archive ================ | created by crontabs: - | cvttdata: - | /home/cvtt/prod/utils/archive_yesterday_md.sh cloud21 - | /home/cvtt/prod/utils/archive_yesterday_md.sh cvttdata + | cvttdata: + | (1) /home/cvtt/prod/utils/archive_yesterday_md.sh cloud21 + | (2) /home/cvtt/prod/utils/archive_yesterday_md.sh cvttdata | stored in: - | /home/cvtt/prod/archive/md_archive/crypto/cloud21 - | /home/cvtt/prod/archive/md_archive/crypto/cvttdata + | cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cloud21 (1) + | cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cvttdata (2) | v ========== Storage ========== | created by crontabs: - | cvttdata: + | cvttdata: + | TBD + | stored in: + | homestore:/works/cvtt/md_archive/crypto/ (cvttdata and cloud21) + | cloud21:/opt/store/cvtt/md_archive/crypto/ (cvttdata and cloud21) | v ========== diff --git a/doc/equity_md.txt b/doc/equity_md.txt new file mode 100644 index 0000000..a34939d --- /dev/null +++ b/doc/equity_md.txt @@ -0,0 +1,46 @@ +------------------------------------- +E Q U I T Y M A R K E T D A T A +------------------------------------- +===================== + Exchanges (Sources) +===================== + | Alpaca + | + | + | TBD ============ + | TBD Databases + | TBD ============ + | TBD | TimescaleDB: cloud21 + | TBD | TimescaleDB: cvttdata + v TBD v +================ + Daily Archive +================ + | created by crontab on cvttdata: + | /home/cvtt/prod/run/alpaca_md.sh + | stored in: + | cvttdata:/home/cvtt/prod/archive/md_archive/equities/alpaca_md (TBD) + | + v + + + + + + +========== + Storage +========== + | created by crontabs: + | cvttdata: + | + | stored in: + | homestore:/works/cvtt/md_archive/crypto/cvttdata + | homestore:/works/cvtt/md_archive/crypto/cloud21 + | backed up in: + | cloud21:/opt/store/cvtt/md_archive/crypto/cvttdata + | cloud21:/opt/store/cvtt/md_archive/crypto/cloud21 + v +========== + Usage +========== diff --git a/release_version.txt b/release_version.txt index 6201b5f..55485e1 100644 --- a/release_version.txt +++ b/release_version.txt @@ -1 +1 @@ -0.8.8 +0.8.9 diff --git a/scripts/alpaca_hist_md/alpaca_md.sh b/scripts/alpaca_hist_md/alpaca_md.sh index 8582a03..396b4db 100755 --- a/scripts/alpaca_hist_md/alpaca_md.sh +++ b/scripts/alpaca_hist_md/alpaca_md.sh @@ -48,7 +48,8 @@ export PYTHONPATH=/home/cvtt/prod export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3 export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py -export OutputDir=/home/cvtt/prod/archive/equities/alpaca_md # Local + +export OutputDir=/home/cvtt/prod/archive/md_archive/equities/alpaca_md # Local export LogDir=/home/cvtt/prod/logs/alpaca_md # ----- T E M P diff --git a/scripts/legacy_eqt_md/process_md.sh b/scripts/legacy_eqt_md/process_md.sh new file mode 100644 index 0000000..8be62ef --- /dev/null +++ b/scripts/legacy_eqt_md/process_md.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Collect all "A" files +letter=${1} + +if [ "${letter}" == "" ]; then + echo "Usage $0 " + exit 1 +fi + +# copy text data to local location +rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/ + +# create db files +PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db + +if [ "$?" != "0" ] ; then + exit +fi + +# Move all files to host drive +# a) create file list +(cd /home/cvtt/tmp/db/ && find . -name '*db' -print | grep "/${letter}/") > /home/cvtt/tmp/tran_db/${letter}_files + +echo ${letter} is done +exit + + +# b) rsync files to host drive +rsync -ahv --remove-source-files --files-from=/home/cvtt/tmp/tran_db/${letter}_files /home/cvtt/tmp/db/ cvtt@my-vm-host:/localdisk/cvtt/eqt_hist_md/db/ + +# Clean directories +( cd /home/cvtt/tmp/db && (for d in $(find . -name $letter -type d -print); do echo $d ; done) | grep -v /$letter/$letter | xargs rm -rf) && rm -rf /home/cvtt/tmp/txt/${letter}