archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Oleg Sheynin 2024-07-17 10:47:52 -04:00
parent a0cf2531e6
commit ba21592700
5 changed files with 96 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
doc/crypto_md.txt
View File

@ -1,9 +1,12 @@
-------------------------------------
C R Y P T O M A R K E T D A T A
C R Y P T O M A R K E T D A T A
-------------------------------------
============
Exchanges
============
| Coinbase
| Binance Spot
| Binance Futures
|
v
============
@ -16,19 +19,23 @@ C R Y P T O M A R K E T D A T A
Daily Archive
================
| created by crontabs:
| cvttdata:
| /home/cvtt/prod/utils/archive_yesterday_md.sh cloud21
| /home/cvtt/prod/utils/archive_yesterday_md.sh cvttdata
| cvttdata:
| (1) /home/cvtt/prod/utils/archive_yesterday_md.sh cloud21
| (2) /home/cvtt/prod/utils/archive_yesterday_md.sh cvttdata
| stored in:
| /home/cvtt/prod/archive/md_archive/crypto/cloud21
| /home/cvtt/prod/archive/md_archive/crypto/cvttdata
| cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cloud21 (1)
| cvttdata:/home/cvtt/prod/archive/md_archive/crypto/cvttdata (2)
|
v
==========
Storage
==========
| created by crontabs:
| cvttdata:
| cvttdata:
| TBD
| stored in:
| homestore:/works/cvtt/md_archive/crypto/ (cvttdata and cloud21)
| cloud21:/opt/store/cvtt/md_archive/crypto/ (cvttdata and cloud21)
|
v
==========

46
doc/equity_md.txt Normal file
View File

@ -0,0 +1,46 @@
-------------------------------------
E Q U I T Y M A R K E T D A T A
-------------------------------------
=====================
Exchanges (Sources)
=====================
| Alpaca
|
|
| TBD ============
| TBD Databases
| TBD ============
| TBD | TimescaleDB: cloud21
| TBD | TimescaleDB: cvttdata
v TBD v
================
Daily Archive
================
| created by crontab on cvttdata:
| /home/cvtt/prod/run/alpaca_md.sh
| stored in:
| cvttdata:/home/cvtt/prod/archive/md_archive/equities/alpaca_md (TBD)
|
v
==========
Storage
==========
| created by crontabs:
| cvttdata:
|
| stored in:
| homestore:/works/cvtt/md_archive/crypto/cvttdata
| homestore:/works/cvtt/md_archive/crypto/cloud21
| backed up in:
| cloud21:/opt/store/cvtt/md_archive/crypto/cvttdata
| cloud21:/opt/store/cvtt/md_archive/crypto/cloud21
v
==========
Usage
==========

2
release_version.txt
View File

@ -1 +1 @@
0.8.8
0.8.9

3
scripts/alpaca_hist_md/alpaca_md.sh
View File

@ -48,7 +48,8 @@ export PYTHONPATH=/home/cvtt/prod
export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
export OutputDir=/home/cvtt/prod/archive/equities/alpaca_md # Local
export OutputDir=/home/cvtt/prod/archive/md_archive/equities/alpaca_md # Local
export LogDir=/home/cvtt/prod/logs/alpaca_md
# ----- T E M P

33
scripts/legacy_eqt_md/process_md.sh Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash
# Collect all "A" files
letter=${1}
if [ "${letter}" == "" ]; then
echo "Usage $0 <letter>"
exit 1
fi
# copy text data to local location
rsync -ahv /home/cvtt/host_drive/eqt_hist_md/txt/${letter} /home/cvtt/tmp/txt/
# create db files
PYTHONPATH=/home/cvtt/prod /home/cvtt/.pyenv/python3.10-venv/bin/python3 /home/cvtt/prod/cvttpy/trading/mkt_data/ad_hoc/eqt_md_to_db.py /home/cvtt/tmp/txt/${letter} /home/cvtt/tmp/db
if [ "$?" != "0" ] ; then
exit
fi
# Move all files to host drive
# a) create file list
(cd /home/cvtt/tmp/db/ && find . -name '*db' -print | grep "/${letter}/") > /home/cvtt/tmp/tran_db/${letter}_files
echo ${letter} is done
exit
# b) rsync files to host drive
rsync -ahv --remove-source-files --files-from=/home/cvtt/tmp/tran_db/${letter}_files /home/cvtt/tmp/db/ cvtt@my-vm-host:/localdisk/cvtt/eqt_hist_md/db/
# Clean directories
( cd /home/cvtt/tmp/db && (for d in $(find . -name $letter -type d -print); do echo $d ; done) | grep -v /$letter/$letter | xargs rm -rf) && rm -rf /home/cvtt/tmp/txt/${letter}