moved daily_sim to cvtt_apps
This commit is contained in:
+278
@@ -0,0 +1,278 @@
|
||||
#!/bin/bash
|
||||
|
||||
# --------------------- Settings
|
||||
# SOURCE_HOST=cloud21.cvtt.vpn
|
||||
# SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto
|
||||
# DB_SOURCES=cloud28,cloud29
|
||||
# OUTPUT_DIR=/tmp
|
||||
# DATE=20250516
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
|
||||
# --------------------- Settings
|
||||
|
||||
if [ -z ${DATE} ] ; then
|
||||
DATE=$(date -d 'yesterday' +'%Y%m%d')
|
||||
fi
|
||||
|
||||
if [ -z ${OUTPUT_DIR} ] ; then
|
||||
OUTPUT_DIR=.
|
||||
fi
|
||||
|
||||
echo "DATE=${DATE} SOURCE_HOST=${SOURCE_HOST}"
|
||||
|
||||
mkdir -p ${OUTPUT_DIR}
|
||||
|
||||
year=$(date -d ${DATE} +"%Y")
|
||||
month=$(date -d ${DATE} +"%m")
|
||||
|
||||
if [ -z "${DB_SOURCES}" ]; then
|
||||
echo "DB_SOURCES is empty"
|
||||
exit
|
||||
fi
|
||||
|
||||
IFS=',' read -r -a db_source_hosts <<< "${DB_SOURCES}"
|
||||
|
||||
SourceFile="${DATE}.mktdata.db.gz"
|
||||
SelectedSourceHost=""
|
||||
SelectedSourceFilePath=""
|
||||
SelectedSourceSize=0
|
||||
|
||||
for db_source_host in "${db_source_hosts[@]}"; do
|
||||
SourceDir="${SOURCE_ROOT_DIR}/${db_source_host}/${year}/${month}"
|
||||
CandidatePath="${SourceDir}/${SourceFile}"
|
||||
remote_stat_cmd="if [ -f '${CandidatePath}' ]; then stat -c %s '${CandidatePath}'; else exit 1; fi"
|
||||
CandidateSize=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SOURCE_HOST} "${remote_stat_cmd}" 2>/dev/null)
|
||||
if [ $? -eq 0 ] && [ -n "${CandidateSize}" ]; then
|
||||
echo "Found ${SOURCE_HOST}:${CandidatePath} (${CandidateSize} bytes)"
|
||||
if [ -z "${SelectedSourceHost}" ] || [ "${CandidateSize}" -gt "${SelectedSourceSize}" ]; then
|
||||
SelectedSourceHost=${db_source_host}
|
||||
SelectedSourceFilePath=${CandidatePath}
|
||||
SelectedSourceSize=${CandidateSize}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
if [ -z "${SelectedSourceHost}" ]; then
|
||||
echo "File ${SourceFile} NOT FOUND on any DB_SOURCES host"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Using source ${SelectedSourceHost} with ${SelectedSourceFilePath} (${SelectedSourceSize} bytes)"
|
||||
|
||||
Cmd="/usr/bin/rsync -ahv"
|
||||
Cmd+=" --mkpath"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${SOURCE_HOST}:${SelectedSourceFilePath}"
|
||||
Cmd+=" $OUTPUT_DIR/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
if [ ! -f ${OUTPUT_DIR}/${SourceFile} ] ; then
|
||||
echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND"
|
||||
exit
|
||||
fi
|
||||
|
||||
Cmd="(cd ${OUTPUT_DIR} && gunzip -f *.db.gz)"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
SourceDbFile="${OUTPUT_DIR}/${DATE}.mktdata.db"
|
||||
ResultDbFile="${OUTPUT_DIR}/${DATE}.crypto_sim_md.db"
|
||||
|
||||
echo "SourceDbFile=${SourceDbFile}"
|
||||
echo "Creating Result Database File ${ResultDbFile}"
|
||||
cleanup() {
|
||||
rm ${SourceDbFile}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "Creating table md_trades ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_trades (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
exch text,
|
||||
px real,
|
||||
qty real,
|
||||
trade_id text,
|
||||
condition text,
|
||||
tape text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_trades_uidx
|
||||
ON md_trades(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_quotes ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_quotes (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
bid_exch text,
|
||||
bid_px real,
|
||||
bid_qty real,
|
||||
ask_exch text,
|
||||
ask_px real,
|
||||
ask_qty real
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_quotes_uidx
|
||||
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_1min_bars ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_1min_bars (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
open real,
|
||||
high real,
|
||||
low real,
|
||||
close real,
|
||||
volume real,
|
||||
vwap real,
|
||||
num_trades integer
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
|
||||
ON md_1min_bars(tstamp, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Loading md_trades ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${SourceDbFile}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_trades SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
"" as exch,
|
||||
price as px,
|
||||
quantity as qty,
|
||||
"" as trade_id,
|
||||
taker_side as condition,
|
||||
"" as tape
|
||||
from source_db.bnbspot_md_trades;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_trades
|
||||
SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
"" as exch,
|
||||
price as px,
|
||||
quantity as qty,
|
||||
"" as trade_id,
|
||||
taker_side as condition,
|
||||
"" as tape
|
||||
from source_db.coinbase_md_trades;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
echo "Loading md_quotes ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${SourceDbFile}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_quotes SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
exchange_id as bid_exch,
|
||||
bid_price as bid_px,
|
||||
bid_quantity as bid_qty,
|
||||
exchange_id as ask_exch,
|
||||
ask_price as ask_px,
|
||||
ask_quantity as ask_qty
|
||||
from bnbspot_md_booktops;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_quotes SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
exchange_id as bid_exch,
|
||||
bid_price as bid_px,
|
||||
bid_quantity as bid_qty,
|
||||
exchange_id as ask_exch,
|
||||
ask_price as ask_px,
|
||||
ask_quantity as ask_qty
|
||||
from coinbase_md_booktops;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
### --- REPLACE 0 with num_trades ---
|
||||
|
||||
echo "Loading md_1min_bars ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${SourceDbFile}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tsatmp,
|
||||
tstamp as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
volume,
|
||||
vwap,
|
||||
0 as num_trades
|
||||
from bnbspot_ohlcv_1min;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstamp,
|
||||
tstamp as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
volume,
|
||||
vwap,
|
||||
0 as num_trades
|
||||
from coinbase_ohlcv_1min;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
Cmd="gzip ${ResultDbFile}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
for tgt in ${RSYNC_TARGETS} ; do
|
||||
tgt="${tgt}/${year}/${month}/"
|
||||
Cmd="/usr/bin/rsync -ahv"
|
||||
Cmd+=" --mkpath"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${ResultDbFile}.gz ${tgt}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
|
||||
echo Done $0 ${*}
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
#!/bin/bash
|
||||
|
||||
# --------------------- Settings
|
||||
# SOURCE_HOST=cloud21.cvtt.vpn
|
||||
# SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
|
||||
# SYMBOL_LIST=CAN #,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
|
||||
# OUTPUT_DIR=/tmp
|
||||
# DATE=20250514
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test/alpaca_md/sim/"
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/
|
||||
#cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
|
||||
# --------------------- Settings
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
date=$(date -d "${dt}" +"%Y-%m-%d")
|
||||
|
||||
CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
|
||||
URL="${CalendarURL}&start=${date}&end=${date}"
|
||||
echo "URL=${URL}"
|
||||
# curl $URL
|
||||
open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
|
||||
echo "open_time=${open_time}"
|
||||
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
if [ -z ${SYMBOL_LIST} ] ; then
|
||||
echo "symbols are not specified"
|
||||
exit
|
||||
fi
|
||||
if [ -z ${DATE} ] ; then
|
||||
DATE=$(date -d 'yesterday' +'%Y%m%d')
|
||||
fi
|
||||
|
||||
mkdir -p ${OUTPUT_DIR}
|
||||
|
||||
if ! is_business_day ${DATE}; then
|
||||
echo "${DATE} is not business day"
|
||||
exit
|
||||
fi
|
||||
|
||||
OLD_IFS=${IFS}
|
||||
IFS=","
|
||||
read -ra Symbols <<< "${SYMBOL_LIST}"
|
||||
IFS=${OLD_IFS}
|
||||
|
||||
echo "DATE=${DATE} Symbols=${Symbols[@]} OUTPUT_DIR=${OUTPUT_DIR}"
|
||||
echo Getting data from ${DataHost} ...
|
||||
|
||||
year=$(date -d ${DATE} +"%Y")
|
||||
month=$(date -d ${DATE} +"%m")
|
||||
|
||||
for sym in ${Symbols[@]}; do
|
||||
inst_id="STOCK-${sym}"
|
||||
capital=${sym:0:1}
|
||||
SourceDir="${SOURCE_ROOT_DIR}/${year}/${capital}/${sym}"
|
||||
SourceHbarFile="${SourceDir}/${DATE}.${sym}.alpaca_1m_bars.db.gz"
|
||||
SourceQatFile="${SourceDir}/${DATE}.${sym}.alpaca_qat.db.gz"
|
||||
|
||||
for src_file in ${SourceHbarFile} ${SourceQatFile}; do
|
||||
Cmd="rsync -ahv"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${SOURCE_HOST}:${src_file}"
|
||||
Cmd+=" $OUTPUT_DIR/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
done
|
||||
|
||||
Cmd="(cd ${OUTPUT_DIR} && gunzip *.db.gz)"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
ResultDbFile="${OUTPUT_DIR}/${DATE}.alpaca_sim_md.db"
|
||||
echo "Creating Result Database File ${ResultDbFile}"
|
||||
|
||||
echo "Creating table md_trades ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_trades (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
exch text,
|
||||
px real,
|
||||
qty real,
|
||||
trade_id text,
|
||||
condition text,
|
||||
tape text
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_trades_uidx
|
||||
ON md_trades(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_quotes ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_quotes (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
bid_exch text,
|
||||
bid_px real,
|
||||
bid_qty real,
|
||||
ask_exch text,
|
||||
ask_px real,
|
||||
ask_qty real
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_quotes_uidx
|
||||
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_1min_bars ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
CREATE TABLE IF NOT EXISTS md_1min_bars (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
open real,
|
||||
high real,
|
||||
low real,
|
||||
close real,
|
||||
volume real,
|
||||
vwap real,
|
||||
num_trades integer
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
|
||||
ON md_1min_bars(tstamp, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
|
||||
# set -f # not to expand *
|
||||
|
||||
for sym in ${Symbols[@]}; do
|
||||
src_hbar_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_1m_bars.db
|
||||
src_qat_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_qat.db
|
||||
|
||||
echo "Loading md_trades and md_quotes from ${src_qat_db} ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${src_qat_db}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_trades SELECT
|
||||
tstamp,
|
||||
tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
exch,
|
||||
px,
|
||||
qty,
|
||||
trade_id,
|
||||
condition,
|
||||
tape
|
||||
FROM source_db.md_trades;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_quotes SELECT
|
||||
tstamp,
|
||||
tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
bid_exch,
|
||||
bid_px,
|
||||
bid_qty,
|
||||
ask_exch,
|
||||
ask_px,
|
||||
ask_qty
|
||||
FROM source_db.md_quotes;
|
||||
COMMIT;
|
||||
EOF
|
||||
|
||||
echo "Loading md_1min_bars from ${src_hbar_db} ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${src_hbar_db}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||
tstamp,
|
||||
tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
volume,
|
||||
vwap,
|
||||
num_trades
|
||||
FROM source_db.md_1min_bars;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
Cmd="rm ${src_hbar_db} ${src_qat_db}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
|
||||
Cmd="gzip ${ResultDbFile}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
for tgt in ${RSYNC_TARGETS} ; do
|
||||
Cmd="/usr/bin/rsync -ahv"
|
||||
Cmd+=" --mkpath"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${ResultDbFile}.gz ${tgt}/${year}/${month}/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
|
||||
echo Done $0 ${*}
|
||||
Reference in New Issue
Block a user