archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prepare sim for crypto (before added num_trades) - progress

Browse Source
This commit is contained in:
Oleg Sheynin 2024-12-06 19:50:02 -05:00
parent 11ab9dd2c2
commit d54a89b360
2 changed files with 118 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
research/prepare_crypto_simdata.sh
View File

@ -204,43 +204,7 @@ COMMIT;
BEGIN;
INSERT OR IGNORE INTO md_quotes SELECT
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
time as tstamp_ns,
exchange_id,
instrument_id,
exchange_id as bid_exch,
bid_price as bid_px,
bid_quantity as bid_qty,
exchange_id as ask_exch,
ask_price as ask_px,
ask_quantity as ask_qty
from coinbase_md_booktops;
COMMIT;
DETACH source_db;
EOF
echo "Loading md_quotes ..."
sqlite3 ${ResultDbFile} <<EOF
ATTACH '${SourceDbFile}' AS source_db;
BEGIN;
INSERT OR IGNORE INTO md_quotes SELECT
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
time as tstamp_ns,
exchange_id,
instrument_id,
exchange_id as bid_exch,
bid_price as bid_px,
bid_quantity as bid_qty,
exchange_id as ask_exch,
ask_price as ask_px,
ask_quantity as ask_qty
from bnbspot_md_booktops;
COMMIT;
BEGIN;
INSERT OR IGNORE INTO md_quotes SELECT
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
time as tstamp_ns,
exchange_id,
instrument_id,
@ -263,7 +227,7 @@ sqlite3 ${ResultDbFile} <<EOF
ATTACH '${SourceDbFile}' AS source_db;
BEGIN;
INSERT OR IGNORE INTO md_1min_bars SELECT
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstmp,
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tsatmp,
tstamp as tstamp_ns,
exchange_id,
instrument_id,
@ -279,7 +243,7 @@ COMMIT;
BEGIN;
INSERT OR IGNORE INTO md_1min_bars SELECT
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstmp,
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstamp,
tstamp as tstamp_ns,
exchange_id,
instrument_id,
@ -296,6 +260,11 @@ COMMIT;
DETACH source_db;
EOF
Cmd="rm ${SourceDbFile}"
echo ${Cmd}
eval ${Cmd}
Cmd="gzip ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}

127
research/prepare_eqt_simdata.sh
View File

@ -99,48 +99,141 @@ eval ${Cmd}
ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
echo "Creating Result Database File ${ResultDbFile}"
sym=${Symbols[0]}
for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
Cmd="sqlite3"
Cmd+=" ${db_file} .schema"
Cmd+=" | sqlite3 ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}
done
set -f # not to expand *
echo "Creating Result Database File ${ResultDbFile}"
echo "Creating table md_trades ..."
sqlite3 ${ResultDbFile} <<EOF
CREATE TABLE IF NOT EXISTS md_trades (
tstamp text,
tstamp_ns integer,
exchange_id text,
instrument_id text,
exch text,
px real,
qty real,
trade_id text,
condition text,
tape text
);
CREATE UNIQUE INDEX IF NOT EXISTS md_trades_uidx
ON md_trades(tstamp_ns, exchange_id, instrument_id);
EOF
echo "Creating table md_quotes ..."
sqlite3 ${ResultDbFile} <<EOF
CREATE TABLE IF NOT EXISTS md_quotes (
tstamp text,
tstamp_ns integer,
exchange_id text,
instrument_id text,
bid_exch text,
bid_px real,
bid_qty real,
ask_exch text,
ask_px real,
ask_qty real
);
CREATE UNIQUE INDEX IF NOT EXISTS md_quotes_uidx
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
EOF
echo "Creating table md_1min_bars ..."
sqlite3 ${ResultDbFile} <<EOF
CREATE TABLE IF NOT EXISTS md_1min_bars (
tstamp text,
tstamp_ns integer,
exchange_id text,
instrument_id text,
open real,
high real,
low real,
close real,
volume real,
vwap real,
num_trades integer
);
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
ON md_1min_bars(tstamp, exchange_id, instrument_id);
EOF
# set -f # not to expand *
for sym in ${Symbols[@]}; do
src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
for srcdb in ${src_hbar_db} ${src_qat_db} ; do
tables=$(sqlite3 ${srcdb} .tables)
for table in $(sqlite3 ${srcdb} .tables) ; do
echo "Loading from ${srcdb}.${table} ..."
sqlite3 ${ResultDbFile} <<EOF
echo "Loading md_trades from ${srcdb} ..."
sqlite3 ${ResultDbFile} <<EOF
ATTACH '${srcdb}' AS source_db;
BEGIN;
INSERT INTO ${table} SELECT * FROM source_db.${table};
INSERT INTO md_trades SELECT
tstamp,
tstamp_ns,
exchange_id,
instrument_id,
exch,
px,
qty,
trade_id,
condition,
tape
FROM source_db.md_trades;
COMMIT;
BEGIN;
INSERT INTO md_quotes SELECT
tstamp,
tstamp_ns,
exchange_id,
instrument_id,
bid_exch,
bid_px,
bid_qty,
ask_exch,
ask_px,
ask_qty
FROM source_db.md_quotes;
COMMIT;
BEGIN;
INSERT INTO md_1min_bars SELECT
tstamp,
tstamp_ns,
exchange_id,
instrument_id,
open,
high,
low,
close,
volume,
vwap,
num_trades
FROM source_db.md_1min_bars;
COMMIT;
DETACH source_db;
EOF
done
Cmd="rm ${srcdb}"
echo ${Cmd}
eval ${Cmd}
done
done
exit
Cmd="gzip ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}
Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
echo ${Cmd}
eval ${Cmd}
Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
echo ${Cmd}
eval ${Cmd}