From 625670c69e90e7754ca853b2e0e50c63d1b41e27 Mon Sep 17 00:00:00 2001
From: Oleg Sheynin <osheynin@gmail.com>
Date: Fri, 6 Dec 2024 19:50:02 -0500
Subject: [PATCH] prepare sim for crypto (before added num_trades) - progress

---
 research/prepare_crypto_simdata.sh |  47 ++---------
 research/prepare_eqt_simdata.sh    | 127 +++++++++++++++++++++++++----
 2 files changed, 118 insertions(+), 56 deletions(-)

diff --git a/research/prepare_crypto_simdata.sh b/research/prepare_crypto_simdata.sh
index b839a4d..72b3c67 100755
--- a/research/prepare_crypto_simdata.sh
+++ b/research/prepare_crypto_simdata.sh
@@ -204,43 +204,7 @@ COMMIT;
 
 BEGIN;
 INSERT OR IGNORE INTO md_quotes SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
-	time as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	exchange_id as bid_exch,
-	bid_price as bid_px,
-	bid_quantity as bid_qty,
-	exchange_id as ask_exch,
-	ask_price as ask_px,
-	ask_quantity as ask_qty
-from coinbase_md_booktops;
-COMMIT;
-
-DETACH source_db;
-EOF
-
-echo "Loading md_quotes ..."
-sqlite3  ${ResultDbFile} <<EOF
-ATTACH '${SourceDbFile}' AS source_db;
-BEGIN;
-INSERT OR IGNORE INTO md_quotes SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
-	time as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	exchange_id as bid_exch,
-	bid_price as bid_px,
-	bid_quantity as bid_qty,
-	exchange_id as ask_exch,
-	ask_price as ask_px,
-	ask_quantity as ask_qty
-from bnbspot_md_booktops;
-COMMIT;
-
-BEGIN;
-INSERT OR IGNORE INTO md_quotes SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -263,7 +227,7 @@ sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
 INSERT OR IGNORE INTO md_1min_bars SELECT
-	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tstmp,
+	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tsatmp,
 	tstamp as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -279,7 +243,7 @@ COMMIT;
 
 BEGIN;
 INSERT OR IGNORE INTO md_1min_bars SELECT
-	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tstmp,
+	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tstamp,
 	tstamp as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -296,6 +260,11 @@ COMMIT;
 DETACH source_db;
 EOF
 
+Cmd="rm ${SourceDbFile}"
+echo ${Cmd}
+eval ${Cmd}
+
+
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
diff --git a/research/prepare_eqt_simdata.sh b/research/prepare_eqt_simdata.sh
index dd4a8fa..c5e5009 100755
--- a/research/prepare_eqt_simdata.sh
+++ b/research/prepare_eqt_simdata.sh
@@ -99,48 +99,141 @@ eval ${Cmd}
 
 ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
 echo "Creating Result Database File ${ResultDbFile}"
-sym=${Symbols[0]}
-for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
-    Cmd="sqlite3"
-    Cmd+=" ${db_file} .schema"
-    Cmd+=" | sqlite3 ${ResultDbFile}"
-    echo ${Cmd}
-    eval ${Cmd}
-done
 
-set -f # not to expand *
+echo "Creating Result Database File ${ResultDbFile}"
+
+echo "Creating table md_trades ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_trades (
+    tstamp text,
+    tstamp_ns integer,
+    exchange_id text,
+    instrument_id text,
+    exch text,
+    px real,
+    qty real,
+    trade_id text,
+    condition text,
+    tape text
+);
+CREATE UNIQUE INDEX  IF NOT EXISTS md_trades_uidx
+ON md_trades(tstamp_ns, exchange_id, instrument_id);
+EOF
+
+echo "Creating table md_quotes ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_quotes (
+    tstamp text,
+    tstamp_ns integer,
+    exchange_id text,
+    instrument_id text,
+    bid_exch text,
+    bid_px real,
+    bid_qty real,
+    ask_exch text,
+    ask_px real,
+    ask_qty real
+);
+CREATE UNIQUE INDEX  IF NOT EXISTS md_quotes_uidx
+ON md_quotes(tstamp_ns, exchange_id, instrument_id);
+EOF
+
+echo "Creating table md_1min_bars ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_1min_bars (
+	tstamp text,
+	tstamp_ns integer,
+	exchange_id text,
+	instrument_id text,
+	open real,
+	high real,
+	low real,
+	close real,
+	volume real,
+	vwap real,
+	num_trades integer
+);
+CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
+ON md_1min_bars(tstamp, exchange_id, instrument_id);
+EOF
+
+
+# set -f # not to expand *
 
 for sym in ${Symbols[@]}; do
    src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
    src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
    for srcdb in ${src_hbar_db} ${src_qat_db} ; do
 
-       tables=$(sqlite3 ${srcdb} .tables)
-       for table in $(sqlite3 ${srcdb} .tables) ; do
-           echo "Loading from ${srcdb}.${table} ..."
-           sqlite3  ${ResultDbFile} <<EOF
+       echo "Loading md_trades from ${srcdb} ..."
+       sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${srcdb}' AS source_db;
 BEGIN;
-INSERT INTO ${table} SELECT * FROM source_db.${table};
+INSERT INTO md_trades SELECT
+	tstamp,
+	tstamp_ns,
+	exchange_id,
+	instrument_id,
+	exch,
+	px,
+	qty,
+	trade_id,
+	condition,
+	tape
+FROM source_db.md_trades;
 COMMIT;
+
+BEGIN;
+INSERT INTO md_quotes SELECT
+	tstamp,
+	tstamp_ns,
+	exchange_id,
+	instrument_id,
+	bid_exch,
+	bid_px,
+	bid_qty,
+	ask_exch,
+	ask_px,
+	ask_qty
+FROM source_db.md_quotes;
+COMMIT;
+
+BEGIN;
+INSERT INTO md_1min_bars SELECT
+	tstamp,
+	tstamp_ns,
+	exchange_id,
+	instrument_id,
+	open,
+	high,
+	low,
+	close,
+	volume,
+	vwap,
+	num_trades
+    FROM source_db.md_1min_bars;
+COMMIT;
+
 DETACH source_db;
 EOF
-       done
+
        Cmd="rm ${srcdb}"
        echo ${Cmd}
        eval ${Cmd}
    done
 done
 
+exit
+
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}