prepare sim for crypto (before added num_trades) - progress

release_version.txt

@@ -1 +1 @@
-1.9.1,16->23
+1.9.2,prepare_crypto_simdata

research/prepare_crypto_simdata.sh

@@ -1,41 +1,30 @@
 #!/bin/bash
 
+# SQLite DDL for simulation
+# =========================
+
+
+# -- md_quotes
+
+# -- md_1min_bars
+
+
 usage() {
-  echo "Usage: $0  -S <symbols> -d <YYYYMMDD Date> [-O <output dir (./) >]"
+  echo "Usage: $0  -d YYYMMDD Date> [-O <output dir (./) >]"
   exit 1
 }
 
 # --------------------- Settings
 SourceHost=cloud21.cvtt.vpn
-SourceRootDir=/opt/store/cvtt/md_archive/crypto/alpaca_md
+SourceRootDir=/opt/store/cvtt/md_archive/crypto
+DbSource=cloud21
 # --------------------- Settings
 
-
-is_business_day() {
-    dt=${1}
-    date=$(date -d "${dt}" +"%Y-%m-%d")
-
-    CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
-    URL="${CalendarURL}&start=${date}&end=${date}"
-    open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
-    if [ -n  "${open_time}" ]; then
-        return 0
-    else
-        return 1
-    fi
-
-}
-export -f is_business_day
-
-
-while getopts ":d:S:O:" opt; do
+while getopts ":d:O:" opt; do
   case ${opt} in
     d )
       Date=$OPTARG
       ;;
-    S )
-      SymList=$OPTARG
-      ;;
     O )
       OutputDir=$OPTARG
       ;;
@@ -50,10 +39,6 @@ while getopts ":d:S:O:" opt; do
   esac
 done
 
-if [ -z ${SymList}  ] ; then
-    echo "symbols are not specified"
-    usage
-fi
 if [ -z ${Date}  ] ; then
     echo "date is not specified"
     usage
@@ -63,84 +48,232 @@ if [ -z ${OutputDir}  ] ; then
 fi
 mkdir -p ${OutputDir}
 
-if ! is_business_day ${Date}; then
-   echo "${Date} is not business day"
-   usage
-fi
+# --- Binance
+Instruments=( PAIR-ADA-USDT )
+Instruments+=( PAIR-BCH-USDT )
+Instruments+=( PAIR-BTC-USDT )
+Instruments+=( PAIR-DOT-USDT )
+Instruments+=( PAIR-ETH-USDT )
+Instruments+=( PAIR-LTC-USDT )
+Instruments+=( PAIR-SOL-USDT )
+Instruments+=( PAIR-USDC-USDT )
+Instruments+=( PAIR-XRP-USDT )
 
-OLD_IFS=${IFS}
-IFS=","
-read -ra Symbols <<< "${SymList}"
-IFS=${OLD_IFS}
+# --- Coinbase
+Instruments+=( PAIR-ADA-USD )
+Instruments+=( PAIR-BCH-USD )
+Instruments+=( PAIR-BTC-USD )
+Instruments+=( PAIR-DOT-USD )
+Instruments+=( PAIR-ETH-USD )
+Instruments+=( PAIR-LTC-USD )
+Instruments+=( PAIR-SOL-USD )
+Instruments+=( PAIR-XRP-USD )
 
-echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}"
+echo "Date=${Date} Instruments=${Instruments[@]} OutputDir=${OutputDir}"
 echo Getting data from ${DataHost} ...
 
 year=$(date -d ${Date} +"%Y")
-for sym in ${Symbols[@]}; do
-    inst_id="STOCK-${sym}"
-    capital=${sym:0:1}
-    SourceDir="${SourceRootDir}/${year}/${capital}/${sym}"
-    SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz"
-    SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz"
+month=$(date -d ${Date} +"%m")
+SourceDir="${SourceRootDir}/${DbSource}/${year}/${month}"
+SourceFile="${SourceDir}/${Date}.mktdata.db.gz"
 
-    for src_file in ${SourceHbarFile} ${SourceQatFile}; do
 Cmd="rsync -ahv"
-        Cmd+=" ${SourceHost}:${src_file}"
+Cmd+=" ${SourceHost}:${SourceFile}"
 Cmd+=" $OutputDir/"
 echo ${Cmd}
 eval ${Cmd}
-    done
-done
 
 Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
 echo ${Cmd}
 eval ${Cmd}
 
-ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
+SourceDbFile="${OutputDir}/${Date}.mktdata.db"
+ResultDbFile="${OutputDir}/${Date}.crypto_sim_md.db"
+
 echo "Creating Result Database File ${ResultDbFile}"
-sym=${Symbols[0]}
-for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
-    Cmd="sqlite3"
-    Cmd+=" ${db_file} .schema"
-    Cmd+=" | sqlite3 ${ResultDbFile}"
-    echo ${Cmd}
-    eval ${Cmd}
-done
 
-set -f # not to expand *
-
-for sym in ${Symbols[@]}; do
-   src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
-   src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
-   for srcdb in ${src_hbar_db} ${src_qat_db} ; do
-
-       tables=$(sqlite3 ${srcdb} .tables)
-       for table in $(sqlite3 ${srcdb} .tables) ; do
-           echo "Loading from ${srcdb}.${table} ..."
+echo "Creating table md_trades ..."
 sqlite3 ${ResultDbFile} <<EOF
-ATTACH '${srcdb}' AS source_db;
+CREATE TABLE  IF NOT EXISTS md_trades (
+    tstamp text,
+    tstamp_ns integer,
+    exchange_id text,
+    instrument_id text,
+    exch text,
+    px real,
+    qty real,
+    trade_id text,
+    condition text,
+    tape text
+);
+CREATE UNIQUE INDEX  IF NOT EXISTS md_trades_uidx
+ON md_trades(tstamp_ns, exchange_id, instrument_id);
+EOF
+
+echo "Creating table md_quotes ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_quotes (
+    tstamp text,
+    tstamp_ns integer,
+    exchange_id text,
+    instrument_id text,
+    bid_exch text,
+    bid_px real,
+    bid_qty real,
+    ask_exch text,
+    ask_px real,
+    ask_qty real
+);
+CREATE UNIQUE INDEX  IF NOT EXISTS md_quotes_uidx
+ON md_quotes(tstamp_ns, exchange_id, instrument_id);
+EOF
+
+echo "Creating table md_1min_bars ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_1min_bars (
+	tstamp text,
+	tstamp_ns integer,
+	exchange_id text,
+	instrument_id text,
+	open real,
+	high real,
+	low real,
+	close real,
+	volume real,
+	vwap real,
+	num_trades integer
+);
+CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
+ON md_1min_bars(tstamp, exchange_id, instrument_id);
+EOF
+
+echo "Loading md_trades ..."
+sqlite3  ${ResultDbFile} <<EOF
+ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
-INSERT INTO ${table} SELECT * FROM source_db.${table};
+INSERT OR IGNORE INTO md_trades SELECT
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	time as tstamp_ns,
+	exchange_id,
+	instrument_id,
+	"" as exch,
+	price as px,
+	quantity as qty,
+	"" as trade_id,
+	taker_side as condition,
+	"" as tape
+from source_db.bnbspot_md_trades;
 COMMIT;
+
+BEGIN;
+INSERT OR IGNORE INTO md_trades
+SELECT
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	time as tstamp_ns,
+	exchange_id,
+	instrument_id,
+	"" as exch,
+	price as px,
+	quantity as qty,
+	"" as trade_id,
+	taker_side as condition,
+	"" as tape
+from source_db.coinbase_md_trades;
+COMMIT;
+
 DETACH source_db;
 EOF
-       done
-       Cmd="rm ${srcdb}"
+
+echo "Loading md_quotes ..."
+sqlite3  ${ResultDbFile} <<EOF
+ATTACH '${SourceDbFile}' AS source_db;
+BEGIN;
+INSERT OR IGNORE INTO md_quotes SELECT
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	time as tstamp_ns,
+	exchange_id,
+	instrument_id,
+	exchange_id as bid_exch,
+	bid_price as bid_px,
+	bid_quantity as bid_qty,
+	exchange_id as ask_exch,
+	ask_price as ask_px,
+	ask_quantity as ask_qty
+from bnbspot_md_booktops;
+COMMIT;
+
+BEGIN;
+INSERT OR IGNORE INTO md_quotes SELECT
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
+	time as tstamp_ns,
+	exchange_id,
+	instrument_id,
+	exchange_id as bid_exch,
+	bid_price as bid_px,
+	bid_quantity as bid_qty,
+	exchange_id as ask_exch,
+	ask_price as ask_px,
+	ask_quantity as ask_qty
+from coinbase_md_booktops;
+COMMIT;
+
+DETACH source_db;
+EOF
+
+### --- REPLACE 0 with num_trades ---
+
+echo "Loading md_1min_bars ..."
+sqlite3  ${ResultDbFile} <<EOF
+ATTACH '${SourceDbFile}' AS source_db;
+BEGIN;
+INSERT OR IGNORE INTO md_1min_bars SELECT
+	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tsatmp,
+	tstamp as tstamp_ns,
+	exchange_id,
+	instrument_id,
+	open,
+	high,
+	low,
+	close,
+	volume,
+	vwap,
+	0 as num_trades
+from bnbspot_ohlcv_1min;
+COMMIT;
+
+BEGIN;
+INSERT OR IGNORE INTO md_1min_bars SELECT
+	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tstamp,
+	tstamp as tstamp_ns,
+	exchange_id,
+	instrument_id,
+	open,
+	high,
+	low,
+	close,
+	volume,
+	vwap,
+	0 as num_trades
+from coinbase_ohlcv_1min;
+COMMIT;
+
+DETACH source_db;
+EOF
+
+Cmd="rm ${SourceDbFile}"
 echo ${Cmd}
 eval ${Cmd}
-   done
-done
+
 
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
 echo ${Cmd}
 eval ${Cmd}
 

research/prepare_eqt_simdata.sh

@@ -99,48 +99,141 @@ eval ${Cmd}
 
 ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
 echo "Creating Result Database File ${ResultDbFile}"
-sym=${Symbols[0]}
-for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
-    Cmd="sqlite3"
-    Cmd+=" ${db_file} .schema"
-    Cmd+=" | sqlite3 ${ResultDbFile}"
-    echo ${Cmd}
-    eval ${Cmd}
-done
 
-set -f # not to expand *
+echo "Creating Result Database File ${ResultDbFile}"
+
+echo "Creating table md_trades ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_trades (
+    tstamp text,
+    tstamp_ns integer,
+    exchange_id text,
+    instrument_id text,
+    exch text,
+    px real,
+    qty real,
+    trade_id text,
+    condition text,
+    tape text
+);
+CREATE UNIQUE INDEX  IF NOT EXISTS md_trades_uidx
+ON md_trades(tstamp_ns, exchange_id, instrument_id);
+EOF
+
+echo "Creating table md_quotes ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_quotes (
+    tstamp text,
+    tstamp_ns integer,
+    exchange_id text,
+    instrument_id text,
+    bid_exch text,
+    bid_px real,
+    bid_qty real,
+    ask_exch text,
+    ask_px real,
+    ask_qty real
+);
+CREATE UNIQUE INDEX  IF NOT EXISTS md_quotes_uidx
+ON md_quotes(tstamp_ns, exchange_id, instrument_id);
+EOF
+
+echo "Creating table md_1min_bars ..."
+sqlite3 ${ResultDbFile} <<EOF
+CREATE TABLE  IF NOT EXISTS md_1min_bars (
+	tstamp text,
+	tstamp_ns integer,
+	exchange_id text,
+	instrument_id text,
+	open real,
+	high real,
+	low real,
+	close real,
+	volume real,
+	vwap real,
+	num_trades integer
+);
+CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
+ON md_1min_bars(tstamp, exchange_id, instrument_id);
+EOF
+
+
+# set -f # not to expand *
 
 for sym in ${Symbols[@]}; do
    src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
    src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
    for srcdb in ${src_hbar_db} ${src_qat_db} ; do
 
-       tables=$(sqlite3 ${srcdb} .tables)
-       for table in $(sqlite3 ${srcdb} .tables) ; do
-           echo "Loading from ${srcdb}.${table} ..."
+       echo "Loading md_trades from ${srcdb} ..."
        sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${srcdb}' AS source_db;
 BEGIN;
-INSERT INTO ${table} SELECT * FROM source_db.${table};
+INSERT INTO md_trades SELECT
+	tstamp,
+	tstamp_ns,
+	exchange_id,
+	instrument_id,
+	exch,
+	px,
+	qty,
+	trade_id,
+	condition,
+	tape
+FROM source_db.md_trades;
 COMMIT;
+
+BEGIN;
+INSERT INTO md_quotes SELECT
+	tstamp,
+	tstamp_ns,
+	exchange_id,
+	instrument_id,
+	bid_exch,
+	bid_px,
+	bid_qty,
+	ask_exch,
+	ask_px,
+	ask_qty
+FROM source_db.md_quotes;
+COMMIT;
+
+BEGIN;
+INSERT INTO md_1min_bars SELECT
+	tstamp,
+	tstamp_ns,
+	exchange_id,
+	instrument_id,
+	open,
+	high,
+	low,
+	close,
+	volume,
+	vwap,
+	num_trades
+    FROM source_db.md_1min_bars;
+COMMIT;
+
 DETACH source_db;
 EOF
-       done
+
        Cmd="rm ${srcdb}"
        echo ${Cmd}
        eval ${Cmd}
    done
 done
 
+exit
+
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}