release_version.txt

@@ -1 +1 @@
-1.9.2,prepare_crypto_simdata
+1.9.1,16->23

research/prepare_crypto_simdata.sh

@@ -1,30 +1,41 @@
 #!/bin/bash
 
-# SQLite DDL for simulation
-# =========================
-
-
-# -- md_quotes
-
-# -- md_1min_bars
-
-
 usage() {
-  echo "Usage: $0  -d YYYMMDD Date> [-O <output dir (./) >]"
+  echo "Usage: $0  -S <symbols> -d <YYYYMMDD Date> [-O <output dir (./) >]"
   exit 1
 }
 
 # --------------------- Settings
 SourceHost=cloud21.cvtt.vpn
-SourceRootDir=/opt/store/cvtt/md_archive/crypto
+SourceRootDir=/opt/store/cvtt/md_archive/crypto/alpaca_md
-DbSource=cloud21
 # --------------------- Settings
 
-while getopts ":d:O:" opt; do
+
+is_business_day() {
+    dt=${1}
+    date=$(date -d "${dt}" +"%Y-%m-%d")
+
+    CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
+    URL="${CalendarURL}&start=${date}&end=${date}"
+    open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
+    if [ -n  "${open_time}" ]; then
+        return 0
+    else
+        return 1
+    fi
+
+}
+export -f is_business_day
+
+
+while getopts ":d:S:O:" opt; do
   case ${opt} in
     d )
       Date=$OPTARG
       ;;
+    S )
+      SymList=$OPTARG
+      ;;
     O )
       OutputDir=$OPTARG
       ;;
@@ -39,6 +50,10 @@ while getopts ":d:O:" opt; do
   esac
 done
 
+if [ -z ${SymList}  ] ; then
+    echo "symbols are not specified"
+    usage
+fi
 if [ -z ${Date}  ] ; then
     echo "date is not specified"
     usage
@@ -48,232 +63,84 @@ if [ -z ${OutputDir}  ] ; then
 fi
 mkdir -p ${OutputDir}
 
-# --- Binance
+if ! is_business_day ${Date}; then
-Instruments=( PAIR-ADA-USDT )
+   echo "${Date} is not business day"
-Instruments+=( PAIR-BCH-USDT )
+   usage
-Instruments+=( PAIR-BTC-USDT )
+fi
-Instruments+=( PAIR-DOT-USDT )
-Instruments+=( PAIR-ETH-USDT )
-Instruments+=( PAIR-LTC-USDT )
-Instruments+=( PAIR-SOL-USDT )
-Instruments+=( PAIR-USDC-USDT )
-Instruments+=( PAIR-XRP-USDT )
 
-# --- Coinbase
+OLD_IFS=${IFS}
-Instruments+=( PAIR-ADA-USD )
+IFS=","
-Instruments+=( PAIR-BCH-USD )
+read -ra Symbols <<< "${SymList}"
-Instruments+=( PAIR-BTC-USD )
+IFS=${OLD_IFS}
-Instruments+=( PAIR-DOT-USD )
-Instruments+=( PAIR-ETH-USD )
-Instruments+=( PAIR-LTC-USD )
-Instruments+=( PAIR-SOL-USD )
-Instruments+=( PAIR-XRP-USD )
 
-echo "Date=${Date} Instruments=${Instruments[@]} OutputDir=${OutputDir}"
+echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}"
 echo Getting data from ${DataHost} ...
 
 year=$(date -d ${Date} +"%Y")
-month=$(date -d ${Date} +"%m")
+for sym in ${Symbols[@]}; do
-SourceDir="${SourceRootDir}/${DbSource}/${year}/${month}"
+    inst_id="STOCK-${sym}"
-SourceFile="${SourceDir}/${Date}.mktdata.db.gz"
+    capital=${sym:0:1}
+    SourceDir="${SourceRootDir}/${year}/${capital}/${sym}"
+    SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz"
+    SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz"
 
-Cmd="rsync -ahv"
+    for src_file in ${SourceHbarFile} ${SourceQatFile}; do
-Cmd+=" ${SourceHost}:${SourceFile}"
+        Cmd="rsync -ahv"
-Cmd+=" $OutputDir/"
+        Cmd+=" ${SourceHost}:${src_file}"
-echo ${Cmd}
+        Cmd+=" $OutputDir/"
-eval ${Cmd}
+        echo ${Cmd}
+        eval ${Cmd}
+    done
+done
 
 Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
 echo ${Cmd}
 eval ${Cmd}
 
-SourceDbFile="${OutputDir}/${Date}.mktdata.db"
+ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
-ResultDbFile="${OutputDir}/${Date}.crypto_sim_md.db"
-
 echo "Creating Result Database File ${ResultDbFile}"
+sym=${Symbols[0]}
+for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
+    Cmd="sqlite3"
+    Cmd+=" ${db_file} .schema"
+    Cmd+=" | sqlite3 ${ResultDbFile}"
+    echo ${Cmd}
+    eval ${Cmd}
+done
 
-echo "Creating table md_trades ..."
+set -f # not to expand *
-sqlite3 ${ResultDbFile} <<EOF
-CREATE TABLE  IF NOT EXISTS md_trades (
-    tstamp text,
-    tstamp_ns integer,
-    exchange_id text,
-    instrument_id text,
-    exch text,
-    px real,
-    qty real,
-    trade_id text,
-    condition text,
-    tape text
-);
-CREATE UNIQUE INDEX  IF NOT EXISTS md_trades_uidx
-ON md_trades(tstamp_ns, exchange_id, instrument_id);
-EOF
 
-echo "Creating table md_quotes ..."
+for sym in ${Symbols[@]}; do
-sqlite3 ${ResultDbFile} <<EOF
+   src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
-CREATE TABLE  IF NOT EXISTS md_quotes (
+   src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
-    tstamp text,
+   for srcdb in ${src_hbar_db} ${src_qat_db} ; do
-    tstamp_ns integer,
-    exchange_id text,
-    instrument_id text,
-    bid_exch text,
-    bid_px real,
-    bid_qty real,
-    ask_exch text,
-    ask_px real,
-    ask_qty real
-);
-CREATE UNIQUE INDEX  IF NOT EXISTS md_quotes_uidx
-ON md_quotes(tstamp_ns, exchange_id, instrument_id);
-EOF
 
-echo "Creating table md_1min_bars ..."
+       tables=$(sqlite3 ${srcdb} .tables)
-sqlite3 ${ResultDbFile} <<EOF
+       for table in $(sqlite3 ${srcdb} .tables) ; do
-CREATE TABLE  IF NOT EXISTS md_1min_bars (
+           echo "Loading from ${srcdb}.${table} ..."
-	tstamp text,
+           sqlite3  ${ResultDbFile} <<EOF
-	tstamp_ns integer,
+ATTACH '${srcdb}' AS source_db;
-	exchange_id text,
-	instrument_id text,
-	open real,
-	high real,
-	low real,
-	close real,
-	volume real,
-	vwap real,
-	num_trades integer
-);
-CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
-ON md_1min_bars(tstamp, exchange_id, instrument_id);
-EOF
-
-echo "Loading md_trades ..."
-sqlite3  ${ResultDbFile} <<EOF
-ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
-INSERT OR IGNORE INTO md_trades SELECT
+INSERT INTO ${table} SELECT * FROM source_db.${table};
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
-	time as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	"" as exch,
-	price as px,
-	quantity as qty,
-	"" as trade_id,
-	taker_side as condition,
-	"" as tape
-from source_db.bnbspot_md_trades;
 COMMIT;
-
-BEGIN;
-INSERT OR IGNORE INTO md_trades
-SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
-	time as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	"" as exch,
-	price as px,
-	quantity as qty,
-	"" as trade_id,
-	taker_side as condition,
-	"" as tape
-from source_db.coinbase_md_trades;
-COMMIT;
-
 DETACH source_db;
 EOF
-
+       done
-echo "Loading md_quotes ..."
+       Cmd="rm ${srcdb}"
-sqlite3  ${ResultDbFile} <<EOF
+       echo ${Cmd}
-ATTACH '${SourceDbFile}' AS source_db;
+       eval ${Cmd}
-BEGIN;
+   done
-INSERT OR IGNORE INTO md_quotes SELECT
+done
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
-	time as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	exchange_id as bid_exch,
-	bid_price as bid_px,
-	bid_quantity as bid_qty,
-	exchange_id as ask_exch,
-	ask_price as ask_px,
-	ask_quantity as ask_qty
-from bnbspot_md_booktops;
-COMMIT;
-
-BEGIN;
-INSERT OR IGNORE INTO md_quotes SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
-	time as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	exchange_id as bid_exch,
-	bid_price as bid_px,
-	bid_quantity as bid_qty,
-	exchange_id as ask_exch,
-	ask_price as ask_px,
-	ask_quantity as ask_qty
-from coinbase_md_booktops;
-COMMIT;
-
-DETACH source_db;
-EOF
-
-### --- REPLACE 0 with num_trades ---
-
-echo "Loading md_1min_bars ..."
-sqlite3  ${ResultDbFile} <<EOF
-ATTACH '${SourceDbFile}' AS source_db;
-BEGIN;
-INSERT OR IGNORE INTO md_1min_bars SELECT
-	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tsatmp,
-	tstamp as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	open,
-	high,
-	low,
-	close,
-	volume,
-	vwap,
-	0 as num_trades
-from bnbspot_ohlcv_1min;
-COMMIT;
-
-BEGIN;
-INSERT OR IGNORE INTO md_1min_bars SELECT
-	datetime(tstamp / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (tstamp % 1000000000) / 1000)  as tstamp,
-	tstamp as tstamp_ns,
-	exchange_id,
-	instrument_id,
-	open,
-	high,
-	low,
-	close,
-	volume,
-	vwap,
-	0 as num_trades
-from coinbase_ohlcv_1min;
-COMMIT;
-
-DETACH source_db;
-EOF
-
-Cmd="rm ${SourceDbFile}"
-echo ${Cmd}
-eval ${Cmd}
-
 
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}
 

research/prepare_eqt_simdata.sh

@@ -99,141 +99,48 @@ eval ${Cmd}
 
 ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
 echo "Creating Result Database File ${ResultDbFile}"
+sym=${Symbols[0]}
+for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
+    Cmd="sqlite3"
+    Cmd+=" ${db_file} .schema"
+    Cmd+=" | sqlite3 ${ResultDbFile}"
+    echo ${Cmd}
+    eval ${Cmd}
+done
 
-echo "Creating Result Database File ${ResultDbFile}"
+set -f # not to expand *
-
-echo "Creating table md_trades ..."
-sqlite3 ${ResultDbFile} <<EOF
-CREATE TABLE  IF NOT EXISTS md_trades (
-    tstamp text,
-    tstamp_ns integer,
-    exchange_id text,
-    instrument_id text,
-    exch text,
-    px real,
-    qty real,
-    trade_id text,
-    condition text,
-    tape text
-);
-CREATE UNIQUE INDEX  IF NOT EXISTS md_trades_uidx
-ON md_trades(tstamp_ns, exchange_id, instrument_id);
-EOF
-
-echo "Creating table md_quotes ..."
-sqlite3 ${ResultDbFile} <<EOF
-CREATE TABLE  IF NOT EXISTS md_quotes (
-    tstamp text,
-    tstamp_ns integer,
-    exchange_id text,
-    instrument_id text,
-    bid_exch text,
-    bid_px real,
-    bid_qty real,
-    ask_exch text,
-    ask_px real,
-    ask_qty real
-);
-CREATE UNIQUE INDEX  IF NOT EXISTS md_quotes_uidx
-ON md_quotes(tstamp_ns, exchange_id, instrument_id);
-EOF
-
-echo "Creating table md_1min_bars ..."
-sqlite3 ${ResultDbFile} <<EOF
-CREATE TABLE  IF NOT EXISTS md_1min_bars (
-	tstamp text,
-	tstamp_ns integer,
-	exchange_id text,
-	instrument_id text,
-	open real,
-	high real,
-	low real,
-	close real,
-	volume real,
-	vwap real,
-	num_trades integer
-);
-CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
-ON md_1min_bars(tstamp, exchange_id, instrument_id);
-EOF
-
-
-# set -f # not to expand *
 
 for sym in ${Symbols[@]}; do
    src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
    src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
    for srcdb in ${src_hbar_db} ${src_qat_db} ; do
 
-       echo "Loading md_trades from ${srcdb} ..."
+       tables=$(sqlite3 ${srcdb} .tables)
-       sqlite3  ${ResultDbFile} <<EOF
+       for table in $(sqlite3 ${srcdb} .tables) ; do
+           echo "Loading from ${srcdb}.${table} ..."
+           sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${srcdb}' AS source_db;
 BEGIN;
-INSERT INTO md_trades SELECT
+INSERT INTO ${table} SELECT * FROM source_db.${table};
-	tstamp,
-	tstamp_ns,
-	exchange_id,
-	instrument_id,
-	exch,
-	px,
-	qty,
-	trade_id,
-	condition,
-	tape
-FROM source_db.md_trades;
 COMMIT;
-
-BEGIN;
-INSERT INTO md_quotes SELECT
-	tstamp,
-	tstamp_ns,
-	exchange_id,
-	instrument_id,
-	bid_exch,
-	bid_px,
-	bid_qty,
-	ask_exch,
-	ask_px,
-	ask_qty
-FROM source_db.md_quotes;
-COMMIT;
-
-BEGIN;
-INSERT INTO md_1min_bars SELECT
-	tstamp,
-	tstamp_ns,
-	exchange_id,
-	instrument_id,
-	open,
-	high,
-	low,
-	close,
-	volume,
-	vwap,
-	num_trades
-    FROM source_db.md_1min_bars;
-COMMIT;
-
 DETACH source_db;
 EOF
-
+       done
        Cmd="rm ${srcdb}"
        echo ${Cmd}
        eval ${Cmd}
    done
 done
 
-exit
-
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}
 
-Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
+Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
 echo ${Cmd}
 eval ${Cmd}