prepare sim for crypto (before added num_trades)
This commit is contained in:
parent
bbbdbb5473
commit
11ab9dd2c2
@ -1 +1 @@
|
|||||||
1.9.1,16->23
|
1.9.2,prepare_crypto_simdata
|
||||||
|
|||||||
@ -1,41 +1,30 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
|
# SQLite DDL for simulation
|
||||||
|
# =========================
|
||||||
|
|
||||||
|
|
||||||
|
# -- md_quotes
|
||||||
|
|
||||||
|
# -- md_1min_bars
|
||||||
|
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
echo "Usage: $0 -S <symbols> -d <YYYYMMDD Date> [-O <output dir (./) >]"
|
echo "Usage: $0 -d YYYMMDD Date> [-O <output dir (./) >]"
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# --------------------- Settings
|
# --------------------- Settings
|
||||||
SourceHost=cloud21.cvtt.vpn
|
SourceHost=cloud21.cvtt.vpn
|
||||||
SourceRootDir=/opt/store/cvtt/md_archive/crypto/alpaca_md
|
SourceRootDir=/opt/store/cvtt/md_archive/crypto
|
||||||
|
DbSource=cloud21
|
||||||
# --------------------- Settings
|
# --------------------- Settings
|
||||||
|
|
||||||
|
while getopts ":d:O:" opt; do
|
||||||
is_business_day() {
|
|
||||||
dt=${1}
|
|
||||||
date=$(date -d "${dt}" +"%Y-%m-%d")
|
|
||||||
|
|
||||||
CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
|
||||||
URL="${CalendarURL}&start=${date}&end=${date}"
|
|
||||||
open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
|
|
||||||
if [ -n "${open_time}" ]; then
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
}
|
|
||||||
export -f is_business_day
|
|
||||||
|
|
||||||
|
|
||||||
while getopts ":d:S:O:" opt; do
|
|
||||||
case ${opt} in
|
case ${opt} in
|
||||||
d )
|
d )
|
||||||
Date=$OPTARG
|
Date=$OPTARG
|
||||||
;;
|
;;
|
||||||
S )
|
|
||||||
SymList=$OPTARG
|
|
||||||
;;
|
|
||||||
O )
|
O )
|
||||||
OutputDir=$OPTARG
|
OutputDir=$OPTARG
|
||||||
;;
|
;;
|
||||||
@ -50,10 +39,6 @@ while getopts ":d:S:O:" opt; do
|
|||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
if [ -z ${SymList} ] ; then
|
|
||||||
echo "symbols are not specified"
|
|
||||||
usage
|
|
||||||
fi
|
|
||||||
if [ -z ${Date} ] ; then
|
if [ -z ${Date} ] ; then
|
||||||
echo "date is not specified"
|
echo "date is not specified"
|
||||||
usage
|
usage
|
||||||
@ -63,84 +48,263 @@ if [ -z ${OutputDir} ] ; then
|
|||||||
fi
|
fi
|
||||||
mkdir -p ${OutputDir}
|
mkdir -p ${OutputDir}
|
||||||
|
|
||||||
if ! is_business_day ${Date}; then
|
# --- Binance
|
||||||
echo "${Date} is not business day"
|
Instruments=( PAIR-ADA-USDT )
|
||||||
usage
|
Instruments+=( PAIR-BCH-USDT )
|
||||||
fi
|
Instruments+=( PAIR-BTC-USDT )
|
||||||
|
Instruments+=( PAIR-DOT-USDT )
|
||||||
|
Instruments+=( PAIR-ETH-USDT )
|
||||||
|
Instruments+=( PAIR-LTC-USDT )
|
||||||
|
Instruments+=( PAIR-SOL-USDT )
|
||||||
|
Instruments+=( PAIR-USDC-USDT )
|
||||||
|
Instruments+=( PAIR-XRP-USDT )
|
||||||
|
|
||||||
OLD_IFS=${IFS}
|
# --- Coinbase
|
||||||
IFS=","
|
Instruments+=( PAIR-ADA-USD )
|
||||||
read -ra Symbols <<< "${SymList}"
|
Instruments+=( PAIR-BCH-USD )
|
||||||
IFS=${OLD_IFS}
|
Instruments+=( PAIR-BTC-USD )
|
||||||
|
Instruments+=( PAIR-DOT-USD )
|
||||||
|
Instruments+=( PAIR-ETH-USD )
|
||||||
|
Instruments+=( PAIR-LTC-USD )
|
||||||
|
Instruments+=( PAIR-SOL-USD )
|
||||||
|
Instruments+=( PAIR-XRP-USD )
|
||||||
|
|
||||||
echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}"
|
echo "Date=${Date} Instruments=${Instruments[@]} OutputDir=${OutputDir}"
|
||||||
echo Getting data from ${DataHost} ...
|
echo Getting data from ${DataHost} ...
|
||||||
|
|
||||||
year=$(date -d ${Date} +"%Y")
|
year=$(date -d ${Date} +"%Y")
|
||||||
for sym in ${Symbols[@]}; do
|
month=$(date -d ${Date} +"%m")
|
||||||
inst_id="STOCK-${sym}"
|
SourceDir="${SourceRootDir}/${DbSource}/${year}/${month}"
|
||||||
capital=${sym:0:1}
|
SourceFile="${SourceDir}/${Date}.mktdata.db.gz"
|
||||||
SourceDir="${SourceRootDir}/${year}/${capital}/${sym}"
|
|
||||||
SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz"
|
|
||||||
SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz"
|
|
||||||
|
|
||||||
for src_file in ${SourceHbarFile} ${SourceQatFile}; do
|
Cmd="rsync -ahv"
|
||||||
Cmd="rsync -ahv"
|
Cmd+=" ${SourceHost}:${SourceFile}"
|
||||||
Cmd+=" ${SourceHost}:${src_file}"
|
Cmd+=" $OutputDir/"
|
||||||
Cmd+=" $OutputDir/"
|
echo ${Cmd}
|
||||||
echo ${Cmd}
|
eval ${Cmd}
|
||||||
eval ${Cmd}
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
|
Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
|
||||||
echo ${Cmd}
|
echo ${Cmd}
|
||||||
eval ${Cmd}
|
eval ${Cmd}
|
||||||
|
|
||||||
ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
|
SourceDbFile="${OutputDir}/${Date}.mktdata.db"
|
||||||
|
ResultDbFile="${OutputDir}/${Date}.crypto_sim_md.db"
|
||||||
|
|
||||||
echo "Creating Result Database File ${ResultDbFile}"
|
echo "Creating Result Database File ${ResultDbFile}"
|
||||||
sym=${Symbols[0]}
|
|
||||||
for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
|
|
||||||
Cmd="sqlite3"
|
|
||||||
Cmd+=" ${db_file} .schema"
|
|
||||||
Cmd+=" | sqlite3 ${ResultDbFile}"
|
|
||||||
echo ${Cmd}
|
|
||||||
eval ${Cmd}
|
|
||||||
done
|
|
||||||
|
|
||||||
set -f # not to expand *
|
echo "Creating table md_trades ..."
|
||||||
|
sqlite3 ${ResultDbFile} <<EOF
|
||||||
|
CREATE TABLE IF NOT EXISTS md_trades (
|
||||||
|
tstamp text,
|
||||||
|
tstamp_ns integer,
|
||||||
|
exchange_id text,
|
||||||
|
instrument_id text,
|
||||||
|
exch text,
|
||||||
|
px real,
|
||||||
|
qty real,
|
||||||
|
trade_id text,
|
||||||
|
condition text,
|
||||||
|
tape text
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS md_trades_uidx
|
||||||
|
ON md_trades(tstamp_ns, exchange_id, instrument_id);
|
||||||
|
EOF
|
||||||
|
|
||||||
for sym in ${Symbols[@]}; do
|
echo "Creating table md_quotes ..."
|
||||||
src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
|
sqlite3 ${ResultDbFile} <<EOF
|
||||||
src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
|
CREATE TABLE IF NOT EXISTS md_quotes (
|
||||||
for srcdb in ${src_hbar_db} ${src_qat_db} ; do
|
tstamp text,
|
||||||
|
tstamp_ns integer,
|
||||||
|
exchange_id text,
|
||||||
|
instrument_id text,
|
||||||
|
bid_exch text,
|
||||||
|
bid_px real,
|
||||||
|
bid_qty real,
|
||||||
|
ask_exch text,
|
||||||
|
ask_px real,
|
||||||
|
ask_qty real
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS md_quotes_uidx
|
||||||
|
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
|
||||||
|
EOF
|
||||||
|
|
||||||
tables=$(sqlite3 ${srcdb} .tables)
|
echo "Creating table md_1min_bars ..."
|
||||||
for table in $(sqlite3 ${srcdb} .tables) ; do
|
sqlite3 ${ResultDbFile} <<EOF
|
||||||
echo "Loading from ${srcdb}.${table} ..."
|
CREATE TABLE IF NOT EXISTS md_1min_bars (
|
||||||
sqlite3 ${ResultDbFile} <<EOF
|
tstamp text,
|
||||||
ATTACH '${srcdb}' AS source_db;
|
tstamp_ns integer,
|
||||||
|
exchange_id text,
|
||||||
|
instrument_id text,
|
||||||
|
open real,
|
||||||
|
high real,
|
||||||
|
low real,
|
||||||
|
close real,
|
||||||
|
volume real,
|
||||||
|
vwap real,
|
||||||
|
num_trades integer
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
|
||||||
|
ON md_1min_bars(tstamp, exchange_id, instrument_id);
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "Loading md_trades ..."
|
||||||
|
sqlite3 ${ResultDbFile} <<EOF
|
||||||
|
ATTACH '${SourceDbFile}' AS source_db;
|
||||||
BEGIN;
|
BEGIN;
|
||||||
INSERT INTO ${table} SELECT * FROM source_db.${table};
|
INSERT OR IGNORE INTO md_trades SELECT
|
||||||
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
||||||
|
time as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
"" as exch,
|
||||||
|
price as px,
|
||||||
|
quantity as qty,
|
||||||
|
"" as trade_id,
|
||||||
|
taker_side as condition,
|
||||||
|
"" as tape
|
||||||
|
from source_db.bnbspot_md_trades;
|
||||||
COMMIT;
|
COMMIT;
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
INSERT OR IGNORE INTO md_trades
|
||||||
|
SELECT
|
||||||
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
||||||
|
time as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
"" as exch,
|
||||||
|
price as px,
|
||||||
|
quantity as qty,
|
||||||
|
"" as trade_id,
|
||||||
|
taker_side as condition,
|
||||||
|
"" as tape
|
||||||
|
from source_db.coinbase_md_trades;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
DETACH source_db;
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "Loading md_quotes ..."
|
||||||
|
sqlite3 ${ResultDbFile} <<EOF
|
||||||
|
ATTACH '${SourceDbFile}' AS source_db;
|
||||||
|
BEGIN;
|
||||||
|
INSERT OR IGNORE INTO md_quotes SELECT
|
||||||
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
||||||
|
time as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
exchange_id as bid_exch,
|
||||||
|
bid_price as bid_px,
|
||||||
|
bid_quantity as bid_qty,
|
||||||
|
exchange_id as ask_exch,
|
||||||
|
ask_price as ask_px,
|
||||||
|
ask_quantity as ask_qty
|
||||||
|
from bnbspot_md_booktops;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
INSERT OR IGNORE INTO md_quotes SELECT
|
||||||
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
||||||
|
time as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
exchange_id as bid_exch,
|
||||||
|
bid_price as bid_px,
|
||||||
|
bid_quantity as bid_qty,
|
||||||
|
exchange_id as ask_exch,
|
||||||
|
ask_price as ask_px,
|
||||||
|
ask_quantity as ask_qty
|
||||||
|
from coinbase_md_booktops;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
DETACH source_db;
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "Loading md_quotes ..."
|
||||||
|
sqlite3 ${ResultDbFile} <<EOF
|
||||||
|
ATTACH '${SourceDbFile}' AS source_db;
|
||||||
|
BEGIN;
|
||||||
|
INSERT OR IGNORE INTO md_quotes SELECT
|
||||||
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
||||||
|
time as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
exchange_id as bid_exch,
|
||||||
|
bid_price as bid_px,
|
||||||
|
bid_quantity as bid_qty,
|
||||||
|
exchange_id as ask_exch,
|
||||||
|
ask_price as ask_px,
|
||||||
|
ask_quantity as ask_qty
|
||||||
|
from bnbspot_md_booktops;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
INSERT OR IGNORE INTO md_quotes SELECT
|
||||||
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
||||||
|
time as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
exchange_id as bid_exch,
|
||||||
|
bid_price as bid_px,
|
||||||
|
bid_quantity as bid_qty,
|
||||||
|
exchange_id as ask_exch,
|
||||||
|
ask_price as ask_px,
|
||||||
|
ask_quantity as ask_qty
|
||||||
|
from coinbase_md_booktops;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
DETACH source_db;
|
||||||
|
EOF
|
||||||
|
|
||||||
|
### --- REPLACE 0 with num_trades ---
|
||||||
|
|
||||||
|
echo "Loading md_1min_bars ..."
|
||||||
|
sqlite3 ${ResultDbFile} <<EOF
|
||||||
|
ATTACH '${SourceDbFile}' AS source_db;
|
||||||
|
BEGIN;
|
||||||
|
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||||
|
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstmp,
|
||||||
|
tstamp as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
open,
|
||||||
|
high,
|
||||||
|
low,
|
||||||
|
close,
|
||||||
|
volume,
|
||||||
|
vwap,
|
||||||
|
0 as num_trades
|
||||||
|
from bnbspot_ohlcv_1min;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||||
|
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstmp,
|
||||||
|
tstamp as tstamp_ns,
|
||||||
|
exchange_id,
|
||||||
|
instrument_id,
|
||||||
|
open,
|
||||||
|
high,
|
||||||
|
low,
|
||||||
|
close,
|
||||||
|
volume,
|
||||||
|
vwap,
|
||||||
|
0 as num_trades
|
||||||
|
from coinbase_ohlcv_1min;
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
DETACH source_db;
|
DETACH source_db;
|
||||||
EOF
|
EOF
|
||||||
done
|
|
||||||
Cmd="rm ${srcdb}"
|
|
||||||
echo ${Cmd}
|
|
||||||
eval ${Cmd}
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
Cmd="gzip ${ResultDbFile}"
|
Cmd="gzip ${ResultDbFile}"
|
||||||
echo ${Cmd}
|
echo ${Cmd}
|
||||||
eval ${Cmd}
|
eval ${Cmd}
|
||||||
|
|
||||||
Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
|
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
|
||||||
echo ${Cmd}
|
echo ${Cmd}
|
||||||
eval ${Cmd}
|
eval ${Cmd}
|
||||||
|
|
||||||
Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
|
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
|
||||||
echo ${Cmd}
|
echo ${Cmd}
|
||||||
eval ${Cmd}
|
eval ${Cmd}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user