285 lines
6.0 KiB
Bash
Executable File
285 lines
6.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# SQLite DDL for simulation
|
|
# =========================
|
|
|
|
|
|
# -- md_quotes
|
|
|
|
# -- md_1min_bars
|
|
|
|
|
|
usage() {
|
|
echo "Usage: $0 -d YYYMMDD Date> [-O <output dir (./) >]"
|
|
exit 1
|
|
}
|
|
|
|
# --------------------- Settings
|
|
SourceHost=cloud21.cvtt.vpn
|
|
SourceRootDir=/opt/store/cvtt/md_archive/crypto
|
|
DbSource=cloud21
|
|
# --------------------- Settings
|
|
|
|
while getopts ":d:O:" opt; do
|
|
case ${opt} in
|
|
d )
|
|
Date=$OPTARG
|
|
;;
|
|
O )
|
|
OutputDir=$OPTARG
|
|
;;
|
|
\? )
|
|
echo "Invalid option: -$OPTARG" >&2
|
|
usage
|
|
;;
|
|
: )
|
|
echo "Option -$OPTARG requires an argument." >&2
|
|
usage
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ -z ${Date} ] ; then
|
|
echo "date is not specified"
|
|
usage
|
|
fi
|
|
if [ -z ${OutputDir} ] ; then
|
|
OutputDir=.
|
|
fi
|
|
mkdir -p ${OutputDir}
|
|
|
|
# --- Binance
|
|
Instruments=( PAIR-ADA-USDT )
|
|
Instruments+=( PAIR-BCH-USDT )
|
|
Instruments+=( PAIR-BTC-USDT )
|
|
Instruments+=( PAIR-DOT-USDT )
|
|
Instruments+=( PAIR-ETH-USDT )
|
|
Instruments+=( PAIR-LTC-USDT )
|
|
Instruments+=( PAIR-SOL-USDT )
|
|
Instruments+=( PAIR-USDC-USDT )
|
|
Instruments+=( PAIR-XRP-USDT )
|
|
|
|
# --- Coinbase
|
|
Instruments+=( PAIR-ADA-USD )
|
|
Instruments+=( PAIR-BCH-USD )
|
|
Instruments+=( PAIR-BTC-USD )
|
|
Instruments+=( PAIR-DOT-USD )
|
|
Instruments+=( PAIR-ETH-USD )
|
|
Instruments+=( PAIR-LTC-USD )
|
|
Instruments+=( PAIR-SOL-USD )
|
|
Instruments+=( PAIR-XRP-USD )
|
|
|
|
echo "Date=${Date} Instruments=${Instruments[@]} OutputDir=${OutputDir}"
|
|
echo Getting data from ${DataHost} ...
|
|
|
|
year=$(date -d ${Date} +"%Y")
|
|
month=$(date -d ${Date} +"%m")
|
|
SourceDir="${SourceRootDir}/${DbSource}/${year}/${month}"
|
|
SourceFile="${SourceDir}/${Date}.mktdata.db.gz"
|
|
|
|
Cmd="rsync -ahv"
|
|
Cmd+=" ${SourceHost}:${SourceFile}"
|
|
Cmd+=" $OutputDir/"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
SourceDbFile="${OutputDir}/${Date}.mktdata.db"
|
|
ResultDbFile="${OutputDir}/${Date}.crypto_sim_md.db"
|
|
|
|
echo "Creating Result Database File ${ResultDbFile}"
|
|
|
|
echo "Creating table md_trades ..."
|
|
sqlite3 ${ResultDbFile} <<EOF
|
|
CREATE TABLE IF NOT EXISTS md_trades (
|
|
tstamp text,
|
|
tstamp_ns integer,
|
|
exchange_id text,
|
|
instrument_id text,
|
|
exch text,
|
|
px real,
|
|
qty real,
|
|
trade_id text,
|
|
condition text,
|
|
tape text
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS md_trades_uidx
|
|
ON md_trades(tstamp_ns, exchange_id, instrument_id);
|
|
EOF
|
|
|
|
echo "Creating table md_quotes ..."
|
|
sqlite3 ${ResultDbFile} <<EOF
|
|
CREATE TABLE IF NOT EXISTS md_quotes (
|
|
tstamp text,
|
|
tstamp_ns integer,
|
|
exchange_id text,
|
|
instrument_id text,
|
|
bid_exch text,
|
|
bid_px real,
|
|
bid_qty real,
|
|
ask_exch text,
|
|
ask_px real,
|
|
ask_qty real
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS md_quotes_uidx
|
|
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
|
|
EOF
|
|
|
|
echo "Creating table md_1min_bars ..."
|
|
sqlite3 ${ResultDbFile} <<EOF
|
|
CREATE TABLE IF NOT EXISTS md_1min_bars (
|
|
tstamp text,
|
|
tstamp_ns integer,
|
|
exchange_id text,
|
|
instrument_id text,
|
|
open real,
|
|
high real,
|
|
low real,
|
|
close real,
|
|
volume real,
|
|
vwap real,
|
|
num_trades integer
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
|
|
ON md_1min_bars(tstamp, exchange_id, instrument_id);
|
|
EOF
|
|
|
|
echo "Loading md_trades ..."
|
|
sqlite3 ${ResultDbFile} <<EOF
|
|
ATTACH '${SourceDbFile}' AS source_db;
|
|
BEGIN;
|
|
INSERT OR IGNORE INTO md_trades SELECT
|
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
|
time as tstamp_ns,
|
|
exchange_id,
|
|
instrument_id,
|
|
"" as exch,
|
|
price as px,
|
|
quantity as qty,
|
|
"" as trade_id,
|
|
taker_side as condition,
|
|
"" as tape
|
|
from source_db.bnbspot_md_trades;
|
|
COMMIT;
|
|
|
|
BEGIN;
|
|
INSERT OR IGNORE INTO md_trades
|
|
SELECT
|
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
|
time as tstamp_ns,
|
|
exchange_id,
|
|
instrument_id,
|
|
"" as exch,
|
|
price as px,
|
|
quantity as qty,
|
|
"" as trade_id,
|
|
taker_side as condition,
|
|
"" as tape
|
|
from source_db.coinbase_md_trades;
|
|
COMMIT;
|
|
|
|
DETACH source_db;
|
|
EOF
|
|
|
|
echo "Loading md_quotes ..."
|
|
sqlite3 ${ResultDbFile} <<EOF
|
|
ATTACH '${SourceDbFile}' AS source_db;
|
|
BEGIN;
|
|
INSERT OR IGNORE INTO md_quotes SELECT
|
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstmp,
|
|
time as tstamp_ns,
|
|
exchange_id,
|
|
instrument_id,
|
|
exchange_id as bid_exch,
|
|
bid_price as bid_px,
|
|
bid_quantity as bid_qty,
|
|
exchange_id as ask_exch,
|
|
ask_price as ask_px,
|
|
ask_quantity as ask_qty
|
|
from bnbspot_md_booktops;
|
|
COMMIT;
|
|
|
|
BEGIN;
|
|
INSERT OR IGNORE INTO md_quotes SELECT
|
|
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
|
time as tstamp_ns,
|
|
exchange_id,
|
|
instrument_id,
|
|
exchange_id as bid_exch,
|
|
bid_price as bid_px,
|
|
bid_quantity as bid_qty,
|
|
exchange_id as ask_exch,
|
|
ask_price as ask_px,
|
|
ask_quantity as ask_qty
|
|
from coinbase_md_booktops;
|
|
COMMIT;
|
|
|
|
DETACH source_db;
|
|
EOF
|
|
|
|
### --- REPLACE 0 with num_trades ---
|
|
|
|
echo "Loading md_1min_bars ..."
|
|
sqlite3 ${ResultDbFile} <<EOF
|
|
ATTACH '${SourceDbFile}' AS source_db;
|
|
BEGIN;
|
|
INSERT OR IGNORE INTO md_1min_bars SELECT
|
|
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tsatmp,
|
|
tstamp as tstamp_ns,
|
|
exchange_id,
|
|
instrument_id,
|
|
open,
|
|
high,
|
|
low,
|
|
close,
|
|
volume,
|
|
vwap,
|
|
0 as num_trades
|
|
from bnbspot_ohlcv_1min;
|
|
COMMIT;
|
|
|
|
BEGIN;
|
|
INSERT OR IGNORE INTO md_1min_bars SELECT
|
|
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstamp,
|
|
tstamp as tstamp_ns,
|
|
exchange_id,
|
|
instrument_id,
|
|
open,
|
|
high,
|
|
low,
|
|
close,
|
|
volume,
|
|
vwap,
|
|
0 as num_trades
|
|
from coinbase_ohlcv_1min;
|
|
COMMIT;
|
|
|
|
DETACH source_db;
|
|
EOF
|
|
|
|
Cmd="rm ${SourceDbFile}"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
|
|
Cmd="gzip ${ResultDbFile}"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
Cmd="rm ${ResultDbFile}.gz"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
|
|
echo Done $0 ${*}
|