daily sim for docker.initial
This commit is contained in:
+242
@@ -0,0 +1,242 @@
|
||||
#!/bin/bash
|
||||
|
||||
# --------------------- Settings
|
||||
SOURCE_HOST=cloud21.cvtt.vpn
|
||||
SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto
|
||||
DB_SOURCE=cloud28
|
||||
OUTPUT_DIR=/tmp
|
||||
DATE=20250516
|
||||
# --------------------- Settings
|
||||
|
||||
if [ -z ${DATE} ] ; then
|
||||
DATE=$(date -d 'yesterday' +'%Y%m%d')
|
||||
fi
|
||||
|
||||
if [ -z ${OUTPUT_DIR} ] ; then
|
||||
OUTPUT_DIR=.
|
||||
fi
|
||||
|
||||
echo "DATE=${DATE} SOURCE_HOST=${SOURCE_HOST}"
|
||||
|
||||
mkdir -p ${OUTPUT_DIR}
|
||||
|
||||
year=$(date -d ${DATE} +"%Y")
|
||||
month=$(date -d ${DATE} +"%m")
|
||||
SourceDir="${SOURCE_ROOT_DIR}/${DB_SOURCE}/${year}/${month}"
|
||||
SourceFile="${SourceDir}/${DATE}.mktdata.db.gz"
|
||||
|
||||
Cmd="rsync -ahv"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${SOURCE_HOST}:${SourceFile}"
|
||||
Cmd+=" $OUTPUT_DIR/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Cmd="(cd ${OUTPUT_DIR} && gunzip -f *.db.gz)"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
SourceDbFile="${OUTPUT_DIR}/${DATE}.mktdata.db"
|
||||
ResultDbFile="${OUTPUT_DIR}/${DATE}.crypto_sim_md.db"
|
||||
|
||||
echo "SourceDbFile=${SourceDbFile}"
|
||||
echo "Creating Result Database File ${ResultDbFile}"
|
||||
cleanup() {
|
||||
rm ${SourceDbFile}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "Creating table md_trades ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_trades (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
exch text,
|
||||
px real,
|
||||
qty real,
|
||||
trade_id text,
|
||||
condition text,
|
||||
tape text
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_trades_uidx
|
||||
ON md_trades(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_quotes ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_quotes (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
bid_exch text,
|
||||
bid_px real,
|
||||
bid_qty real,
|
||||
ask_exch text,
|
||||
ask_px real,
|
||||
ask_qty real
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_quotes_uidx
|
||||
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_1min_bars ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_1min_bars (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
open real,
|
||||
high real,
|
||||
low real,
|
||||
close real,
|
||||
volume real,
|
||||
vwap real,
|
||||
num_trades integer
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
|
||||
ON md_1min_bars(tstamp, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Loading md_trades ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${SourceDbFile}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_trades SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
"" as exch,
|
||||
price as px,
|
||||
quantity as qty,
|
||||
"" as trade_id,
|
||||
taker_side as condition,
|
||||
"" as tape
|
||||
from source_db.bnbspot_md_trades;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_trades
|
||||
SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
"" as exch,
|
||||
price as px,
|
||||
quantity as qty,
|
||||
"" as trade_id,
|
||||
taker_side as condition,
|
||||
"" as tape
|
||||
from source_db.coinbase_md_trades;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
echo "Loading md_quotes ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${SourceDbFile}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_quotes SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
exchange_id as bid_exch,
|
||||
bid_price as bid_px,
|
||||
bid_quantity as bid_qty,
|
||||
exchange_id as ask_exch,
|
||||
ask_price as ask_px,
|
||||
ask_quantity as ask_qty
|
||||
from bnbspot_md_booktops;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_quotes SELECT
|
||||
datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' || printf('%06d', (exchange_ts_ns % 1000000000) / 1000) as tstamp,
|
||||
time as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
exchange_id as bid_exch,
|
||||
bid_price as bid_px,
|
||||
bid_quantity as bid_qty,
|
||||
exchange_id as ask_exch,
|
||||
ask_price as ask_px,
|
||||
ask_quantity as ask_qty
|
||||
from coinbase_md_booktops;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
### --- REPLACE 0 with num_trades ---
|
||||
|
||||
echo "Loading md_1min_bars ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${SourceDbFile}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tsatmp,
|
||||
tstamp as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
volume,
|
||||
vwap,
|
||||
0 as num_trades
|
||||
from bnbspot_ohlcv_1min;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||
datetime(tstamp / 1000000000, 'unixepoch') || '.' || printf('%06d', (tstamp % 1000000000) / 1000) as tstamp,
|
||||
tstamp as tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
volume,
|
||||
vwap,
|
||||
0 as num_trades
|
||||
from coinbase_ohlcv_1min;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
exit
|
||||
|
||||
Cmd="gzip ${ResultDbFile}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Cmd="rsync -ahvv ${ResultDbFile}.gz "
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Cmd="rm ${ResultDbFile}.gz"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
echo Done $0 ${*}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -S <symbols> -d <YYYYMMDD Date> [-O <output dir (./) >]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --------------------- Settings
|
||||
SourceHost=cloud21.cvtt.vpn
|
||||
SourceRootDir=/opt/store/cvtt/md_archive/equity/alpaca_md
|
||||
# --------------------- Settings
|
||||
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
date=$(date -d "${dt}" +"%Y-%m-%d")
|
||||
|
||||
CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
|
||||
URL="${CalendarURL}&start=${date}&end=${date}"
|
||||
open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
|
||||
while getopts ":d:S:O:" opt; do
|
||||
case ${opt} in
|
||||
d )
|
||||
Date=$OPTARG
|
||||
;;
|
||||
S )
|
||||
SymList=$OPTARG
|
||||
;;
|
||||
O )
|
||||
OutputDir=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z ${SymList} ] ; then
|
||||
echo "symbols are not specified"
|
||||
usage
|
||||
fi
|
||||
if [ -z ${Date} ] ; then
|
||||
echo "date is not specified"
|
||||
usage
|
||||
fi
|
||||
if [ -z ${OutputDir} ] ; then
|
||||
OutputDir=.
|
||||
fi
|
||||
mkdir -p ${OutputDir}
|
||||
|
||||
if ! is_business_day ${Date}; then
|
||||
echo "${Date} is not business day"
|
||||
usage
|
||||
fi
|
||||
|
||||
OLD_IFS=${IFS}
|
||||
IFS=","
|
||||
read -ra Symbols <<< "${SymList}"
|
||||
IFS=${OLD_IFS}
|
||||
|
||||
echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}"
|
||||
echo Getting data from ${DataHost} ...
|
||||
|
||||
year=$(date -d ${Date} +"%Y")
|
||||
for sym in ${Symbols[@]}; do
|
||||
inst_id="STOCK-${sym}"
|
||||
capital=${sym:0:1}
|
||||
SourceDir="${SourceRootDir}/${year}/${capital}/${sym}"
|
||||
SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz"
|
||||
SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz"
|
||||
|
||||
for src_file in ${SourceHbarFile} ${SourceQatFile}; do
|
||||
Cmd="rsync -ahv"
|
||||
Cmd+=" ${SourceHost}:${src_file}"
|
||||
Cmd+=" $OutputDir/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
done
|
||||
|
||||
Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
|
||||
echo "Creating Result Database File ${ResultDbFile}"
|
||||
|
||||
echo "Creating Result Database File ${ResultDbFile}"
|
||||
|
||||
echo "Creating table md_trades ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_trades (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
exch text,
|
||||
px real,
|
||||
qty real,
|
||||
trade_id text,
|
||||
condition text,
|
||||
tape text
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_trades_uidx
|
||||
ON md_trades(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_quotes ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
CREATE TABLE IF NOT EXISTS md_quotes (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
bid_exch text,
|
||||
bid_px real,
|
||||
bid_qty real,
|
||||
ask_exch text,
|
||||
ask_px real,
|
||||
ask_qty real
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_quotes_uidx
|
||||
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
echo "Creating table md_1min_bars ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
CREATE TABLE IF NOT EXISTS md_1min_bars (
|
||||
tstamp text,
|
||||
tstamp_ns integer,
|
||||
exchange_id text,
|
||||
instrument_id text,
|
||||
open real,
|
||||
high real,
|
||||
low real,
|
||||
close real,
|
||||
volume real,
|
||||
vwap real,
|
||||
num_trades integer
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
|
||||
ON md_1min_bars(tstamp, exchange_id, instrument_id);
|
||||
EOF
|
||||
|
||||
|
||||
# set -f # not to expand *
|
||||
|
||||
for sym in ${Symbols[@]}; do
|
||||
src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
|
||||
src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
|
||||
|
||||
echo "Loading md_trades and md_quotes from ${src_qat_db} ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${src_qat_db}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_trades SELECT
|
||||
tstamp,
|
||||
tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
exch,
|
||||
px,
|
||||
qty,
|
||||
trade_id,
|
||||
condition,
|
||||
tape
|
||||
FROM source_db.md_trades;
|
||||
COMMIT;
|
||||
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_quotes SELECT
|
||||
tstamp,
|
||||
tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
bid_exch,
|
||||
bid_px,
|
||||
bid_qty,
|
||||
ask_exch,
|
||||
ask_px,
|
||||
ask_qty
|
||||
FROM source_db.md_quotes;
|
||||
COMMIT;
|
||||
EOF
|
||||
|
||||
echo "Loading md_1min_bars from ${src_hbar_db} ..."
|
||||
sqlite3 ${ResultDbFile} <<EOF
|
||||
.echo ON
|
||||
ATTACH '${src_hbar_db}' AS source_db;
|
||||
BEGIN;
|
||||
INSERT OR IGNORE INTO md_1min_bars SELECT
|
||||
tstamp,
|
||||
tstamp_ns,
|
||||
exchange_id,
|
||||
instrument_id,
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
volume,
|
||||
vwap,
|
||||
num_trades
|
||||
FROM source_db.md_1min_bars;
|
||||
COMMIT;
|
||||
|
||||
DETACH source_db;
|
||||
EOF
|
||||
|
||||
Cmd="rm ${src_hbar_db} ${src_qat_db}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
|
||||
Cmd="gzip ${ResultDbFile}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Cmd="rm ${ResultDbFile}.gz"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
echo Done $0 ${*}
|
||||
Reference in New Issue
Block a user