ops/scripts/mkt_data/sim_prepare/prepare_eqt_simdata.sh

#!/bin/bash

# --------------------- Settings
SOURCE_HOST=cloud21.cvtt.vpn
SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/equity/alpaca_md
SYMBOL_LIST=CAN #,COIN,GBTC,HOOD,MSTR,PYPL,XYZ
OUTPUT_DIR=/tmp
DATE=20250514
RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/test/alpaca_md/sim/"
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/ 
#cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
# --------------------- Settings

# --------------------- Settings


is_business_day() {
    dt=${1}
    date=$(date -d "${dt}" +"%Y-%m-%d")

    CalendarURL=https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS
    URL="${CalendarURL}&start=${date}&end=${date}"
    echo "URL=${URL}"
    curl $URL
    open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
    echo "open_time=${open_time}"

    if [ -n  "${open_time}" ]; then
        return 0
    else
        return 1
    fi

}
export -f is_business_day

if [ -z ${SYMBOL_LIST}  ] ; then
    echo "symbols are not specified"
    exit
fi
if [ -z ${DATE}  ] ; then
    DATE=$(date -d 'yesterday' +'%Y%m%d')
fi

mkdir -p ${OUTPUT_DIR}

if ! is_business_day ${DATE}; then
   echo "${DATE} is not business day"
   exit
fi

OLD_IFS=${IFS}
IFS=","
read -ra Symbols <<< "${SYMBOL_LIST}"
IFS=${OLD_IFS}

echo "DATE=${DATE} Symbols=${Symbols[@]} OUTPUT_DIR=${OUTPUT_DIR}"
echo Getting data from ${DataHost} ...

year=$(date -d ${DATE} +"%Y")
for sym in ${Symbols[@]}; do
    inst_id="STOCK-${sym}"
    capital=${sym:0:1}
    SourceDir="${SOURCE_ROOT_DIR}/${year}/${capital}/${sym}"
    SourceHbarFile="${SourceDir}/${DATE}.${sym}.alpaca_1m_bars.db.gz"
    SourceQatFile="${SourceDir}/${DATE}.${sym}.alpaca_qat.db.gz"

    for src_file in ${SourceHbarFile} ${SourceQatFile}; do
        Cmd="rsync -ahv"
        Cmd+=" ${SOURCE_HOST}:${src_file}"
        Cmd+=" $OUTPUT_DIR/"
        echo ${Cmd}
        eval ${Cmd}
    done
done

Cmd="(cd ${OUTPUT_DIR} && gunzip *.db.gz)"
echo ${Cmd}
eval ${Cmd}

ResultDbFile="${OUTPUT_DIR}/${DATE}.alpaca_sim_md.db"
echo "Creating Result Database File ${ResultDbFile}"

echo "Creating table md_trades ..."
sqlite3 ${ResultDbFile} <<EOF
.echo ON
CREATE TABLE  IF NOT EXISTS md_trades (
    tstamp text,
    tstamp_ns integer,
    exchange_id text,
    instrument_id text,
    exch text,
    px real,
    qty real,
    trade_id text,
    condition text,
    tape text
);
CREATE UNIQUE INDEX  IF NOT EXISTS md_trades_uidx
ON md_trades(tstamp_ns, exchange_id, instrument_id);
EOF

echo "Creating table md_quotes ..."
sqlite3 ${ResultDbFile} <<EOF
.echo ON
CREATE TABLE  IF NOT EXISTS md_quotes (
    tstamp text,
    tstamp_ns integer,
    exchange_id text,
    instrument_id text,
    bid_exch text,
    bid_px real,
    bid_qty real,
    ask_exch text,
    ask_px real,
    ask_qty real
);
CREATE UNIQUE INDEX  IF NOT EXISTS md_quotes_uidx
ON md_quotes(tstamp_ns, exchange_id, instrument_id);
EOF

echo "Creating table md_1min_bars ..."
sqlite3 ${ResultDbFile} <<EOF
CREATE TABLE  IF NOT EXISTS md_1min_bars (
	tstamp text,
	tstamp_ns integer,
	exchange_id text,
	instrument_id text,
	open real,
	high real,
	low real,
	close real,
	volume real,
	vwap real,
	num_trades integer
);
CREATE UNIQUE INDEX IF NOT EXISTS md_1min_bars_uidx
ON md_1min_bars(tstamp, exchange_id, instrument_id);
EOF


# set -f # not to expand *

for sym in ${Symbols[@]}; do
   src_hbar_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_1m_bars.db
   src_qat_db=${OUTPUT_DIR}/${DATE}.${sym}.alpaca_qat.db

    echo "Loading md_trades and md_quotes from ${src_qat_db} ..."
    sqlite3  ${ResultDbFile} <<EOF
.echo ON
ATTACH '${src_qat_db}' AS source_db;
BEGIN;
INSERT OR IGNORE INTO md_trades SELECT
	tstamp,
	tstamp_ns,
	exchange_id,
	instrument_id,
	exch,
	px,
	qty,
	trade_id,
	condition,
	tape
FROM source_db.md_trades;
COMMIT;

BEGIN;
INSERT OR IGNORE INTO md_quotes SELECT
	tstamp,
	tstamp_ns,
	exchange_id,
	instrument_id,
	bid_exch,
	bid_px,
	bid_qty,
	ask_exch,
	ask_px,
	ask_qty
FROM source_db.md_quotes;
COMMIT;
EOF

    echo "Loading md_1min_bars from ${src_hbar_db} ..."
    sqlite3  ${ResultDbFile} <<EOF
.echo ON
ATTACH '${src_hbar_db}' AS source_db;
BEGIN;
INSERT OR IGNORE INTO md_1min_bars SELECT
	tstamp,
	tstamp_ns,
	exchange_id,
	instrument_id,
	open,
	high,
	low,
	close,
	volume,
	vwap,
	num_trades
    FROM source_db.md_1min_bars;
COMMIT;

DETACH source_db;
EOF

Cmd="rm ${src_hbar_db} ${src_qat_db}"
    echo ${Cmd}
    eval ${Cmd}
done

Cmd="gzip ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}

for tgt in ${RSYNC_TARGETS} ; do 
    Cmd="/usr/bin/rsync -ahv"
    Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
    Cmd+=" ${ResultDbFile}.gz ${tgt}"
    echo ${Cmd}
    eval ${Cmd}
done 

echo Done $0 ${*}