version only

docker/market_data/hist/crypto_md_day.sh

@@ -15,6 +15,7 @@ Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
 Cmd="${Cmd} --rm"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -h ${host}"
 Cmd="${Cmd} -d ${date}"

docker/market_data/hist/load_alpaca_hbar.sh

@@ -91,6 +91,7 @@ Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=${ContainerName}"
 Cmd="${Cmd} --rm"
 Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -d ${date_to_load}"
 if [ -n "${NumSymbols}" ]; then

docker/market_data/hist/load_alpaca_qat.sh

@@ -96,6 +96,7 @@ Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=${ContainerName}"
 Cmd="${Cmd} --rm"
 Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -d ${date_to_load}"
 Cmd="${Cmd} -S ${Symbols}"

docker/market_data/recorder/md_recorder.sh

@@ -49,6 +49,7 @@ Cmd="${Cmd} --rm"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/.creds:/.creds"
 Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
 Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
 Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"

docker/market_data/recorder/md_recorder_monitor.sh

@@ -27,9 +27,6 @@ is_container_running() {
     fi
 }
 
-
-
-
 Instance=${1}
 if ! is_valid "${Instance}"; then
     usage
@@ -53,6 +50,7 @@ Cmd="${Cmd} --rm"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/.creds:/.creds"
 Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
 Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
 Cmd="${Cmd} ${Instance} ${AdminPort}"

release_version.txt

@@ -1 +1 @@
-1.9.2,prepare_crypto_simdata
+1.9.3.F1,creds to be taken from ~/.creds

research/prepare_crypto_simdata.sh

@@ -152,7 +152,7 @@ sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
 INSERT OR IGNORE INTO md_trades SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -168,7 +168,7 @@ COMMIT;
 BEGIN;
 INSERT OR IGNORE INTO md_trades
 SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -189,7 +189,7 @@ sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
 INSERT OR IGNORE INTO md_quotes SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,

research/prepare_eqt_simdata.sh

@@ -163,13 +163,12 @@ EOF
 for sym in ${Symbols[@]}; do
    src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
    src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
-   for srcdb in ${src_hbar_db} ${src_qat_db} ; do
 
-       echo "Loading md_trades from ${srcdb} ..."
+    echo "Loading md_trades and md_quotes from ${src_qat_db} ..."
     sqlite3  ${ResultDbFile} <<EOF
-ATTACH '${srcdb}' AS source_db;
+ATTACH '${src_qat_db}' AS source_db;
 BEGIN;
-INSERT INTO md_trades SELECT
+INSERT OR IGNORE INTO md_trades SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -184,7 +183,7 @@ FROM source_db.md_trades;
 COMMIT;
 
 BEGIN;
-INSERT INTO md_quotes SELECT
+INSERT OR IGNORE INTO md_quotes SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -197,9 +196,13 @@ INSERT INTO md_quotes SELECT
 	ask_qty
 FROM source_db.md_quotes;
 COMMIT;
+EOF
 
+    echo "Loading md_1min_bars from ${src_hbar_db} ..."
+    sqlite3  ${ResultDbFile} <<EOF
+ATTACH '${src_hbar_db}' AS source_db;
 BEGIN;
-INSERT INTO md_1min_bars SELECT
+INSERT OR IGNORE INTO md_1min_bars SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -217,13 +220,10 @@ COMMIT;
 DETACH source_db;
 EOF
 
-       Cmd="rm ${srcdb}"
+    Cmd="rm ${src_hbar_db} ${src_qat_db}"
     echo ${Cmd}
     eval ${Cmd}
 done
-done
-
-exit
 
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}