minor - docker build script logging

build/docker_images_builder.sh

@@ -66,7 +66,9 @@ build_docker_image() {
     convert_symlink_to_dir ${project}
 
     DockerDir=./docker_dev/${docker_dev_path}
-    echo "Building ${title}... on ${DockerDev}"
+    echo "================================================================================"
+    echo "Building ${title}... on ${DockerDir}"
+    echo "================================================================================"
 
     Cmd="docker build"
     Cmd+=" -t ${image_name}"
@@ -194,5 +196,6 @@ Cmd="rm -rf ${RootDir}/docker_dev"
 echo ${Cmd}
 eval ${Cmd}
 
+echo "Done ${0} ${*}"
 #===============================================================================
 #===============================================================================

docker/market_data/hist/crypto_md_day.sh

@@ -15,6 +15,7 @@ Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
 Cmd="${Cmd} --rm"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -h ${host}"
 Cmd="${Cmd} -d ${date}"

docker/market_data/hist/load_alpaca_hbar.sh

@@ -91,6 +91,7 @@ Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=${ContainerName}"
 Cmd="${Cmd} --rm"
 Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -d ${date_to_load}"
 if [ -n "${NumSymbols}" ]; then

docker/market_data/hist/load_alpaca_qat.sh

@@ -96,6 +96,7 @@ Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --name=${ContainerName}"
 Cmd="${Cmd} --rm"
 Cmd="${Cmd} --volume=${LogDir}:/logs"
+Cmd="${Cmd} --volume=${HOME}/.creds:/.creds"
 Cmd="${Cmd} ${DockerImage}"
 Cmd="${Cmd} -d ${date_to_load}"
 Cmd="${Cmd} -S ${Symbols}"

docker/market_data/recorder/md_recorder.sh

@@ -49,6 +49,7 @@ Cmd="${Cmd} --rm"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/.creds:/.creds"
 Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
 Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
 Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"

docker/market_data/recorder/md_recorder_monitor.sh

@@ -27,9 +27,6 @@ is_container_running() {
     fi
 }
 
-
-
-
 Instance=${1}
 if ! is_valid "${Instance}"; then
     usage
@@ -53,6 +50,7 @@ Cmd="${Cmd} --rm"
 Cmd="${Cmd} --network=host"
 Cmd="${Cmd} --pull=always"
 Cmd="${Cmd} --name=${ContainerName}"
+Cmd="${Cmd} -v /home/cvtt/.creds:/.creds"
 Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
 Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
 Cmd="${Cmd} ${Instance} ${AdminPort}"

release_version.txt

@@ -1 +1 @@
-1.9.2.F1,fix: prepare eqt simdata script
+1.9.4,minor - docker build script logging

research/prepare_crypto_simdata.sh

@@ -152,7 +152,7 @@ sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
 INSERT OR IGNORE INTO md_trades SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -168,7 +168,7 @@ COMMIT;
 BEGIN;
 INSERT OR IGNORE INTO md_trades
 SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,
@@ -189,7 +189,7 @@ sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${SourceDbFile}' AS source_db;
 BEGIN;
 INSERT OR IGNORE INTO md_quotes SELECT
-	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstmp,
+	datetime(exchange_ts_ns / 1000000000, 'unixepoch') || '.' ||  printf('%06d', (exchange_ts_ns % 1000000000) / 1000)  as tstamp,
 	time as tstamp_ns,
 	exchange_id,
 	instrument_id,

research/prepare_eqt_simdata.sh

@@ -168,7 +168,7 @@ for sym in ${Symbols[@]}; do
     sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${src_qat_db}' AS source_db;
 BEGIN;
-INSERT INTO md_trades SELECT
+INSERT OR IGNORE INTO md_trades SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -183,7 +183,7 @@ FROM source_db.md_trades;
 COMMIT;
 
 BEGIN;
-INSERT INTO md_quotes SELECT
+INSERT OR IGNORE INTO md_quotes SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -202,7 +202,7 @@ EOF
     sqlite3  ${ResultDbFile} <<EOF
 ATTACH '${src_hbar_db}' AS source_db;
 BEGIN;
-INSERT INTO md_1min_bars SELECT
+INSERT OR IGNORE INTO md_1min_bars SELECT
 	tstamp,
 	tstamp_ns,
 	exchange_id,
@@ -225,8 +225,6 @@ EOF
     eval ${Cmd}
 done
 
-exit
-
 Cmd="gzip ${ResultDbFile}"
 echo ${Cmd}
 eval ${Cmd}