Compare commits

..

11 Commits

Author SHA1 Message Date
oleg d5e1f0a446 fix 2025-01-27 18:24:47 -05:00
oleg f639bd41f5 docker build imagages script fix 2025-01-27 18:15:30 -05:00
oleg a35bcf089e minor 2025-01-27 17:07:28 -05:00
oleg 5bf02ae67a release_version.txt -> VERSION 2025-01-27 16:41:22 -05:00
oleg 43c50befc4 docker to use CONFIG_SERVICE 2025-01-24 11:39:04 -05:00
oleg 48ecf7cca2 storage healthcheck - ignore known_hosts 2025-01-22 17:48:13 -05:00
oleg 5671ec5044 homestore ==> hs01 2025-01-13 21:54:30 -05:00
oleg a7e442b954 homestore ==> hs01 2025-01-13 20:04:52 -05:00
oleg 7e0c389536 minor - docker build script logging 2024-12-16 19:58:02 -05:00
oleg 058431a37c version only 2024-12-16 18:08:40 -05:00
oleg 9de3ed3ab9 creds to be taken from ~/.creds 2024-12-16 17:46:05 -05:00
23 changed files with 188 additions and 94 deletions
+1
View File
@@ -0,0 +1 @@
1.9.9.F2,docker build imagages script fix
+9 -2
View File
@@ -1,6 +1,6 @@
#!/bin/env bash
Registry=http://homestore.cvtt.vpn:5500
Registry=http://hs01.cvtt.vpn:5500
Catalog=${Registry}/v2/_catalog
@@ -11,5 +11,12 @@ jstr=$(curl -s -X GET ${Catalog})
echo "${jstr}" | jq -r '.repositories[]' | while read repo; do
TagListURL="${Registry}/v2/${repo}/tags/list"
# echo $repo
curl -s -X GET ${TagListURL}
Cmd="curl -s -X GET ${TagListURL}"
echo ${Cmd}
eval ${Cmd}
# TagList=$(eval ${Cmd})
# # echo ${TagList}
# echo "${TagList}" | jq -r '.tags[]' | while read tag; do
# echo "${Registry}/${repo}:${tag}"
# done
done
+6 -5
View File
@@ -14,9 +14,9 @@ dist_user=cvttdist
dist_host="cloud21.cvtt.vpn"
dist_ssh_port="22"
dist_locations="cloud21.cvtt.vpn:22 homestore.cvtt.vpn:22"
# ---------------- Settings
dist_locations="cloud21.cvtt.vpn:22 hs01.cvtt.vpn:22"
version_file="VERSION"
# ---------------- cmdline
prj=
brnch=master
@@ -145,11 +145,12 @@ if [ "${?}" != "0" ]; then
fi
release_version=$(cat release_version.txt | awk -F',' '{print $1}')
whats_new=$(cat release_version.txt | awk -F',' '{print $2}')
release_version=$(cat ${version_file} | awk -F',' '{print $1}')
whats_new=$(cat ${version_file} | awk -F',' '{print $2}')
echo "--------------------------------"
echo "Version file: ${version_file}"
echo "Release version: ${release_version}"
confirm
@@ -175,7 +176,7 @@ if [ "${project}" == "cvtt-rust" ]; then
release_dir=${prj_dir}/dist/${project}
cmd_arr+=("mkdir -p ${release_dir}")
cmd_arr+=("cp release_version.txt ${release_dir}")
cmd_arr+=("cp ${version_file} ${release_dir}")
jq_cmd="jq '.packages[] | select(.targets[].kind[] == \"bin\") | .name'"
apps=$(cargo metadata --no-deps --format-version 1 --manifest-path=${prj_dir}/Cargo.toml | eval $jq_cmd | uniq)
+82 -7
View File
@@ -5,6 +5,11 @@ usage() {
echo
exit 1
}
error_exit() {
echo "ERROR: ${1}"
exit 1
}
# --- Settings
RootDir=${1}
if [ -z "${RootDir}" ] ; then
@@ -37,6 +42,7 @@ build_docker_image() {
image_name=${2}
docker_dev_path=${3}
project=${4}
test_script=${5}
echo "Building ${title}..."
echo "ImageName=${image_name}"
@@ -66,7 +72,9 @@ build_docker_image() {
convert_symlink_to_dir ${project}
DockerDir=./docker_dev/${docker_dev_path}
echo "Building ${title}... on ${DockerDev}"
echo "================================================================================"
echo "Building ${title}... on ${DockerDir}"
echo "================================================================================"
Cmd="docker build"
Cmd+=" -t ${image_name}"
@@ -74,29 +82,93 @@ build_docker_image() {
Cmd+=" -t ${image_name}:${image_version}"
Cmd+=" -f ${DockerDir}/Dockerfile ${RootDir}"
echo ${Cmd}
eval ${Cmd} 1>&2 || exit
eval ${Cmd} 1>&2 || error_exit "Docker build failed"
if [ ! -z "${test_script}" ] && [ "${test_script}" != "null" ]; then
if [ ! -f ${DockerDir}/${test_script} ]; then
error_exit "Test script ${test_script} is missing"
fi
echo "Running test script ${test_script} for ${image_name}"
Cmd="docker cp ${DockerDir}/${test_script} ${image_name}:/"
echo ${Cmd}
eval ${Cmd} 1>&2 || error_exit "Test script copy failed"
Cmd="docker exec ${image_name} chmod +x /${test_script}"
echo ${Cmd}
eval ${Cmd} 1>&2 || error_exit "Test script chmod failed"
Cmd="docker run"
Cmd+=" --rm"
Cmd+=" -d"
Cmd+=" --name test-container"
Cmd+=" ${image_name}"
Cmd+=" /bin/bash -c /${TestScript}"
echo ${Cmd}
eval ${Cmd} 1>&2 || error_exit "Test script run failed"
echo "Removing test script ${test_script} from ${image_name}"
Cmd="docker exec ${image_name} rm /${test_script}"
echo ${Cmd}
eval ${Cmd} 1>&2 || error_exit "Test script remove failed"
fi
: <<'COMMENT'
-------------------------
Every single docker_dev project must have a test script
image_test.sh
that is copied to the image's root directory.
-------------------------
Cmd="docker run"
Cmd+=" --rm"
Cmd+=" -d"
Cmd+=" --name test-container"
Cmd+=" ${image_name}"
Cmd+=" /bin/bash -c /${TestScript}"
echo ${Cmd}
eval ${Cmd} 1>&2 || exit 1
# Get the exit status of the test script
docker wait test-container
# Check the exit status of the container
EXIT_STATUS=$?
# Analyze the exit status
if [ ${EXIT_STATUS} -ne 0 ]; then
echo "Tests Failed"
exit 1
fi
COMMENT
Cmd="docker tag"
Cmd+=" ${image_name}:latest"
Cmd+=" ${RegistryService}/${image_name}:latest"
echo ${Cmd}
eval ${Cmd} || exit
eval ${Cmd} || error_exit "Docker tag failed"
Cmd="docker tag"
Cmd+=" ${image_name}:${image_version}"
Cmd+=" ${RegistryService}/${image_name}:${image_version}"
echo ${Cmd}
eval ${Cmd} || exit
eval ${Cmd} || error_exit "Docker tag 2failed"
Cmd="docker push"
Cmd+=" ${RegistryService}/${image_name}:latest"
echo ${Cmd}
eval ${Cmd} || exit
eval ${Cmd} || error_exit "Docker push failed"
Cmd="docker push"
Cmd+=" ${RegistryService}/${image_name}:${image_version}"
echo ${Cmd}
eval ${Cmd} || exit
eval ${Cmd} || error_exit "Docker push 2 failed"
echo "Removing the copy of ${project} from ${RootDir}"
Cmd="rm -rf ${RootDir}/${project}"
@@ -164,6 +236,7 @@ get_latest_dist_version() {
echo "${Version}"
}
echo "------------------------------------- Starting ${0} ${*}"
echo RootDir=${RootDir}
cd ${RootDir}
@@ -186,13 +259,15 @@ for item in "${Jobs[@]}"; do
image_name=$(echo "$item" | jq -r '.value.image_name')
docker_dev_path=$(echo "$item" | jq -r '.value.docker_dev_path')
project=$(echo "$item" | jq -r '.value.based_on_project')
test_script=$(echo "$item" | jq -r '.value.test_script')
build_docker_image "$title" "$image_name" "$docker_dev_path" "$project"
build_docker_image "$title" "$image_name" "$docker_dev_path" "$project" "${test_script}"
done
Cmd="rm -rf ${RootDir}/docker_dev"
echo ${Cmd}
eval ${Cmd}
echo "Done ${0} ${*}"
#===============================================================================
#===============================================================================
+11 -9
View File
@@ -11,15 +11,17 @@ if [ -z ${date} ] ; then
fi
Cmd="docker run"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --name=crypto_md_day.${host}.${date}"
Cmd="${Cmd} --rm"
Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} -h ${host}"
Cmd="${Cmd} -d ${date}"
# Cmd="${Cmd} -s coinbase,bnbspot,bnbfut"
Cmd="${Cmd} -s coinbase,bnbspot"
Cmd+=" --pull=always"
Cmd+=" --network=host"
Cmd+=" --name=crypto_md_day.${host}.${date}"
Cmd+=" --rm"
Cmd+=" --volume=${HOME}/.creds:/.creds"
Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789"
Cmd+=" ${DockerImage}"
Cmd+=" -h ${host}"
Cmd+=" -d ${date}"
# Cmd+=" -s coinbase,bnbspot,bnbfut"
Cmd+=" -s coinbase,bnbspot"
echo $Cmd
eval $Cmd
+10 -8
View File
@@ -86,15 +86,17 @@ fi
echo "date_to_load=${date_to_load}"
Cmd="docker run"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --name=${ContainerName}"
Cmd="${Cmd} --rm"
Cmd="${Cmd} --volume=${LogDir}:/logs"
Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} -d ${date_to_load}"
Cmd+=" --pull=always"
Cmd+=" --network=host"
Cmd+=" --name=${ContainerName}"
Cmd+=" --rm"
Cmd+=" --volume=${LogDir}:/logs"
Cmd+=" --volume=${HOME}/.creds:/.creds"
Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789"
Cmd+=" ${DockerImage}"
Cmd+=" -d ${date_to_load}"
if [ -n "${NumSymbols}" ]; then
Cmd="${Cmd} -N ${NumSymbols}"
Cmd+=" -N ${NumSymbols}"
fi
echo $Cmd
+10 -8
View File
@@ -91,14 +91,16 @@ fi
echo "date_to_load=${date_to_load}"
Cmd="docker run"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --name=${ContainerName}"
Cmd="${Cmd} --rm"
Cmd="${Cmd} --volume=${LogDir}:/logs"
Cmd="${Cmd} ${DockerImage}"
Cmd="${Cmd} -d ${date_to_load}"
Cmd="${Cmd} -S ${Symbols}"
Cmd+=" --pull=always"
Cmd+=" --network=host"
Cmd+=" --name=${ContainerName}"
Cmd+=" --rm"
Cmd+=" --volume=${LogDir}:/logs"
Cmd+=" --volume=${HOME}/.creds:/.creds"
Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789"
Cmd+=" ${DockerImage}"
Cmd+=" -d ${date_to_load}"
Cmd+=" -S ${Symbols}"
echo $Cmd
eval $Cmd
+11 -9
View File
@@ -44,15 +44,17 @@ fi
Cmd="docker run"
Cmd="${Cmd} -d"
Cmd="${Cmd} --rm"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --name=${ContainerName}"
Cmd="${Cmd} -v /home/cvtt/prod/data:/app/data"
Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder:latest"
Cmd="${Cmd} ${job}"
Cmd+=" -d"
Cmd+=" --rm"
Cmd+=" --network=host"
Cmd+=" --pull=always"
Cmd+=" --name=${ContainerName}"
Cmd+=" -v /home/cvtt/.creds:/.creds"
Cmd+=" -v /home/cvtt/prod/data:/app/data"
Cmd+=" -v /home/cvtt/prod/logs:/logs"
Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789"
Cmd+=" cloud21.cvtt.vpn:5500/md_recorder:latest"
Cmd+=" ${job}"
echo ${Cmd}
eval ${Cmd}
@@ -27,9 +27,6 @@ is_container_running() {
fi
}
Instance=${1}
if ! is_valid "${Instance}"; then
usage
@@ -48,14 +45,16 @@ if is_container_running "$ContainerName"; then
fi
Cmd="docker run"
Cmd="${Cmd} -d"
Cmd="${Cmd} --rm"
Cmd="${Cmd} --network=host"
Cmd="${Cmd} --pull=always"
Cmd="${Cmd} --name=${ContainerName}"
Cmd="${Cmd} -v /home/cvtt/prod/logs:/logs"
Cmd="${Cmd} cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
Cmd="${Cmd} ${Instance} ${AdminPort}"
Cmd+=" -d"
Cmd+=" --rm"
Cmd+=" --network=host"
Cmd+=" --pull=always"
Cmd+=" --name=${ContainerName}"
Cmd+=" -v /home/cvtt/.creds:/.creds"
Cmd+=" -v /home/cvtt/prod/logs:/logs"
Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789"
Cmd+=" cloud21.cvtt.vpn:5500/md_recorder_monitor:latest"
Cmd+=" ${Instance} ${AdminPort}"
echo ${Cmd}
eval ${Cmd}
+1
View File
@@ -8,6 +8,7 @@ Cmd+=" --network=host"
Cmd+=" --name=crypto_exch_stats"
Cmd+=" --volume=${HOME}/prod/data:/app/data"
Cmd+=" --volume=${HOME}/prod/logs:/logs"
Cmd+=" -e CONFIG_SERVICE=cloud23.cvtt.vpn:6789"
Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest"
echo ${Cmd}
-1
View File
@@ -1 +0,0 @@
1.9.2.F3,fix: prepare eqt simdata script
+1 -1
View File
@@ -2,7 +2,7 @@
# by chatGPT
# ==========================================
# homestore.cvtt.vpn SETTINGS
# hs01.cvtt.vpn SETTINGS
# ==========================================
# Define source and destination base directories
+5 -5
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
usage() {
echo "Usage: $0 [-h <host (homestore*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
echo "Usage: $0 [-h <host (hs01*/cloud21)>] [-d <YYYYMMDD> (yesterday*)] [-s <source> (cvttdata/cloud21*)>]"
exit 1
}
@@ -20,7 +20,7 @@ is_valid() {
}
date=""
host=homestore
host=hs01
source=cloud21
while getopts ":h:d:s:" opt; do
@@ -51,7 +51,7 @@ fi
echo "$date $host $source"
valid_hosts=('homestore' 'cloud21')
valid_hosts=('hs01' 'cloud21')
if ! is_valid "${host}" "${valid_hosts[@]}" ; then
echo "Host '${host}' is not valid"
usage
@@ -67,8 +67,8 @@ if [ "${host}" == "cloud21" ] ; then
SourceHost=cloud21.cvtt.vpn
SourceUser=cvtt
SourceRootDir="/opt/store/cvtt/md_archive/crypto/${source}"
elif [ "${host}" == "homestore" ]; then
SourceHost=homestore.cvtt.vpn
elif [ "${host}" == "hs01" ]; then
SourceHost=hs01.cvtt.vpn
SourceUser=cvtt
SourceRootDir=/works/cvtt/md_archive/crypto/${source}
else
+5 -5
View File
@@ -2,7 +2,7 @@
usage() {
echo -n "Usage: $0"
echo -n " [-h <host (homestore*/cloud21)>]"
echo -n " [-h <host (hs01*/cloud21)>]"
echo -n " [-d <YYYYMMDD> (yesterday*)]"
echo -n " [-s <stocks comma separated>"
echo -n " [-t <target directory>"
@@ -19,7 +19,7 @@ fi
# ------------------ Settings
md_date=""
host=homestore
host=hs01
stocks=COIN,GBTC,SQ
TargetDir="/opt/jupyter_gpu/data/eqty_md"
@@ -81,7 +81,7 @@ fi
echo "$md_date $host $source"
valid_hosts=('homestore' 'cloud21')
valid_hosts=('hs01' 'cloud21')
if ! is_valid "${host}" "${valid_hosts[@]}" ; then
echo "Host '${host}' is not valid"
usage
@@ -108,8 +108,8 @@ if [ "${host}" == "cloud21" ] ; then
SourceHost=cloud21.cvtt.vpn
SourceUser=cvtt
SourceRootDir="/opt/store/cvtt/md_archive/equity/alpaca_md"
elif [ "${host}" == "homestore" ]; then
SourceHost=homestore.cvtt.vpn
elif [ "${host}" == "hs01" ]; then
SourceHost=hs01.cvtt.vpn
SourceUser=cvtt
SourceRootDir="/works/cvtt/md_archive/equity/alpaca_md"
else
+1 -1
View File
@@ -269,7 +269,7 @@ Cmd="gzip ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/"
echo ${Cmd}
eval ${Cmd}
+1 -1
View File
@@ -229,7 +229,7 @@ Cmd="gzip ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
Cmd="rsync -ahvv ${ResultDbFile}.gz cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
echo ${Cmd}
eval ${Cmd}
+16 -13
View File
@@ -19,13 +19,13 @@ get_user_hosts() {
local Domain=${2}
Cmd="curl -s http://cloud23.cvtt.vpn:6789/admin/cvtt_hosts"
Cmd="${Cmd} | ${HOME}/bin/hjson -j"
Cmd="${Cmd} | jq -r"
Cmd="${Cmd} --arg domain \"${Domain}\""
Cmd="${Cmd} --arg usr \"${User}\""
Cmd="${Cmd} '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
Cmd+=" | ${HOME}/bin/hjson -j"
Cmd+=" | jq -r"
Cmd+=" --arg domain \"${Domain}\""
Cmd+=" --arg usr \"${User}\""
Cmd+=" '.[\$domain] | to_entries[] | select(.value.users[] | contains(\$usr)) | .key'"
Cmd="${Cmd} | sed 's/\$/.${Domain}/'"
Cmd+=" | sed 's/\$/.${Domain}/'"
eval ${Cmd}
}
@@ -66,13 +66,16 @@ function storage_check() {
else
port=22
fi
Cmd="ssh -p ${port} $host"
Cmd="${Cmd} eval \"df -hTl"
Cmd="${Cmd} -x squashfs"
Cmd="${Cmd} -x tmpfs"
Cmd="${Cmd} -x vfat"
Cmd="${Cmd} -x devtmpfs"
Cmd="${Cmd} | grep -v Filesystem\""
Cmd="ssh -p ${port}"
Cmd+=" -o StrictHostKeyChecking=no"
Cmd+=" -o UserKnownHostsFile=/dev/null"
Cmd+=" $host"
Cmd+=" eval \"df -hTl"
Cmd+=" -x squashfs"
Cmd+=" -x tmpfs"
Cmd+=" -x vfat"
Cmd+=" -x devtmpfs"
Cmd+=" | grep -v Filesystem\""
IFS=$'\n' ; lines=$(eval ${Cmd})
for ln in $lines
@@ -25,8 +25,8 @@ run_checklist() {
declare -A Commands
Commands=(
["homestore:cloud21"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["homestore:cvttdata"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["hs01:cloud21"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["hs01:cvttdata"]="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["cloud21:cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cloud21/${yr}/${mn} | tail -5"
["cloud21:cvttdata"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/crypto/cvttdata/${yr}/${mn} | tail -5"
["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/crypto_md | tail -10'"
@@ -24,11 +24,11 @@ run_checklist() {
yr=$(date -d 'yesterday' '+%Y')
CheckSymbols="A/AAPL N/NVDA M/META"
declare -A Commands
Commands["homestore"]=""
Commands["hs01"]=""
for sym in ${CheckSymbols}; do
Commands["homestore"]+="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
Commands["hs01"]+="ssh cvtt@hs01.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
done
Commands["homestore"]+="echo"
Commands["hs01"]+="echo"
Commands["cloud21"]=""
for sym in ${CheckSymbols}; do
+1 -1
View File
@@ -2,7 +2,7 @@
Source=/home/cvtt/prod/archive/md_archive/
Targets=
Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/"
Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/"
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/"
+1 -1
View File
@@ -47,7 +47,7 @@ function rsync_load_version() {
if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
then
echo "Directory found..."
rsync_cmd="rsync -ahvv -e \"ssh -p ${port}\""
rsync_cmd="rsync -ahv -e \"ssh -p ${port}\""
rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}"
rsync_cmd="${rsync_cmd} ${LocalSoftwareDir}/${Project}/"
echo ${rsync_cmd}
+1 -1
View File
@@ -6,7 +6,7 @@ SUBDIRS += configs
FILES=
FILES += release_version.txt
FILES += VERSION
all: install