Compare commits

..

19 Commits

Author SHA1 Message Date
oleg cade1cd055 simdata prepare 2024-12-03 20:11:36 -05:00
oleg ed44757900 fix 2024-12-03 13:55:39 -05:00
oleg 340283c0a9 fix: syntax 2024-12-03 13:54:56 -05:00
oleg f4365d32e5 fix: no boolean for jq, it considers "false" as missing (null) 2024-12-03 13:41:32 -05:00
oleg 1fe5edcd74 build script fix 2024-12-03 12:37:48 -05:00
oleg 8510226599 host checking improving 2024-12-03 12:32:40 -05:00
oleg 10105ad639 prepare equity simulation data 2024-11-27 20:46:10 -05:00
oleg 9b2bda6257 crypto_exch_stats docker run 2024-11-06 10:46:41 -05:00
oleg 80a6ffeb18 docker_image_builder fix 2024-11-02 23:36:26 -04:00
oleg e7c02587c2 progress docker image builder 2024-11-02 21:22:23 -04:00
oleg cc2c07741a fix 2024-11-02 17:54:16 -04:00
oleg 36d6914c8c initial version of docker_image_builder.sh 2024-11-02 16:42:06 -04:00
oleg 86a58462bf initial (not working) version of docker_image_builder.sh 2024-11-02 13:23:51 -04:00
oleg f0a639940b fix 2024-11-01 17:30:59 -04:00
oleg 3fec261a46 changes to build scripts 2024-11-01 17:26:45 -04:00
oleg 618a7eb0cf md eqty checklist to use new structure 2024-11-01 13:15:34 -04:00
oleg 512f5ce14d build for cvtt-rust uses cargo install 2024-10-31 23:35:41 -04:00
oleg 9a3215524e build for cvtt-rust 2024-10-31 19:50:56 -04:00
oleg 84a8c1cb31 case insensitive to_check parameter host and services avalability check 2024-10-31 17:37:19 -04:00
10 changed files with 479 additions and 38 deletions
+45 -11
View File
@@ -3,9 +3,11 @@
# ---------------- Settings
declare -A git_repo_arr
git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvtt2/cvttpy.git
git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/cvtt2/ops.git
git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/cvtt2/research.git
git_repo_arr[cvtt-rust]=git@cloud21.cvtt.vpn:/opt/store/git/cvtt2/cvtt-rust.git
git_repo_arr[docker_dev]=git@cloud21.cvtt.vpn:/opt/store/git/cvtt2/docker_dev.git
dist_root=/home/cvttdist/software/cvtt2
dist_user=cvttdist
@@ -48,13 +50,29 @@ while getopts ":p:b:i" opt; do
done
# ---------------- cmdline
function confirm {
confirm() {
if [ "${interactive}" == "Y" ]; then
echo "--------------------------------"
echo -n "Press <Enter> to continue" && read
fi
}
rust_binaries() {
res=()
for binname in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name'); do
res+=("${binname}")
done
echo "${res[@]}"
}
rust_libraries() {
res=()
for libname in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].targets[] | select(.kind | index("lib")) | .name'); do
res+=("lib${libname}.rlib")
done
echo "${res[@]}"
}
if [ "${interactive}" == "Y" ]; then
echo -n "Enter project [${prj}]: "
read project
@@ -149,6 +167,26 @@ cmd_arr+=("${Cmd}")
Cmd="rm -rf .git"
cmd_arr+=("${Cmd}")
SourceLoc=../${project}
if [ "${project}" == "cvtt-rust" ]; then
cmd_arr+=("cd ${prj_dir}")
release_dir=${prj_dir}/dist/${project}
cmd_arr+=("mkdir -p ${release_dir}")
cmd_arr+=("cp release_version.txt ${release_dir}")
jq_cmd="jq '.packages[] | select(.targets[].kind[] == \"bin\") | .name'"
apps=$(cargo metadata --no-deps --format-version 1 --manifest-path=${prj_dir}/Cargo.toml | eval $jq_cmd | uniq)
for app in ${apps[@]}; do
app=${app//\"/} # remove quotes
cmd_arr+=("cargo install --root ${release_dir} --path ${prj_dir}/apps/${app}")
done
SourceLoc=${release_dir}
fi
dist_path="${dist_root}/${project}/${release_version}"
for dist_loc in ${dist_locations}; do
@@ -156,9 +194,9 @@ for dist_loc in ${dist_locations}; do
dist_host=${dhp[0]}
dist_port=${dhp[1]}
Cmd="rsync -avzh"
Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path} && rsync\""
Cmd="${Cmd} -e \"ssh -p ${dist_ssh_port}\""
Cmd="${Cmd} ../${project} ${dist_user}@${dist_host}:${dist_path}/"
Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path}"
Cmd="${Cmd} && rsync\" -e \"ssh -p ${dist_ssh_port}\""
Cmd="${Cmd} $SourceLoc ${dist_user}@${dist_host}:${dist_path}/"
cmd_arr+=("${Cmd}")
done
@@ -179,8 +217,4 @@ do
pwd && echo ${cmd} && eval ${cmd}
done
echo "PROJECT ${project} **** P U L L N E W T A G S F R O M o r i g i n ****"
echo "----"
echo "(cd ${project}; git pull --tags; git pushall)"
echo "----"
echo "$0 Done ${project} ${release_version}"
+198
View File
@@ -0,0 +1,198 @@
#!/bin/bash
usage() {
echo -n "Usage: ${0} <RootDir>"
echo
exit 1
}
# --- Settings
RootDir=${1}
if [ -z "${RootDir}" ] ; then
usage
fi
RegistryService=cloud21.cvtt.vpn:5500
RegistryProtocol=http
ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/docker_images
ReleaseHost="cloud21.cvtt.vpn"
ReleasePort="22"
ReleaseUser="cvttdist"
ReleaseDir="/home/cvttdist/software/cvtt2"
DIMAGES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
SetVersionScript=${HOME}/prod/set_version.sh
# --------------- D E B U G
mkdir -p ${RootDir}
build_docker_image() {
title=${1}
image_name=${2}
docker_dev_path=${3}
project=${4}
echo "Building ${title}..."
echo "ImageName=${image_name}"
echo "DockerDevPath=${docker_dev_path}"
echo "BasedOnDist=${project}"
# Get Latest DIST version
echo "Checking for latest version of ${project} on ${ReleaseUser}@${ReleaseHost}:${ReleaseDir}"
image_version=$(get_latest_dist_version ${project})
if [ -z "${image_version}" ]; then
echo "No latest version found for ${project}"
return
fi
echo "Latest version is ${image_version}"
if image_exists "${image_name}" "${image_version}"; then
echo "Image $image_name:$image_version exists in the registry. Building is not required."
return
fi
echo "Image $image_name:$image_version does not exist in the registry."
echo "getting lastest version of ${project} on ${ReleaseUser}@${ReleaseHost}:${ReleaseDir}"
Cmd="${SetVersionScript} ${project} ${image_version}"
echo ${Cmd}
eval ${Cmd}
convert_symlink_to_dir ${project}
DockerDir=./docker_dev/${docker_dev_path}
echo "Building ${title}... on ${DockerDev}"
Cmd="docker build"
Cmd+=" -t ${image_name}"
Cmd+=" -t ${image_name}:latest"
Cmd+=" -t ${image_name}:${image_version}"
Cmd+=" -f ${DockerDir}/Dockerfile ${RootDir}"
echo ${Cmd}
eval ${Cmd} 1>&2 || exit
Cmd="docker tag"
Cmd+=" ${image_name}:latest"
Cmd+=" ${RegistryService}/${image_name}:latest"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker tag"
Cmd+=" ${image_name}:${image_version}"
Cmd+=" ${RegistryService}/${image_name}:${image_version}"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker push"
Cmd+=" ${RegistryService}/${image_name}:latest"
echo ${Cmd}
eval ${Cmd} || exit
Cmd="docker push"
Cmd+=" ${RegistryService}/${image_name}:${image_version}"
echo ${Cmd}
eval ${Cmd} || exit
echo "Removing the copy of ${project} from ${RootDir}"
Cmd="rm -rf ${RootDir}/${project}"
echo ${Cmd}
eval ${Cmd}
print_all_reg_images ${project}
}
convert_symlink_to_dir() {
project=${1}
if [ -L ${RootDir}/${project} ]; then
echo "Converting symlink ${RootDir}/${project} to directory"
Cmd="mv ${RootDir}/${project} ${RootDir}/${project}.symlink"
echo ${Cmd} && eval ${Cmd}
src_path=$(readlink -f ${RootDir}/${project}.symlink)
Cmd="cp -r ${src_path} ${RootDir}/${project}"
echo ${Cmd} && eval ${Cmd}
fi
if [ -d ${project} ]; then
echo "${RootDir}/${project} is a Directory"
fi
Cmd="rm ${RootDir}/${project}.symlink"
echo ${Cmd} && eval ${Cmd}
}
image_exists() {
image_name=${1}
version=${2}
response=$(curl -s -o /dev/null -w "%{http_code}" "${RegistryProtocol}://${RegistryService}/v2/${image_name}/manifests/${version}" -H "Accept: application/vnd.docker.distribution.manifest.v2+json")
if [ "$response" -eq 200 ]; then
return 0
else
return 1
fi
}
print_all_reg_images() {
project=${1}
repositories=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/_catalog" | jq -r '.repositories[]')
for repo in $repositories; do
# Fetch all tags for the repository
tags=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/$repo/tags/list" | jq -r '.tags[]')
# List each tag
for tag in $tags; do
if [ "${project}" == "" ];then
echo "$repo:$tag"
else
echo "$repo:$tag" | grep ${project}
fi
done
done
}
get_latest_dist_version() {
project=${1}
Version=$(ssh -q -p ${ReleasePort} ${ReleaseUser}@${ReleaseHost} "ls -tr ${ReleaseDir}/${project} | tail -1" )
echo "${Version}"
}
echo RootDir=${RootDir}
cd ${RootDir}
echo "Getting latest version of docker_dev"
Cmd="${SetVersionScript} docker_dev latest"
echo ${Cmd}
eval ${Cmd}
convert_symlink_to_dir docker_dev
Jobs=()
while IFS= read -r item; do
Jobs+=("$item")
done < <(echo "${DIMAGES_CONFIG}" | jq -c 'to_entries[]')
for item in "${Jobs[@]}"; do
# Extract key (name) and values
title=$(echo "$item" | jq -r '.key')
image_name=$(echo "$item" | jq -r '.value.image_name')
docker_dev_path=$(echo "$item" | jq -r '.value.docker_dev_path')
project=$(echo "$item" | jq -r '.value.based_on_project')
build_docker_image "$title" "$image_name" "$docker_dev_path" "$project"
done
Cmd="rm -rf ${RootDir}/docker_dev"
echo ${Cmd}
eval ${Cmd}
#===============================================================================
#===============================================================================
+33
View File
@@ -0,0 +1,33 @@
#!/bin/bash
usage() {
echo -n "Usage: ${0} <grep_filter>"
echo
exit 1
}
# --- Settings
RegistryService=cloud21.cvtt.vpn:5500
RegistryProtocol=http
print_all_reg_images() {
project=${1}
repositories=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/_catalog" | jq -r '.repositories[]')
for repo in $repositories; do
# Fetch all tags for the repository
tags=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/$repo/tags/list" | jq -r '.tags[]')
# List each tag
echo "REPO: $repo"
for tag in $tags; do
if [ "${project}" == "" ];then
echo "$repo:$tag"
else
echo "$repo:$tag" | grep ${project}
fi
done
done
}
print_all_reg_images ${1}
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
Cmd="docker run"
Cmd+=" -d"
Cmd+=" --rm"
Cmd+=" --pull=always"
Cmd+=" --network=host"
Cmd+=" --name=crypto_exch_stats"
Cmd+=" --volume=${HOME}/prod/data:/app/data"
Cmd+=" --volume=${HOME}/prod/logs:/logs"
Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest"
echo ${Cmd}
eval ${Cmd}
# Cmd+=" cloud21.cvtt.vpn:5500/relative_liquidity:latest"
+1 -1
View File
@@ -1 +1 @@
1.7.7.fx1,services avalability check
1.8.9,prepare sim
+151
View File
@@ -0,0 +1,151 @@
#!/bin/bash
usage() {
echo "Usage: $0 -S <symbols> -d <YYYYMMDD Date> [-O <output dir (./) >]"
exit 1
}
# --------------------- Settings
SourceHost=cloud21.cvtt.vpn
SourceRootDir=/opt/store/cvtt/md_archive/equity/alpaca_md
# --------------------- Settings
is_business_day() {
dt=${1}
date=$(date -d "${dt}" +"%Y-%m-%d")
CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
URL="${CalendarURL}&start=${date}&end=${date}"
open_time=$(curl -s "${URL}" | jq '.[] | .open_time')
if [ -n "${open_time}" ]; then
return 0
else
return 1
fi
}
export -f is_business_day
while getopts ":d:S:O:" opt; do
case ${opt} in
d )
Date=$OPTARG
;;
S )
SymList=$OPTARG
;;
O )
OutputDir=$OPTARG
;;
\? )
echo "Invalid option: -$OPTARG" >&2
usage
;;
: )
echo "Option -$OPTARG requires an argument." >&2
usage
;;
esac
done
if [ -z ${SymList} ] ; then
echo "symbols are not specified"
usage
fi
if [ -z ${Date} ] ; then
echo "date is not specified"
usage
fi
if [ -z ${OutputDir} ] ; then
OutputDir=.
fi
mkdir -p ${OutputDir}
if ! is_business_day ${Date}; then
echo "${Date} is not business day"
usage
fi
OLD_IFS=${IFS}
IFS=","
read -ra Symbols <<< "${SymList}"
IFS=${OLD_IFS}
echo "Date=${Date} Symbols=${Symbols[@]} OutputDir=${OutputDir}"
echo Getting data from ${DataHost} ...
year=$(date -d ${Date} +"%Y")
for sym in ${Symbols[@]}; do
inst_id="STOCK-${sym}"
capital=${sym:0:1}
SourceDir="${SourceRootDir}/${year}/${capital}/${sym}"
SourceHbarFile="${SourceDir}/${Date}.${sym}.alpaca_1m_bars.db.gz"
SourceQatFile="${SourceDir}/${Date}.${sym}.alpaca_qat.db.gz"
for src_file in ${SourceHbarFile} ${SourceQatFile}; do
Cmd="rsync -ahv"
Cmd+=" ${SourceHost}:${src_file}"
Cmd+=" $OutputDir/"
echo ${Cmd}
eval ${Cmd}
done
done
Cmd="(cd ${OutputDir} && gunzip *.db.gz)"
echo ${Cmd}
eval ${Cmd}
ResultDbFile="${OutputDir}/${Date}.alpaca_sim_md.db"
echo "Creating Result Database File ${ResultDbFile}"
sym=${Symbols[0]}
for db_file in ${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db ${OutputDir}/${Date}.${sym}.alpaca_qat.db ; do
Cmd="sqlite3"
Cmd+=" ${db_file} .schema"
Cmd+=" | sqlite3 ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}
done
set -f # not to expand *
for sym in ${Symbols[@]}; do
src_hbar_db=${OutputDir}/${Date}.${sym}.alpaca_1m_bars.db
src_qat_db=${OutputDir}/${Date}.${sym}.alpaca_qat.db
for srcdb in ${src_hbar_db} ${src_qat_db} ; do
tables=$(sqlite3 ${srcdb} .tables)
for table in $(sqlite3 ${srcdb} .tables) ; do
echo "Loading from ${srcdb}.${table} ..."
sqlite3 ${ResultDbFile} <<EOF
ATTACH '${srcdb}' AS source_db;
BEGIN;
INSERT INTO ${table} SELECT * FROM source_db.${table};
COMMIT;
DETACH source_db;
EOF
done
Cmd="rm ${srcdb}"
echo ${Cmd}
eval ${Cmd}
done
done
Cmd="gzip ${ResultDbFile}"
echo ${Cmd}
eval ${Cmd}
Cmd="rsync -ahvv ${ResultDbFile}.gz homestore.cvtt.vpn:/works/cvtt/md_archive/equity/alpaca_md/sim/"
echo ${Cmd}
eval ${Cmd}
Cmd="rsync -ahvv ${ResultDbFile}.gz cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity/alpaca_md/sim/"
echo ${Cmd}
eval ${Cmd}
Cmd="rm ${ResultDbFile}.gz"
echo ${Cmd}
eval ${Cmd}
echo Done $0 ${*}
+14 -9
View File
@@ -5,11 +5,11 @@
# "users": ["oleg"],
# "type": "cloud",
# "ssh_port": 7822,
# # "to_check": "No"
# "to_check": "false"
# "timeout_sec": 5
# },
RootDir="${HOME}/prod"
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
AlertChannel=Alerts-CVTT
Sender=${RootDir}/ops/utils/send_mmost.sh
@@ -46,6 +46,8 @@ function host_alert() {
User=oleg
Hosts=()
DEFAULT_SSH_PORT=22
DEFAULT_TIMEOUT=5
DEFAULT_TO_CHECK="true"
Domains=("${Domains[@]}" "$(get_domains)")
for Domain in ${Domains[@]} ; do
@@ -55,19 +57,22 @@ done
for Host in ${Hosts[@]} ; do
host=$(echo $Host | cut -d'.' -f1)
Domain=$(echo $Host | cut -d'.' -f2-)
# echo "Host=$Host host=$host Domain=$Domain"
# Get SSH port for the host, or use default if not specified
ToCheck=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].to_check // "Yes"')
if [ "${ToCheck}" == "No" ]; then
continue
fi
PortSSH=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].ssh_port // '"$DEFAULT_SSH_PORT"'')
Timeout=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].timeout_sec // '"$DEFAULT_TIMEOUT"'')
ToCheck=$(echo "$HOSTS_CONFIG" | jq -r --arg domain "$Domain" --arg host "$host" '.[$domain][$host].to_check // '"$DEFAULT_TO_CHECK"'')
to_check="${ToCheck^^}"
if [ "${to_check}" == "TRUE" -o "${to_check}" == "YES" -o "${to_check}" == "Y" -o "${to_check}" == "T" ] ; then
echo "Checking host: $Host on port $PortSSH"
else
continue
fi
# Use nc to check if the specified port is open
if ! nc -z -w5 "$Host" "$PortSSH"; then
if ! nc -z -w ${Timeout} "$Host" "$PortSSH"; then
echo "Host $Host is not available on port $PortSSH"
host_alert "Host $Host is not available on port $PortSSH"
fi
+1 -1
View File
@@ -40,7 +40,7 @@ mapfile -t SvcNames < <(echo ${SERVICES_CONFIG} | jq -r '. | keys[]')
DEFAULT_TO_CHECK=Yes
for SvcName in "${SvcNames[@]}" ; do
ToCheck=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].to_check // "Yes"')
if [ "${ToCheck}" == "No" ]; then
if [ "${ToCheck^^}" == "NO" ]; then
continue
fi
Host=$(echo "$SERVICES_CONFIG" | jq -r --arg svcname "$SvcName" '.[$svcname].host ')
+15 -11
View File
@@ -7,7 +7,6 @@ usage() {
}
RootDir="${HOME}/prod"
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
export PYTHONPATH=${RootDir}
@@ -15,28 +14,33 @@ StatusChannel="MD-Status"
Sender=${RootDir}/ops/utils/send_mmost.sh
# ----- For DEBUGGING
# Sender=cat
# StatusChannel=
# RootDir=/home/oleg/develop/cvtt2 ###### D E B U G
# Sender=cat #### D E B U G
# StatusChannel= #### D E B U G
run_checklist() {
yr=$(date -d 'yesterday' '+%Y')
CheckSymbols="A/AAPL N/NVDA M/META"
declare -A Commands
Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
Commands["homestore"]+="; echo"
Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
Commands["homestore"]=""
for sym in ${CheckSymbols}; do
Commands["homestore"]+="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
done
Commands["homestore"]+="echo"
Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
Commands["cloud21"]+="; echo"
Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
Commands["cloud21"]=""
for sym in ${CheckSymbols}; do
Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
done
Commands["cloud21"]+="echo"
Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
Cmd=${Commands[${name}]}
echo "------- ${name}"
Cmd=${Commands[${name}]}
echo ${Cmd}
eval ${Cmd}
done
+4 -4
View File
@@ -4,7 +4,7 @@
# ----- Settings
LocalSoftwareDir=${HOME}/software/cvtt2
ProdDir=${HOME}/prod
ProdDir=$(pwd)
ReleaseHosts=("cloud21.cvtt.vpn")
ReleasePorts=("22")
@@ -77,9 +77,9 @@ rsync_load_version
Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
Cmd="cd ${ProdDir}"
Cmd="${Cmd} && rm -rf ${Project}"
Cmd="${Cmd} && ln -snf ${Location} ${Project}"
Cmd+=" && rm -rf ${Project}"
Cmd+=" && ln -snf ${Location} ${Project}"
echo ${Cmd} && eval ${Cmd}
echo "Done: $0 $*"
echo "Done: $0 ${*}"