From 25120230793dd2760119bc723bfa0260239e007c Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Sun, 19 Oct 2025 15:13:50 +0000 Subject: [PATCH] smarter crypto simdata - using DB_HOST_BACKUP --- VERSION | 2 +- build_ops.sh | 183 ++++++++++++++++++ .../sim_prepare/prepare_crypto_simdata.sh | 45 ++++- 3 files changed, 224 insertions(+), 6 deletions(-) create mode 100755 build_ops.sh diff --git a/VERSION b/VERSION index a6333e4..2bf1c1c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2.9 +2.3.1 diff --git a/build_ops.sh b/build_ops.sh new file mode 100755 index 0000000..8875e8a --- /dev/null +++ b/build_ops.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +# ---------------- Settings + +repo=git@cloud21.cvtt.vpn:/works/git/cvtt2/ops.git + +dist_root=/home/cvttdist/software/cvtt2 +dist_user=cvttdist +dist_host="cloud21.cvtt.vpn" +dist_ssh_port="22" + +dist_locations="cloud21.cvtt.vpn:22 hs01.cvtt.vpn:22" +version_file="VERSION" + +prj=ops +brnch=master +interactive=N + +# ---------------- Settings + +# ---------------- cmdline + +usage() { + echo "Usage: $0 [-b -i (interactive)" + exit 1 +} + +while getopts "b:i" opt; do + case ${opt} in + b ) + brnch=$OPTARG + ;; + i ) + interactive=Y + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done +# ---------------- cmdline + +confirm() { + if [ "${interactive}" == "Y" ]; then + echo "--------------------------------" + echo -n "Press to continue" && read + fi +} + + +if [ "${interactive}" == "Y" ]; then + echo -n "Enter project [${prj}]: " + read project + if [ "${project}" == "" ] + then + project=${prj} + fi +else + project=${prj} +fi + +# repo=${git_repo_arr[${project}]} +if [ -z ${repo} ]; then + echo "ERROR: Project repository for ${project} not found" + exit -1 +fi +echo "Project repo: ${repo}" + +if [ "${interactive}" == "Y" ]; then + echo -n "Enter branch to build release from [${brnch}]: " + read branch + if [ "${branch}" == "" ] + then + branch=${brnch} + fi +else + branch=${brnch} +fi + +tmp_dir=$(mktemp -d) +function cleanup { + cd ${HOME} + rm -rf ${tmp_dir} +} +trap cleanup EXIT + + +prj_dir="${tmp_dir}/${prj}" + +cmd_arr=() +Cmd="git clone ${repo} ${prj_dir}" +cmd_arr+=("${Cmd}") + +Cmd="cd ${prj_dir}" +cmd_arr+=("${Cmd}") + +if [ "${interactive}" == "Y" ]; then + echo "------------------------------------" + echo "The following commands will execute:" + echo "------------------------------------" + for cmd in "${cmd_arr[@]}" + do + echo ${cmd} + done +fi + +confirm + +for cmd in "${cmd_arr[@]}" +do + echo ${cmd} && eval ${cmd} +done + +Cmd="git checkout ${branch}" +echo ${Cmd} && eval ${Cmd} +if [ "${?}" != "0" ]; then + echo "ERROR: Branch ${branch} is not found" + cd ${HOME} && rm -rf ${tmp_dir} + exit -1 +fi + + +release_version=$(cat ${version_file} | awk -F',' '{print $1}') +whats_new=$(cat ${version_file} | awk -F',' '{print $2}') + + +echo "--------------------------------" +echo "Version file: ${version_file}" +echo "Release version: ${release_version}" + +confirm + +version_tag="v${release_version}" +version_comment="'${version_tag} ${project} ${branch} $(date +%Y-%m-%d)\n${whats_new}'" + +cmd_arr=() +Cmd="git tag -a ${version_tag} -m ${version_comment}" +cmd_arr+=("${Cmd}") + +Cmd="git push origin --tags" +cmd_arr+=("${Cmd}") + +Cmd="rm -rf .git" +cmd_arr+=("${Cmd}") + +SourceLoc=../${project} + +dist_path="${dist_root}/${project}/${release_version}" + +for dist_loc in ${dist_locations}; do + dhp=(${dist_loc//:/ }) + dist_host=${dhp[0]} + dist_port=${dhp[1]} + Cmd="rsync -avzh" + Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path}" + Cmd="${Cmd} && rsync\" -e \"ssh -p ${dist_ssh_port}\"" + Cmd="${Cmd} $SourceLoc ${dist_user}@${dist_host}:${dist_path}/" + cmd_arr+=("${Cmd}") +done + +if [ "${interactive}" == "Y" ]; then + echo "------------------------------------" + echo "The following commands will execute:" + echo "------------------------------------" + for cmd in "${cmd_arr[@]}" + do + echo ${cmd} + done +fi + +confirm + +for cmd in "${cmd_arr[@]}" +do + pwd && echo ${cmd} && eval ${cmd} +done + +echo "$0 Done ${project} ${release_version}" diff --git a/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh b/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh index 9b03015..e8657b7 100755 --- a/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh +++ b/scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh @@ -4,6 +4,7 @@ # SOURCE_HOST=cloud21.cvtt.vpn # SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto # DB_SOURCE=cloud28 +# DB_SOURCE_BACKUP=cloud29 # OUTPUT_DIR=/tmp # DATE=20250516 # RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/" @@ -23,22 +24,56 @@ mkdir -p ${OUTPUT_DIR} year=$(date -d ${DATE} +"%Y") month=$(date -d ${DATE} +"%m") -SourceDir="${SOURCE_ROOT_DIR}/${DB_SOURCE}/${year}/${month}" + + +# read the file prepare_crypto_simdata.sh +# implement the following logic. +# 1. check SourceFilePath on on SURCE_HOST for both DB_SOURCE and DB_SOURCE_BACKUP +# 2. if the file does not exist on either, exit like in line 47 +# 3. if file exist on only one of them, use ti to rsync +# 4. if the file exists on both, use the one that is larger in size + SourceFile="${DATE}.mktdata.db.gz" -SourceFilePath="${SourceDir}/${SourceFile}" +SelectedSourceHost="" +SelectedSourceFilePath="" +SelectedSourceSize=0 + +for db_source_host in ${DB_SOURCE} ${DB_SOURCE_BACKUP}; do + SourceDir="${SOURCE_ROOT_DIR}/${db_source_host}/${year}/${month}" + echo "SourceDir=${SourceDir}" + CandidatePath="${SourceDir}/${SourceFile}" + remote_stat_cmd="if [ -f '${CandidatePath}' ]; then stat -c %s '${CandidatePath}'; else exit 1; fi" + CandidateSize=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SOURCE_HOST} "${remote_stat_cmd}" 2>/dev/null) + if [ $? -eq 0 ] && [ -n "${CandidateSize}" ]; then + echo "Found ${SOURCE_HOST}:${CandidatePath} (${CandidateSize} bytes)" + if [ -z "${SelectedSourceHost}" ] || [ "${CandidateSize}" -gt "${SelectedSourceSize}" ]; then + SelectedSourceHost=${db_source_host} + SelectedSourceFilePath=${CandidatePath} + SelectedSourceSize=${CandidateSize} + fi + fi +done + +if [ -z "${SelectedSourceHost}" ]; then + echo "File ${SOURCE_HOST}:${SourceFile} NOT FOUND" + exit +fi + +echo "Using source ${SelectedSourceHost} with ${SelectedSourceFilePath} (${SelectedSourceSize} bytes)" Cmd="/usr/bin/rsync -ahv" Cmd+=" --mkpath" Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" -Cmd+=" ${SOURCE_HOST}:${SourceFilePath}" +Cmd+=" ${SOURCE_HOST}:${SelectedSourceFilePath}" Cmd+=" $OUTPUT_DIR/" echo ${Cmd} eval ${Cmd} if [ ! -f ${OUTPUT_DIR}/${SourceFile} ] ; then - echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND" - exit + echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND" + exit fi + Cmd="(cd ${OUTPUT_DIR} && gunzip -f *.db.gz)" echo ${Cmd} eval ${Cmd}