smarter crypto simdata - using DB_HOST_BACKUP
This commit is contained in:
Cryptoval2
parent
7f05a919d3
commit
2512023079
183
build_ops.sh
Executable file
183
build_ops.sh
Executable file
@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# ---------------- Settings
|
||||
|
||||
repo=git@cloud21.cvtt.vpn:/works/git/cvtt2/ops.git
|
||||
|
||||
dist_root=/home/cvttdist/software/cvtt2
|
||||
dist_user=cvttdist
|
||||
dist_host="cloud21.cvtt.vpn"
|
||||
dist_ssh_port="22"
|
||||
|
||||
dist_locations="cloud21.cvtt.vpn:22 hs01.cvtt.vpn:22"
|
||||
version_file="VERSION"
|
||||
|
||||
prj=ops
|
||||
brnch=master
|
||||
interactive=N
|
||||
|
||||
# ---------------- Settings
|
||||
|
||||
# ---------------- cmdline
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-b <branch (master)> -i (interactive)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
while getopts "b:i" opt; do
|
||||
case ${opt} in
|
||||
b )
|
||||
brnch=$OPTARG
|
||||
;;
|
||||
i )
|
||||
interactive=Y
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
# ---------------- cmdline
|
||||
|
||||
confirm() {
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo "--------------------------------"
|
||||
echo -n "Press <Enter> to continue" && read
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo -n "Enter project [${prj}]: "
|
||||
read project
|
||||
if [ "${project}" == "" ]
|
||||
then
|
||||
project=${prj}
|
||||
fi
|
||||
else
|
||||
project=${prj}
|
||||
fi
|
||||
|
||||
# repo=${git_repo_arr[${project}]}
|
||||
if [ -z ${repo} ]; then
|
||||
echo "ERROR: Project repository for ${project} not found"
|
||||
exit -1
|
||||
fi
|
||||
echo "Project repo: ${repo}"
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo -n "Enter branch to build release from [${brnch}]: "
|
||||
read branch
|
||||
if [ "${branch}" == "" ]
|
||||
then
|
||||
branch=${brnch}
|
||||
fi
|
||||
else
|
||||
branch=${brnch}
|
||||
fi
|
||||
|
||||
tmp_dir=$(mktemp -d)
|
||||
function cleanup {
|
||||
cd ${HOME}
|
||||
rm -rf ${tmp_dir}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
prj_dir="${tmp_dir}/${prj}"
|
||||
|
||||
cmd_arr=()
|
||||
Cmd="git clone ${repo} ${prj_dir}"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
Cmd="cd ${prj_dir}"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo "------------------------------------"
|
||||
echo "The following commands will execute:"
|
||||
echo "------------------------------------"
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
echo ${cmd}
|
||||
done
|
||||
fi
|
||||
|
||||
confirm
|
||||
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
echo ${cmd} && eval ${cmd}
|
||||
done
|
||||
|
||||
Cmd="git checkout ${branch}"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
if [ "${?}" != "0" ]; then
|
||||
echo "ERROR: Branch ${branch} is not found"
|
||||
cd ${HOME} && rm -rf ${tmp_dir}
|
||||
exit -1
|
||||
fi
|
||||
|
||||
|
||||
release_version=$(cat ${version_file} | awk -F',' '{print $1}')
|
||||
whats_new=$(cat ${version_file} | awk -F',' '{print $2}')
|
||||
|
||||
|
||||
echo "--------------------------------"
|
||||
echo "Version file: ${version_file}"
|
||||
echo "Release version: ${release_version}"
|
||||
|
||||
confirm
|
||||
|
||||
version_tag="v${release_version}"
|
||||
version_comment="'${version_tag} ${project} ${branch} $(date +%Y-%m-%d)\n${whats_new}'"
|
||||
|
||||
cmd_arr=()
|
||||
Cmd="git tag -a ${version_tag} -m ${version_comment}"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
Cmd="git push origin --tags"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
Cmd="rm -rf .git"
|
||||
cmd_arr+=("${Cmd}")
|
||||
|
||||
SourceLoc=../${project}
|
||||
|
||||
dist_path="${dist_root}/${project}/${release_version}"
|
||||
|
||||
for dist_loc in ${dist_locations}; do
|
||||
dhp=(${dist_loc//:/ })
|
||||
dist_host=${dhp[0]}
|
||||
dist_port=${dhp[1]}
|
||||
Cmd="rsync -avzh"
|
||||
Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path}"
|
||||
Cmd="${Cmd} && rsync\" -e \"ssh -p ${dist_ssh_port}\""
|
||||
Cmd="${Cmd} $SourceLoc ${dist_user}@${dist_host}:${dist_path}/"
|
||||
cmd_arr+=("${Cmd}")
|
||||
done
|
||||
|
||||
if [ "${interactive}" == "Y" ]; then
|
||||
echo "------------------------------------"
|
||||
echo "The following commands will execute:"
|
||||
echo "------------------------------------"
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
echo ${cmd}
|
||||
done
|
||||
fi
|
||||
|
||||
confirm
|
||||
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
pwd && echo ${cmd} && eval ${cmd}
|
||||
done
|
||||
|
||||
echo "$0 Done ${project} ${release_version}"
|
||||
@ -4,6 +4,7 @@
|
||||
# SOURCE_HOST=cloud21.cvtt.vpn
|
||||
# SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto
|
||||
# DB_SOURCE=cloud28
|
||||
# DB_SOURCE_BACKUP=cloud29
|
||||
# OUTPUT_DIR=/tmp
|
||||
# DATE=20250516
|
||||
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
|
||||
@ -23,22 +24,56 @@ mkdir -p ${OUTPUT_DIR}
|
||||
|
||||
year=$(date -d ${DATE} +"%Y")
|
||||
month=$(date -d ${DATE} +"%m")
|
||||
SourceDir="${SOURCE_ROOT_DIR}/${DB_SOURCE}/${year}/${month}"
|
||||
|
||||
|
||||
# read the file prepare_crypto_simdata.sh
|
||||
# implement the following logic.
|
||||
# 1. check SourceFilePath on on SURCE_HOST for both DB_SOURCE and DB_SOURCE_BACKUP
|
||||
# 2. if the file does not exist on either, exit like in line 47
|
||||
# 3. if file exist on only one of them, use ti to rsync
|
||||
# 4. if the file exists on both, use the one that is larger in size
|
||||
|
||||
SourceFile="${DATE}.mktdata.db.gz"
|
||||
SourceFilePath="${SourceDir}/${SourceFile}"
|
||||
SelectedSourceHost=""
|
||||
SelectedSourceFilePath=""
|
||||
SelectedSourceSize=0
|
||||
|
||||
for db_source_host in ${DB_SOURCE} ${DB_SOURCE_BACKUP}; do
|
||||
SourceDir="${SOURCE_ROOT_DIR}/${db_source_host}/${year}/${month}"
|
||||
echo "SourceDir=${SourceDir}"
|
||||
CandidatePath="${SourceDir}/${SourceFile}"
|
||||
remote_stat_cmd="if [ -f '${CandidatePath}' ]; then stat -c %s '${CandidatePath}'; else exit 1; fi"
|
||||
CandidateSize=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SOURCE_HOST} "${remote_stat_cmd}" 2>/dev/null)
|
||||
if [ $? -eq 0 ] && [ -n "${CandidateSize}" ]; then
|
||||
echo "Found ${SOURCE_HOST}:${CandidatePath} (${CandidateSize} bytes)"
|
||||
if [ -z "${SelectedSourceHost}" ] || [ "${CandidateSize}" -gt "${SelectedSourceSize}" ]; then
|
||||
SelectedSourceHost=${db_source_host}
|
||||
SelectedSourceFilePath=${CandidatePath}
|
||||
SelectedSourceSize=${CandidateSize}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "${SelectedSourceHost}" ]; then
|
||||
echo "File ${SOURCE_HOST}:${SourceFile} NOT FOUND"
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Using source ${SelectedSourceHost} with ${SelectedSourceFilePath} (${SelectedSourceSize} bytes)"
|
||||
|
||||
Cmd="/usr/bin/rsync -ahv"
|
||||
Cmd+=" --mkpath"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${SOURCE_HOST}:${SourceFilePath}"
|
||||
Cmd+=" ${SOURCE_HOST}:${SelectedSourceFilePath}"
|
||||
Cmd+=" $OUTPUT_DIR/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
if [ ! -f ${OUTPUT_DIR}/${SourceFile} ] ; then
|
||||
echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND"
|
||||
exit
|
||||
echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND"
|
||||
exit
|
||||
fi
|
||||
|
||||
Cmd="(cd ${OUTPUT_DIR} && gunzip -f *.db.gz)"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user