archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

smarter crypto simdata - using DB_HOST_BACKUP

Browse Source
This commit is contained in:
Cryptoval2
2025-10-19 15:13:50 +00:00
parent 7f05a919d3
commit 2512023079
3 changed files with 224 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh
+40 -5
View File
@@ -4,6 +4,7 @@
# SOURCE_HOST=cloud21.cvtt.vpn
# SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto
# DB_SOURCE=cloud28
# DB_SOURCE_BACKUP=cloud29
# OUTPUT_DIR=/tmp
# DATE=20250516
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
@@ -23,22 +24,56 @@ mkdir -p ${OUTPUT_DIR}
year=$(date -d ${DATE} +"%Y")
month=$(date -d ${DATE} +"%m")
SourceDir="${SOURCE_ROOT_DIR}/${DB_SOURCE}/${year}/${month}"
# read the file prepare_crypto_simdata.sh
# implement the following logic.
# 1. check SourceFilePath on on SURCE_HOST for both DB_SOURCE and DB_SOURCE_BACKUP
# 2. if the file does not exist on either, exit like in line 47
# 3. if file exist on only one of them, use ti to rsync
# 4. if the file exists on both, use the one that is larger in size
SourceFile="${DATE}.mktdata.db.gz"
SourceFilePath="${SourceDir}/${SourceFile}"
SelectedSourceHost=""
SelectedSourceFilePath=""
SelectedSourceSize=0
for db_source_host in ${DB_SOURCE} ${DB_SOURCE_BACKUP}; do
SourceDir="${SOURCE_ROOT_DIR}/${db_source_host}/${year}/${month}"
echo "SourceDir=${SourceDir}"
CandidatePath="${SourceDir}/${SourceFile}"
remote_stat_cmd="if [ -f '${CandidatePath}' ]; then stat -c %s '${CandidatePath}'; else exit 1; fi"
CandidateSize=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SOURCE_HOST} "${remote_stat_cmd}" 2>/dev/null)
if [ $? -eq 0 ] && [ -n "${CandidateSize}" ]; then
echo "Found ${SOURCE_HOST}:${CandidatePath} (${CandidateSize} bytes)"
if [ -z "${SelectedSourceHost}" ] || [ "${CandidateSize}" -gt "${SelectedSourceSize}" ]; then
SelectedSourceHost=${db_source_host}
SelectedSourceFilePath=${CandidatePath}
SelectedSourceSize=${CandidateSize}
fi
fi
done
if [ -z "${SelectedSourceHost}" ]; then
echo "File ${SOURCE_HOST}:${SourceFile} NOT FOUND"
exit
fi
echo "Using source ${SelectedSourceHost} with ${SelectedSourceFilePath} (${SelectedSourceSize} bytes)"
Cmd="/usr/bin/rsync -ahv"
Cmd+=" --mkpath"
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
Cmd+=" ${SOURCE_HOST}:${SourceFilePath}"
Cmd+=" ${SOURCE_HOST}:${SelectedSourceFilePath}"
Cmd+=" $OUTPUT_DIR/"
echo ${Cmd}
eval ${Cmd}
if [ ! -f ${OUTPUT_DIR}/${SourceFile} ] ; then
echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND"
exit
echo "File ${OUTPUT_DIR}/${SourceFile} NOT FOUND"
exit
fi
Cmd="(cd ${OUTPUT_DIR} && gunzip -f *.db.gz)"
echo ${Cmd}
eval ${Cmd}