archive/ops
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Cryptoval2 2025-10-19 15:41:05 +00:00
parent 2512023079
commit 231bfd079f
1 changed files with 9 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/mkt_data/sim_prepare/prepare_crypto_simdata.sh
View File

@ -3,8 +3,7 @@
# --------------------- Settings # --------------------- Settings
# SOURCE_HOST=cloud21.cvtt.vpn # SOURCE_HOST=cloud21.cvtt.vpn
# SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto # SOURCE_ROOT_DIR=/opt/store/cvtt/md_archive/crypto
# DB_SOURCE=cloud28 # DB_SOURCES=cloud28,cloud29
# DB_SOURCE_BACKUP=cloud29
# OUTPUT_DIR=/tmp # OUTPUT_DIR=/tmp
# DATE=20250516 # DATE=20250516
# RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/" # RSYNC_TARGETS="cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto/sim/ cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto/sim/"
@ -25,22 +24,20 @@ mkdir -p ${OUTPUT_DIR}
year=$(date -d ${DATE} +"%Y") year=$(date -d ${DATE} +"%Y")
month=$(date -d ${DATE} +"%m") month=$(date -d ${DATE} +"%m")
if [ -z "${DB_SOURCES}" ]; then
echo "DB_SOURCES is empty"
exit
fi
# read the file prepare_crypto_simdata.sh IFS=',' read -r -a db_source_hosts <<< "${DB_SOURCES}"
# implement the following logic.
# 1. check SourceFilePath on on SURCE_HOST for both DB_SOURCE and DB_SOURCE_BACKUP
# 2. if the file does not exist on either, exit like in line 47
# 3. if file exist on only one of them, use ti to rsync
# 4. if the file exists on both, use the one that is larger in size
SourceFile="${DATE}.mktdata.db.gz" SourceFile="${DATE}.mktdata.db.gz"
SelectedSourceHost="" SelectedSourceHost=""
SelectedSourceFilePath="" SelectedSourceFilePath=""
SelectedSourceSize=0 SelectedSourceSize=0
for db_source_host in ${DB_SOURCE} ${DB_SOURCE_BACKUP}; do for db_source_host in "${db_source_hosts[@]}"; do
SourceDir="${SOURCE_ROOT_DIR}/${db_source_host}/${year}/${month}" SourceDir="${SOURCE_ROOT_DIR}/${db_source_host}/${year}/${month}"
echo "SourceDir=${SourceDir}"
CandidatePath="${SourceDir}/${SourceFile}" CandidatePath="${SourceDir}/${SourceFile}"
remote_stat_cmd="if [ -f '${CandidatePath}' ]; then stat -c %s '${CandidatePath}'; else exit 1; fi" remote_stat_cmd="if [ -f '${CandidatePath}' ]; then stat -c %s '${CandidatePath}'; else exit 1; fi"
CandidateSize=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SOURCE_HOST} "${remote_stat_cmd}" 2>/dev/null) CandidateSize=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SOURCE_HOST} "${remote_stat_cmd}" 2>/dev/null)
@ -54,8 +51,9 @@ for db_source_host in ${DB_SOURCE} ${DB_SOURCE_BACKUP}; do
fi fi
done done
if [ -z "${SelectedSourceHost}" ]; then if [ -z "${SelectedSourceHost}" ]; then
echo "File ${SOURCE_HOST}:${SourceFile} NOT FOUND" echo "File ${SourceFile} NOT FOUND on any DB_SOURCES host"
exit exit
fi fi