From 23290aea2b951e59d6c444aa5665b2cf4e7708c6 Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Thu, 16 Oct 2025 19:37:50 +0000 Subject: [PATCH] retrofit dates retrieval automation --- .gitignore | 2 ++ retrofit/get_retrofit_dates.sh | 63 ++++++++++++++++++++++++++++++++++ retrofit/retrofit_crypto.sh | 28 +++++++++++++++ retrofit/retrofit_crypto28.sh | 49 -------------------------- retrofit/retrofit_crypto29.sh | 49 -------------------------- retrofit/run_retrofit.sh | 5 +-- 6 files changed, 96 insertions(+), 100 deletions(-) create mode 100644 retrofit/get_retrofit_dates.sh create mode 100755 retrofit/retrofit_crypto.sh delete mode 100755 retrofit/retrofit_crypto28.sh delete mode 100755 retrofit/retrofit_crypto29.sh diff --git a/.gitignore b/.gitignore index d7d3286..c66e0c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ logs/ *.log nohup.out +.history/ +__OLD__/ diff --git a/retrofit/get_retrofit_dates.sh b/retrofit/get_retrofit_dates.sh new file mode 100644 index 0000000..26db3b4 --- /dev/null +++ b/retrofit/get_retrofit_dates.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host. + +get_retrofit_dates() { + local host=${1:-} + local root_dir=${2:-} + local filename_glob=${3:-} + local day_count=${4:-} + + if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then + echo "usage: get_retrofit_dates " >&2 + return 1 + fi + + if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then + echo "get_retrofit_dates: must be a positive integer" >&2 + return 1 + fi + + local -a target_dates=() + declare -A month_dirs=() + + if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then + echo "get_retrofit_dates: requires GNU date arithmetic" >&2 + return 1 + fi + + local offset date + for (( offset = 1; offset <= day_count; offset++ )); do + date=$(date -d "$offset day ago" +%Y%m%d) + target_dates+=("$date") + month_dirs["${date:0:4}/${date:4:2}"]=1 + done + + declare -A existing_dates=() + + local dir remote_path remote_cmd remote_output entry + for dir in "${!month_dirs[@]}"; do + remote_path="$root_dir/$dir" + printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path" + remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true) + + if [[ -n "$remote_output" ]]; then + while IFS= read -r entry; do + [[ -z "$entry" ]] && continue + [[ $entry == $filename_glob ]] || continue + if [[ "$entry" =~ ^([0-9]{8}) ]]; then + existing_dates["${BASH_REMATCH[1]}"]=1 + fi + done <<<"$remote_output" + fi + done + + local -a missing_dates=() + local idx + for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do + local dt=${target_dates[$idx]} + [[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt") + done + + printf '%s\n' "${missing_dates[@]}" +} diff --git a/retrofit/retrofit_crypto.sh b/retrofit/retrofit_crypto.sh new file mode 100755 index 0000000..64f0459 --- /dev/null +++ b/retrofit/retrofit_crypto.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 + +function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh +source ${function_file} + +SourceHost=${1} + +Host=cvtt@cloud21.cvtt.vpn +FileGlob='*.mktdata.db.gz' +BackDaysCount=10 + +Cmd="get_retrofit_dates ${Host} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}" +echo $Cmd + +DATES=$($Cmd) + +for dt in ${DATES}; do + echo $dt + Cmd="CRYPTO_MD_DATE=${dt}" + Cmd+=" /usr/bin/docker compose" + Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" + Cmd+=" daily_crypto_${SourceHost}" + + echo ${Cmd} + eval ${Cmd} +done diff --git a/retrofit/retrofit_crypto28.sh b/retrofit/retrofit_crypto28.sh deleted file mode 100755 index c929ddc..0000000 --- a/retrofit/retrofit_crypto28.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 - -host=cloud28 - -DATES="" -#DATES+=" 20250801" -#DATES+=" 20250802" -#DATES+=" 20250803" -#DATES+=" 20251004" -#DATES+=" 20251005" -#DATES+=" 20251006" -DATES+=" 20251007" -DATES+=" 20251008" -#DATES+=" 20251009" -#DATES+=" 20250910" -#DATES+=" 20251011" -DATES+=" 20251012" -#DATES+=" 20251013" -#DATES+=" 20251014" -DATES+=" 20251015" -#DATES+=" 20250816" -#DATES+=" 20250817" -#DATES+=" 20250818" -#DATES+=" 20250919" -#DATES+=" 20250920" -#DATES+=" 20250921" -#DATES+=" 20250922" -#DATES+=" 20250823" -#DATES+=" 20250824" -#DATES+=" 20250825" -#DATES+=" 20250826" -#DATES+=" 20250827" -#DATES+=" 20250828" -#DATES+=" 20250829" -#DATES+=" 20250830" -#DATES+=" 20250831" - -for dt in ${DATES}; do - echo $dt - Cmd="CRYPTO_MD_DATE=${dt}" - Cmd+=" /usr/bin/docker compose" - Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" - Cmd+=" daily_crypto_${host}" - - echo ${Cmd} - eval ${Cmd} -done diff --git a/retrofit/retrofit_crypto29.sh b/retrofit/retrofit_crypto29.sh deleted file mode 100755 index 98e1e7d..0000000 --- a/retrofit/retrofit_crypto29.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 - -host=cloud29 - -DATES="" -#DATES+=" 20250801" -#DATES+=" 20250802" -#DATES+=" 20250803" -#DATES+=" 20251004" -#DATES+=" 20251005" -#DATES+=" 20251006" -#DATES+=" 20251007" -#DATES+=" 20251008" -DATES+=" 20251009" -#DATES+=" 20251010" -#DATES+=" 20251011" -#DATES+=" 20251012" -#DATES+=" 20251013" -DATES+=" 20251014" -DATES+=" 20251015" -#DATES+=" 20250816" -#DATES+=" 20250817" -#DATES+=" 20250818" -#DATES+=" 20250919" -#DATES+=" 20250920" -#DATES+=" 20250921" -#DATES+=" 20250922" -#DATES+=" 20250823" -#DATES+=" 20250824" -#DATES+=" 20250825" -#DATES+=" 20250826" -#DATES+=" 20250827" -#DATES+=" 20250828" -#DATES+=" 20250829" -#DATES+=" 20250830" -#DATES+=" 20250831" - -for dt in ${DATES}; do - echo $dt - Cmd="CRYPTO_MD_DATE=${dt}" - Cmd+=" /usr/bin/docker compose" - Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" - Cmd+=" daily_crypto_${host}" - - echo ${Cmd} - eval ${Cmd} -done diff --git a/retrofit/run_retrofit.sh b/retrofit/run_retrofit.sh index 5acfae5..e00b19c 100755 --- a/retrofit/run_retrofit.sh +++ b/retrofit/run_retrofit.sh @@ -1,11 +1,12 @@ #!/bin/bash Script=${1} +shift +Params=${*} ScriptDir=$(realpath $(dirname ${0})) LogDir=${ScriptDir}/log mkdir -p ${LogDir} -Cmd="nohup ${ScriptDir}/${Script} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &" - +Cmd="nohup ${ScriptDir}/${Script} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &" print_and_run ${Cmd}