retrofit dates retrieval automation

.gitignore

@@ -1,3 +1,5 @@
 logs/
 *.log
 nohup.out
+.history/
+__OLD__/

retrofit/get_retrofit_dates.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+# Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host.
+
+get_retrofit_dates() {
+  local host=${1:-}
+  local root_dir=${2:-}
+  local filename_glob=${3:-}
+  local day_count=${4:-}
+
+  if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
+    echo "usage: get_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
+    return 1
+  fi
+
+  if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then
+    echo "get_retrofit_dates: <day_count> must be a positive integer" >&2
+    return 1
+  fi
+
+  local -a target_dates=()
+  declare -A month_dirs=()
+
+  if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then
+    echo "get_retrofit_dates: requires GNU date arithmetic" >&2
+    return 1
+  fi
+
+  local offset date
+  for (( offset = 1; offset <= day_count; offset++ )); do
+    date=$(date -d "$offset day ago" +%Y%m%d)
+    target_dates+=("$date")
+    month_dirs["${date:0:4}/${date:4:2}"]=1
+  done
+
+  declare -A existing_dates=()
+
+  local dir remote_path remote_cmd remote_output entry
+  for dir in "${!month_dirs[@]}"; do
+    remote_path="$root_dir/$dir"
+    printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path"
+    remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
+
+    if [[ -n "$remote_output" ]]; then
+      while IFS= read -r entry; do
+        [[ -z "$entry" ]] && continue
+        [[ $entry == $filename_glob ]] || continue
+        if [[ "$entry" =~ ^([0-9]{8}) ]]; then
+          existing_dates["${BASH_REMATCH[1]}"]=1
+        fi
+      done <<<"$remote_output"
+    fi
+  done
+
+  local -a missing_dates=()
+  local idx
+  for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
+    local dt=${target_dates[$idx]}
+    [[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt")
+  done
+
+  printf '%s\n' "${missing_dates[@]}"
+}

retrofit/retrofit_crypto.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 
+
+function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
+source ${function_file}
+
+SourceHost=${1}
+
+Host=cvtt@cloud21.cvtt.vpn
+FileGlob='*.mktdata.db.gz'
+BackDaysCount=10
+
+Cmd="get_retrofit_dates  ${Host} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
+echo $Cmd
+
+DATES=$($Cmd)
+
+for dt in ${DATES}; do
+   echo $dt
+   Cmd="CRYPTO_MD_DATE=${dt}"
+   Cmd+=" /usr/bin/docker compose" 
+   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" 
+   Cmd+=" daily_crypto_${SourceHost}"
+
+   echo ${Cmd}
+   eval ${Cmd}
+done

retrofit/retrofit_crypto28.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 
-
-host=cloud28
-
-DATES=""
-#DATES+=" 20250801"
-#DATES+=" 20250802"
-#DATES+=" 20250803"
-#DATES+=" 20251004"
-#DATES+=" 20251005"
-#DATES+=" 20251006"
-DATES+=" 20251007"
-DATES+=" 20251008"
-#DATES+=" 20251009"
-#DATES+=" 20250910"
-#DATES+=" 20251011"
-DATES+=" 20251012"
-#DATES+=" 20251013"
-#DATES+=" 20251014"
-DATES+=" 20251015"
-#DATES+=" 20250816"
-#DATES+=" 20250817"
-#DATES+=" 20250818"
-#DATES+=" 20250919"
-#DATES+=" 20250920"
-#DATES+=" 20250921"
-#DATES+=" 20250922"
-#DATES+=" 20250823"
-#DATES+=" 20250824"
-#DATES+=" 20250825"
-#DATES+=" 20250826"
-#DATES+=" 20250827"
-#DATES+=" 20250828"
-#DATES+=" 20250829"
-#DATES+=" 20250830"
-#DATES+=" 20250831"
-
-for dt in ${DATES}; do
-   echo $dt
-   Cmd="CRYPTO_MD_DATE=${dt}"
-   Cmd+=" /usr/bin/docker compose" 
-   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" 
-   Cmd+=" daily_crypto_${host}"
-
-   echo ${Cmd}
-   eval ${Cmd}
-done

retrofit/retrofit_crypto29.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29 
-
-host=cloud29
-
-DATES=""
-#DATES+=" 20250801"
-#DATES+=" 20250802"
-#DATES+=" 20250803"
-#DATES+=" 20251004"
-#DATES+=" 20251005"
-#DATES+=" 20251006"
-#DATES+=" 20251007"
-#DATES+=" 20251008"
-DATES+=" 20251009"
-#DATES+=" 20251010"
-#DATES+=" 20251011"
-#DATES+=" 20251012"
-#DATES+=" 20251013"
-DATES+=" 20251014"
-DATES+=" 20251015"
-#DATES+=" 20250816"
-#DATES+=" 20250817"
-#DATES+=" 20250818"
-#DATES+=" 20250919"
-#DATES+=" 20250920"
-#DATES+=" 20250921"
-#DATES+=" 20250922"
-#DATES+=" 20250823"
-#DATES+=" 20250824"
-#DATES+=" 20250825"
-#DATES+=" 20250826"
-#DATES+=" 20250827"
-#DATES+=" 20250828"
-#DATES+=" 20250829"
-#DATES+=" 20250830"
-#DATES+=" 20250831"
-
-for dt in ${DATES}; do
-   echo $dt
-   Cmd="CRYPTO_MD_DATE=${dt}"
-   Cmd+=" /usr/bin/docker compose" 
-   Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" 
-   Cmd+=" daily_crypto_${host}"
-
-   echo ${Cmd}
-   eval ${Cmd}
-done

retrofit/run_retrofit.sh

@@ -1,11 +1,12 @@
 #!/bin/bash
 
 Script=${1}
+shift
+Params=${*}
 
 ScriptDir=$(realpath $(dirname ${0}))
 LogDir=${ScriptDir}/log
 mkdir -p ${LogDir}
 
-Cmd="nohup ${ScriptDir}/${Script} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &"
+Cmd="nohup ${ScriptDir}/${Script} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &"
-
 print_and_run ${Cmd}