prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

retrofit dates retrieval automation

Browse Source
This commit is contained in:
Oleg Sheynin 2025-10-16 19:37:50 +00:00
parent da1eb3bcb9
commit 23290aea2b
6 changed files with 96 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
logs/
*.log
nohup.out
.history/
__OLD__/

63
retrofit/get_retrofit_dates.sh Normal file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env bash
# Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host.
get_retrofit_dates() {
local host=${1:-}
local root_dir=${2:-}
local filename_glob=${3:-}
local day_count=${4:-}
if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
echo "usage: get_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
return 1
fi
if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then
echo "get_retrofit_dates: <day_count> must be a positive integer" >&2
return 1
fi
local -a target_dates=()
declare -A month_dirs=()
if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then
echo "get_retrofit_dates: requires GNU date arithmetic" >&2
return 1
fi
local offset date
for (( offset = 1; offset <= day_count; offset++ )); do
date=$(date -d "$offset day ago" +%Y%m%d)
target_dates+=("$date")
month_dirs["${date:0:4}/${date:4:2}"]=1
done
declare -A existing_dates=()
local dir remote_path remote_cmd remote_output entry
for dir in "${!month_dirs[@]}"; do
remote_path="$root_dir/$dir"
printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path"
remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
if [[ -n "$remote_output" ]]; then
while IFS= read -r entry; do
[[ -z "$entry" ]] && continue
[[ $entry == $filename_glob ]] || continue
if [[ "$entry" =~ ^([0-9]{8}) ]]; then
existing_dates["${BASH_REMATCH[1]}"]=1
fi
done <<<"$remote_output"
fi
done
local -a missing_dates=()
local idx
for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
local dt=${target_dates[$idx]}
[[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt")
done
printf '%s\n' "${missing_dates[@]}"
}

28
retrofit/retrofit_crypto.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29
function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
source ${function_file}
SourceHost=${1}
Host=cvtt@cloud21.cvtt.vpn
FileGlob='*.mktdata.db.gz'
BackDaysCount=10
Cmd="get_retrofit_dates ${Host} /works/cvtt/md_archive/crypto/${SourceHost} ${FileGlob} ${BackDaysCount}"
echo $Cmd
DATES=$($Cmd)
for dt in ${DATES}; do
echo $dt
Cmd="CRYPTO_MD_DATE=${dt}"
Cmd+=" /usr/bin/docker compose"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" daily_crypto_${SourceHost}"
echo ${Cmd}
eval ${Cmd}
done

49
retrofit/retrofit_crypto28.sh
View File

@ -1,49 +0,0 @@
#!/bin/bash
# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29
host=cloud28
DATES=""
#DATES+=" 20250801"
#DATES+=" 20250802"
#DATES+=" 20250803"
#DATES+=" 20251004"
#DATES+=" 20251005"
#DATES+=" 20251006"
DATES+=" 20251007"
DATES+=" 20251008"
#DATES+=" 20251009"
#DATES+=" 20250910"
#DATES+=" 20251011"
DATES+=" 20251012"
#DATES+=" 20251013"
#DATES+=" 20251014"
DATES+=" 20251015"
#DATES+=" 20250816"
#DATES+=" 20250817"
#DATES+=" 20250818"
#DATES+=" 20250919"
#DATES+=" 20250920"
#DATES+=" 20250921"
#DATES+=" 20250922"
#DATES+=" 20250823"
#DATES+=" 20250824"
#DATES+=" 20250825"
#DATES+=" 20250826"
#DATES+=" 20250827"
#DATES+=" 20250828"
#DATES+=" 20250829"
#DATES+=" 20250830"
#DATES+=" 20250831"
for dt in ${DATES}; do
echo $dt
Cmd="CRYPTO_MD_DATE=${dt}"
Cmd+=" /usr/bin/docker compose"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" daily_crypto_${host}"
echo ${Cmd}
eval ${Cmd}
done

49
retrofit/retrofit_crypto29.sh
View File

@ -1,49 +0,0 @@
#!/bin/bash
# CRYPTO_MD_DATE=20250725 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up -d daily_crypto_cloud29
host=cloud29
DATES=""
#DATES+=" 20250801"
#DATES+=" 20250802"
#DATES+=" 20250803"
#DATES+=" 20251004"
#DATES+=" 20251005"
#DATES+=" 20251006"
#DATES+=" 20251007"
#DATES+=" 20251008"
DATES+=" 20251009"
#DATES+=" 20251010"
#DATES+=" 20251011"
#DATES+=" 20251012"
#DATES+=" 20251013"
DATES+=" 20251014"
DATES+=" 20251015"
#DATES+=" 20250816"
#DATES+=" 20250817"
#DATES+=" 20250818"
#DATES+=" 20250919"
#DATES+=" 20250920"
#DATES+=" 20250921"
#DATES+=" 20250922"
#DATES+=" 20250823"
#DATES+=" 20250824"
#DATES+=" 20250825"
#DATES+=" 20250826"
#DATES+=" 20250827"
#DATES+=" 20250828"
#DATES+=" 20250829"
#DATES+=" 20250830"
#DATES+=" 20250831"
for dt in ${DATES}; do
echo $dt
Cmd="CRYPTO_MD_DATE=${dt}"
Cmd+=" /usr/bin/docker compose"
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" daily_crypto_${host}"
echo ${Cmd}
eval ${Cmd}
done

5
retrofit/run_retrofit.sh
View File

@ -1,11 +1,12 @@
#!/bin/bash
Script=${1}
shift
Params=${*}
ScriptDir=$(realpath $(dirname ${0}))
LogDir=${ScriptDir}/log
mkdir -p ${LogDir}
Cmd="nohup ${ScriptDir}/${Script} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &"
Cmd="nohup ${ScriptDir}/${Script} ${Params} > ${LogDir}/$(date '+%Y%m%d_%H%M%S').${Script}.log 2>&1 &"
print_and_run ${Cmd}