prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

equity retrofit dates automation

Browse Source
This commit is contained in:
Oleg Sheynin 2025-10-16 20:25:00 +00:00
parent 23290aea2b
commit 98c954d578
2 changed files with 120 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
retrofit/get_retrofit_dates.sh Normal file → Executable file
View File

@ -1,63 +1,122 @@
#!/usr/bin/env bash #!/bin/bash
# Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host. # Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host.
get_retrofit_dates() { get_retrofit_dates() {
local host=${1:-} local host=${1:-}
local root_dir=${2:-} local root_dir=${2:-}
local filename_glob=${3:-} local filename_glob=${3:-}
local day_count=${4:-} local day_count=${4:-}
if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
echo "usage: get_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2 echo "usage: get_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
return 1 return 1
fi fi
if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then
echo "get_retrofit_dates: <day_count> must be a positive integer" >&2 echo "get_retrofit_dates: <day_count> must be a positive integer" >&2
return 1 return 1
fi fi
local -a target_dates=() local -a target_dates=()
declare -A month_dirs=() declare -A month_dirs=()
if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then
echo "get_retrofit_dates: requires GNU date arithmetic" >&2 echo "get_retrofit_dates: requires GNU date arithmetic" >&2
return 1 return 1
fi fi
local offset date local offset date
for (( offset = 1; offset <= day_count; offset++ )); do for (( offset = 1; offset <= day_count; offset++ )); do
date=$(date -d "$offset day ago" +%Y%m%d) date=$(date -d "$offset day ago" +%Y%m%d)
target_dates+=("$date") target_dates+=("$date")
month_dirs["${date:0:4}/${date:4:2}"]=1 month_dirs["${date:0:4}/${date:4:2}"]=1
done done
declare -A existing_dates=() declare -A existing_dates=()
local dir remote_path remote_cmd remote_output entry local dir remote_path remote_cmd remote_output entry
for dir in "${!month_dirs[@]}"; do for dir in "${!month_dirs[@]}"; do
remote_path="$root_dir/$dir" remote_path="$root_dir/$dir"
printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path" printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path"
remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
if [[ -n "$remote_output" ]]; then
while IFS= read -r entry; do
[[ -z "$entry" ]] && continue
[[ $entry == $filename_glob ]] || continue
if [[ "$entry" =~ ^([0-9]{8}) ]]; then
existing_dates["${BASH_REMATCH[1]}"]=1
fi
done <<<"$remote_output"
fi
done
local -a missing_dates=()
local idx
for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
local dt=${target_dates[$idx]}
[[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt")
done
printf '%s\n' "${missing_dates[@]}"
}
get_equity_retrofit_dates() {
local host=${1:-}
local root_dir=${2:-}
local filename_glob=${3:-}
local day_count=${4:-}
if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
echo "usage: get_equity_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
return 1
fi
if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then
echo "get_equity_retrofit_dates: <day_count> must be a positive integer" >&2
return 1
fi
if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then
echo "get_equity_retrofit_dates: requires GNU date arithmetic" >&2
return 1
fi
local -a target_dates=()
local offset date weekday
for (( offset = 1; offset <= day_count; offset++ )); do
date=$(date -d "$offset day ago" +%Y%m%d)
weekday=$(date -d "$date" +%u)
(( weekday >= 6 )) && continue
target_dates+=("$date")
done
declare -A existing_dates=()
local remote_cmd remote_output entry
printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1 %s" "$root_dir" "$filename_glob"
remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true) remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
if [[ -n "$remote_output" ]]; then if [[ -n "$remote_output" ]]; then
while IFS= read -r entry; do while IFS= read -r entry; do
[[ -z "$entry" ]] && continue [[ -z "$entry" ]] && continue
[[ $entry == $filename_glob ]] || continue if [[ "$entry" =~ ^([0-9]{8}) ]]; then
if [[ "$entry" =~ ^([0-9]{8}) ]]; then existing_dates["${BASH_REMATCH[1]}"]=1
existing_dates["${BASH_REMATCH[1]}"]=1 fi
fi done <<<"$remote_output"
done <<<"$remote_output"
fi fi
done
local -a missing_dates=() local -a missing_dates=()
local idx local idx dt tmp
for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
local dt=${target_dates[$idx]} dt=${target_dates[$idx]}
[[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt") [[ -n "${existing_dates[$dt]:-}" ]] && continue
done if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then
echo "invalid date \"$dt\"" >&2
return 1
fi
missing_dates+=("$tmp")
done
printf '%s\n' "${missing_dates[@]}" printf '%s\n' "${missing_dates[@]}"
} }

47
retrofit/retrofit_equity.sh
View File

@ -1,40 +1,22 @@
#!/bin/bash #!/bin/bash
# EQUITY_HBAR_DATE=2025-07-23 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_equity_hbar # EQUITY_HBAR_DATE=2025-07-23 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_equity_hbar
function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
source ${function_file}
DATES=""
#DATES+=" 2025-09-01" Host=cvtt@cloud21.cvtt.vpn
#DATES+=" 2025-09-02" FileGlob='*.alpaca_1m_bars.db.gz'
#DATES+=" 2025-09-03" BackDaysCount=14
#DATES+=" 2025-09-04"
#DATES+=" 2025-09-05" Cmd="get_equity_retrofit_dates ${Host} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}"
#DATES+=" 2025-10-06" echo $Cmd
DATES+=" 2025-10-07"
#DATES+=" 2025-10-08" DATES=$($Cmd)
DATES+=" 2025-10-09"
DATES+=" 2025-10-10" echo $DATES
#DATES+=" 2025-09-11" exit
#DATES+=" 2025-09-12"
DATES+=" 2025-10-13"
DATES+=" 2025-10-14"
DATES+=" 2025-10-15"
#DATES+=" 2025-09-16"
#DATES+=" 2025-09-17"
#DATES+=" 2025-09-18"
#DATES+=" 2025-09-19"
#DATES+=" 2025-09-20"
#DATES+=" 2025-09-21"
#DATES+=" 2025-09-22"
#DATES+=" 2025-09-23"
#DATES+=" 2025-09-24"
#DATES+=" 2025-09-25"
#DATES+=" 2025-09-26"
#DATES+=" 2025-09-27"
#DATES+=" 2025-09-28"
#DATES+=" 2025-09-29"
#DATES+=" 2025-09-30"
#DATES+=" 2025-09-31"
for dt in ${DATES}; do for dt in ${DATES}; do
echo $dt echo $dt
@ -43,6 +25,7 @@ for dt in ${DATES}; do
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" daily_equity_hbar" Cmd+=" daily_equity_hbar"
echo ${Cmd} echo ${Cmd}
eval ${Cmd} eval ${Cmd}
done done