152 lines
4.6 KiB
Bash
Executable File
152 lines
4.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host.
|
|
get_retrofit_dates() {
|
|
local host=${1:-}
|
|
local root_dir=${2:-}
|
|
local filename_glob=${3:-}
|
|
local day_count=${4:-}
|
|
|
|
if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
|
|
echo "usage: get_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
|
|
return 1
|
|
fi
|
|
|
|
if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then
|
|
echo "get_retrofit_dates: <day_count> must be a positive integer" >&2
|
|
return 1
|
|
fi
|
|
|
|
local -a target_dates=()
|
|
declare -A month_dirs=()
|
|
|
|
if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then
|
|
echo "get_retrofit_dates: requires GNU date arithmetic" >&2
|
|
return 1
|
|
fi
|
|
|
|
local offset date
|
|
for (( offset = 1; offset <= day_count; offset++ )); do
|
|
date=$(date -d "$offset day ago" +%Y%m%d)
|
|
target_dates+=("$date")
|
|
month_dirs["${date:0:4}/${date:4:2}"]=1
|
|
done
|
|
|
|
declare -A existing_dates=()
|
|
|
|
local dir remote_path remote_cmd remote_output entry
|
|
for dir in "${!month_dirs[@]}"; do
|
|
remote_path="$root_dir/$dir"
|
|
printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path"
|
|
remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
|
|
|
|
if [[ -n "$remote_output" ]]; then
|
|
while IFS= read -r entry; do
|
|
[[ -z "$entry" ]] && continue
|
|
[[ $entry == $filename_glob ]] || continue
|
|
if [[ "$entry" =~ ^([0-9]{8}) ]]; then
|
|
existing_dates["${BASH_REMATCH[1]}"]=1
|
|
fi
|
|
done <<<"$remote_output"
|
|
fi
|
|
done
|
|
|
|
local -a missing_dates=()
|
|
local idx
|
|
for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
|
|
local dt=${target_dates[$idx]}
|
|
[[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt")
|
|
done
|
|
|
|
printf '%s\n' "${missing_dates[@]}"
|
|
}
|
|
|
|
is_equity_business_date() {
|
|
local ymd=${1:-}
|
|
|
|
if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then
|
|
echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2
|
|
return 2
|
|
fi
|
|
|
|
local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}"
|
|
local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}"
|
|
local response
|
|
|
|
if ! response=$(curl -fsS "$url" 2>/dev/null); then
|
|
echo "is_equity_business_date: failed to query trading calendar API" >&2
|
|
return 1
|
|
fi
|
|
|
|
if [[ "$response" =~ \[\s*\] ]]; then
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
get_equity_retrofit_dates() {
|
|
local host=${1:-}
|
|
local root_dir=${2:-}
|
|
local filename_glob=${3:-}
|
|
local day_count=${4:-}
|
|
|
|
if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
|
|
echo "usage: get_equity_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
|
|
return 1
|
|
fi
|
|
|
|
if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then
|
|
echo "get_equity_retrofit_dates: <day_count> must be a positive integer" >&2
|
|
return 1
|
|
fi
|
|
|
|
if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then
|
|
echo "get_equity_retrofit_dates: requires GNU date arithmetic" >&2
|
|
return 1
|
|
fi
|
|
|
|
local -a target_dates=()
|
|
local offset date weekday
|
|
for (( offset = 1; offset <= day_count; offset++ )); do
|
|
date=$(date -d "$offset day ago" +%Y%m%d)
|
|
weekday=$(date -d "$date" +%u)
|
|
(( weekday >= 6 )) && continue
|
|
target_dates+=("$date")
|
|
done
|
|
|
|
declare -A existing_dates=()
|
|
|
|
local remote_cmd remote_output entry
|
|
printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1 %s" "$root_dir" "$filename_glob"
|
|
remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
|
|
|
|
if [[ -n "$remote_output" ]]; then
|
|
while IFS= read -r entry; do
|
|
[[ -z "$entry" ]] && continue
|
|
if [[ "$entry" =~ ^([0-9]{8}) ]]; then
|
|
existing_dates["${BASH_REMATCH[1]}"]=1
|
|
fi
|
|
done <<<"$remote_output"
|
|
fi
|
|
|
|
local -a missing_dates=()
|
|
local idx dt tmp rc
|
|
for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
|
|
dt=${target_dates[$idx]}
|
|
[[ -n "${existing_dates[$dt]:-}" ]] && continue
|
|
is_equity_business_date "$dt"
|
|
rc=$?
|
|
if (( rc != 0 )); then
|
|
continue
|
|
fi
|
|
if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then
|
|
echo "invalid date \"$dt\"" >&2
|
|
return 1
|
|
fi
|
|
missing_dates+=("$tmp")
|
|
done
|
|
|
|
printf '%s\n' "${missing_dates[@]}"
|
|
}
|