#!/bin/bash # Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host. get_retrofit_dates() { local host=${1:-} local root_dir=${2:-} local filename_glob=${3:-} local day_count=${4:-} if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then echo "usage: get_retrofit_dates " >&2 return 1 fi if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then echo "get_retrofit_dates: must be a positive integer" >&2 return 1 fi local -a target_dates=() declare -A month_dirs=() if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then echo "get_retrofit_dates: requires GNU date arithmetic" >&2 return 1 fi local offset date for (( offset = 1; offset <= day_count; offset++ )); do date=$(date -d "$offset day ago" +%Y%m%d) target_dates+=("$date") month_dirs["${date:0:4}/${date:4:2}"]=1 done declare -A existing_dates=() local dir remote_path remote_cmd remote_output entry for dir in "${!month_dirs[@]}"; do remote_path="$root_dir/$dir" printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path" remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true) if [[ -n "$remote_output" ]]; then while IFS= read -r entry; do [[ -z "$entry" ]] && continue [[ $entry == $filename_glob ]] || continue if [[ "$entry" =~ ^([0-9]{8}) ]]; then existing_dates["${BASH_REMATCH[1]}"]=1 fi done <<<"$remote_output" fi done local -a missing_dates=() local idx for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do local dt=${target_dates[$idx]} [[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt") done printf '%s\n' "${missing_dates[@]}" } is_equity_business_date() { local ymd=${1:-} if [[ ! "$ymd" =~ ^[0-9]{8}$ ]]; then echo "is_equity_business_date: expected YYYYMMDD, got \"$ymd\"" >&2 return 2 fi local iso_date="${ymd:0:4}-${ymd:4:2}-${ymd:6:2}" local url="https://trading-calendar.cvtt.net/api/v1/markets/hours?mic=XNYS&start=${iso_date}&end=${iso_date}" local response if ! response=$(curl -fsS "$url" 2>/dev/null); then echo "is_equity_business_date: failed to query trading calendar API" >&2 return 1 fi if [[ "$response" =~ \[\s*\] ]]; then return 1 fi return 0 } get_equity_retrofit_dates() { local host=${1:-} local root_dir=${2:-} local filename_glob=${3:-} local day_count=${4:-} if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then echo "usage: get_equity_retrofit_dates " >&2 return 1 fi if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then echo "get_equity_retrofit_dates: must be a positive integer" >&2 return 1 fi if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then echo "get_equity_retrofit_dates: requires GNU date arithmetic" >&2 return 1 fi local -a target_dates=() local offset date weekday for (( offset = 1; offset <= day_count; offset++ )); do date=$(date -d "$offset day ago" +%Y%m%d) weekday=$(date -d "$date" +%u) (( weekday >= 6 )) && continue target_dates+=("$date") done declare -A existing_dates=() local remote_cmd remote_output entry printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1 %s" "$root_dir" "$filename_glob" remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true) if [[ -n "$remote_output" ]]; then while IFS= read -r entry; do [[ -z "$entry" ]] && continue if [[ "$entry" =~ ^([0-9]{8}) ]]; then existing_dates["${BASH_REMATCH[1]}"]=1 fi done <<<"$remote_output" fi local -a missing_dates=() local idx dt tmp rc for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do dt=${target_dates[$idx]} [[ -n "${existing_dates[$dt]:-}" ]] && continue is_equity_business_date "$dt" rc=$? if (( rc != 0 )); then continue fi if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then echo "invalid date \"$dt\"" >&2 return 1 fi missing_dates+=("$tmp") done printf '%s\n' "${missing_dates[@]}" }