From 98c954d578cb28aa677923b372626b4643bce44a Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Thu, 16 Oct 2025 20:25:00 +0000 Subject: [PATCH] equity retrofit dates automation --- retrofit/get_retrofit_dates.sh | 151 +++++++++++++++++++++++---------- retrofit/retrofit_equity.sh | 47 ++++------ 2 files changed, 120 insertions(+), 78 deletions(-) mode change 100644 => 100755 retrofit/get_retrofit_dates.sh diff --git a/retrofit/get_retrofit_dates.sh b/retrofit/get_retrofit_dates.sh old mode 100644 new mode 100755 index 26db3b4..cf3452b --- a/retrofit/get_retrofit_dates.sh +++ b/retrofit/get_retrofit_dates.sh @@ -1,63 +1,122 @@ -#!/usr/bin/env bash +#!/bin/bash # Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host. - get_retrofit_dates() { - local host=${1:-} - local root_dir=${2:-} - local filename_glob=${3:-} - local day_count=${4:-} + local host=${1:-} + local root_dir=${2:-} + local filename_glob=${3:-} + local day_count=${4:-} - if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then - echo "usage: get_retrofit_dates " >&2 - return 1 - fi + if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then + echo "usage: get_retrofit_dates " >&2 + return 1 + fi - if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then - echo "get_retrofit_dates: must be a positive integer" >&2 - return 1 - fi + if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then + echo "get_retrofit_dates: must be a positive integer" >&2 + return 1 + fi - local -a target_dates=() - declare -A month_dirs=() + local -a target_dates=() + declare -A month_dirs=() - if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then - echo "get_retrofit_dates: requires GNU date arithmetic" >&2 - return 1 - fi + if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then + echo "get_retrofit_dates: requires GNU date arithmetic" >&2 + return 1 + fi - local offset date - for (( offset = 1; offset <= day_count; offset++ )); do - date=$(date -d "$offset day ago" +%Y%m%d) - target_dates+=("$date") - month_dirs["${date:0:4}/${date:4:2}"]=1 - done + local offset date + for (( offset = 1; offset <= day_count; offset++ )); do + date=$(date -d "$offset day ago" +%Y%m%d) + target_dates+=("$date") + month_dirs["${date:0:4}/${date:4:2}"]=1 + done - declare -A existing_dates=() + declare -A existing_dates=() - local dir remote_path remote_cmd remote_output entry - for dir in "${!month_dirs[@]}"; do - remote_path="$root_dir/$dir" - printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path" + local dir remote_path remote_cmd remote_output entry + for dir in "${!month_dirs[@]}"; do + remote_path="$root_dir/$dir" + printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1" "$remote_path" + remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true) + + if [[ -n "$remote_output" ]]; then + while IFS= read -r entry; do + [[ -z "$entry" ]] && continue + [[ $entry == $filename_glob ]] || continue + if [[ "$entry" =~ ^([0-9]{8}) ]]; then + existing_dates["${BASH_REMATCH[1]}"]=1 + fi + done <<<"$remote_output" + fi + done + + local -a missing_dates=() + local idx + for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do + local dt=${target_dates[$idx]} + [[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt") + done + + printf '%s\n' "${missing_dates[@]}" +} + +get_equity_retrofit_dates() { + local host=${1:-} + local root_dir=${2:-} + local filename_glob=${3:-} + local day_count=${4:-} + + if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then + echo "usage: get_equity_retrofit_dates " >&2 + return 1 + fi + + if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then + echo "get_equity_retrofit_dates: must be a positive integer" >&2 + return 1 + fi + + if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then + echo "get_equity_retrofit_dates: requires GNU date arithmetic" >&2 + return 1 + fi + + local -a target_dates=() + local offset date weekday + for (( offset = 1; offset <= day_count; offset++ )); do + date=$(date -d "$offset day ago" +%Y%m%d) + weekday=$(date -d "$date" +%u) + (( weekday >= 6 )) && continue + target_dates+=("$date") + done + + declare -A existing_dates=() + + local remote_cmd remote_output entry + printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1 %s" "$root_dir" "$filename_glob" remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true) if [[ -n "$remote_output" ]]; then - while IFS= read -r entry; do - [[ -z "$entry" ]] && continue - [[ $entry == $filename_glob ]] || continue - if [[ "$entry" =~ ^([0-9]{8}) ]]; then - existing_dates["${BASH_REMATCH[1]}"]=1 - fi - done <<<"$remote_output" + while IFS= read -r entry; do + [[ -z "$entry" ]] && continue + if [[ "$entry" =~ ^([0-9]{8}) ]]; then + existing_dates["${BASH_REMATCH[1]}"]=1 + fi + done <<<"$remote_output" fi - done - local -a missing_dates=() - local idx - for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do - local dt=${target_dates[$idx]} - [[ -n "${existing_dates[$dt]:-}" ]] || missing_dates+=("$dt") - done + local -a missing_dates=() + local idx dt tmp + for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do + dt=${target_dates[$idx]} + [[ -n "${existing_dates[$dt]:-}" ]] && continue + if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then + echo "invalid date \"$dt\"" >&2 + return 1 + fi + missing_dates+=("$tmp") + done - printf '%s\n' "${missing_dates[@]}" + printf '%s\n' "${missing_dates[@]}" } diff --git a/retrofit/retrofit_equity.sh b/retrofit/retrofit_equity.sh index 0d8cf91..8fae031 100755 --- a/retrofit/retrofit_equity.sh +++ b/retrofit/retrofit_equity.sh @@ -1,40 +1,22 @@ #!/bin/bash # EQUITY_HBAR_DATE=2025-07-23 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_equity_hbar +function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh +source ${function_file} -DATES="" -#DATES+=" 2025-09-01" -#DATES+=" 2025-09-02" -#DATES+=" 2025-09-03" -#DATES+=" 2025-09-04" -#DATES+=" 2025-09-05" -#DATES+=" 2025-10-06" -DATES+=" 2025-10-07" -#DATES+=" 2025-10-08" -DATES+=" 2025-10-09" -DATES+=" 2025-10-10" -#DATES+=" 2025-09-11" -#DATES+=" 2025-09-12" -DATES+=" 2025-10-13" -DATES+=" 2025-10-14" -DATES+=" 2025-10-15" -#DATES+=" 2025-09-16" -#DATES+=" 2025-09-17" -#DATES+=" 2025-09-18" -#DATES+=" 2025-09-19" -#DATES+=" 2025-09-20" -#DATES+=" 2025-09-21" -#DATES+=" 2025-09-22" -#DATES+=" 2025-09-23" -#DATES+=" 2025-09-24" -#DATES+=" 2025-09-25" -#DATES+=" 2025-09-26" -#DATES+=" 2025-09-27" -#DATES+=" 2025-09-28" -#DATES+=" 2025-09-29" -#DATES+=" 2025-09-30" -#DATES+=" 2025-09-31" + +Host=cvtt@cloud21.cvtt.vpn +FileGlob='*.alpaca_1m_bars.db.gz' +BackDaysCount=14 + +Cmd="get_equity_retrofit_dates ${Host} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}" +echo $Cmd + +DATES=$($Cmd) + +echo $DATES +exit for dt in ${DATES}; do echo $dt @@ -43,6 +25,7 @@ for dt in ${DATES}; do Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up" Cmd+=" daily_equity_hbar" + echo ${Cmd} eval ${Cmd} done