prod/daily_mktdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

equity retrofit dates automation

Browse Source
This commit is contained in:
Oleg Sheynin 2025-10-16 20:25:00 +00:00
parent 23290aea2b
commit 98c954d578
2 changed files with 120 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
retrofit/get_retrofit_dates.sh Normal file → Executable file
View File

@ -1,7 +1,6 @@
#!/usr/bin/env bash
#!/bin/bash
# Provides the get_retrofit_dates helper for identifying missing market-data archives on a remote host.
get_retrofit_dates() {
local host=${1:-}
local root_dir=${2:-}
@ -61,3 +60,63 @@ get_retrofit_dates() {
printf '%s\n' "${missing_dates[@]}"
}
get_equity_retrofit_dates() {
local host=${1:-}
local root_dir=${2:-}
local filename_glob=${3:-}
local day_count=${4:-}
if [[ -z "$host" || -z "$root_dir" || -z "$filename_glob" || -z "$day_count" ]]; then
echo "usage: get_equity_retrofit_dates <host> <root_dir> <filename_glob> <day_count>" >&2
return 1
fi
if ! [[ "$day_count" =~ ^[0-9]+$ ]] || (( day_count <= 0 )); then
echo "get_equity_retrofit_dates: <day_count> must be a positive integer" >&2
return 1
fi
if ! date -d "1 day ago" +%Y%m%d >/dev/null 2>&1; then
echo "get_equity_retrofit_dates: requires GNU date arithmetic" >&2
return 1
fi
local -a target_dates=()
local offset date weekday
for (( offset = 1; offset <= day_count; offset++ )); do
date=$(date -d "$offset day ago" +%Y%m%d)
weekday=$(date -d "$date" +%u)
(( weekday >= 6 )) && continue
target_dates+=("$date")
done
declare -A existing_dates=()
local remote_cmd remote_output entry
printf -v remote_cmd "cd %q 2>/dev/null && LC_ALL=C ls -1 %s" "$root_dir" "$filename_glob"
remote_output=$(ssh "$host" "$remote_cmd" 2>/dev/null || true)
if [[ -n "$remote_output" ]]; then
while IFS= read -r entry; do
[[ -z "$entry" ]] && continue
if [[ "$entry" =~ ^([0-9]{8}) ]]; then
existing_dates["${BASH_REMATCH[1]}"]=1
fi
done <<<"$remote_output"
fi
local -a missing_dates=()
local idx dt tmp
for (( idx = ${#target_dates[@]} - 1; idx >= 0; idx-- )); do
dt=${target_dates[$idx]}
[[ -n "${existing_dates[$dt]:-}" ]] && continue
if ! tmp=$(date -d "$dt" +%Y-%m-%d 2>/dev/null); then
echo "invalid date \"$dt\"" >&2
return 1
fi
missing_dates+=("$tmp")
done
printf '%s\n' "${missing_dates[@]}"
}

47
retrofit/retrofit_equity.sh
View File

@ -1,40 +1,22 @@
#!/bin/bash
# EQUITY_HBAR_DATE=2025-07-23 /usr/bin/docker compose -f /works/docker/daily_mktdata/docker-compose.yml up daily_equity_hbar
function_file=$(realpath $(dirname $0))/get_retrofit_dates.sh
source ${function_file}
DATES=""
#DATES+=" 2025-09-01"
#DATES+=" 2025-09-02"
#DATES+=" 2025-09-03"
#DATES+=" 2025-09-04"
#DATES+=" 2025-09-05"
#DATES+=" 2025-10-06"
DATES+=" 2025-10-07"
#DATES+=" 2025-10-08"
DATES+=" 2025-10-09"
DATES+=" 2025-10-10"
#DATES+=" 2025-09-11"
#DATES+=" 2025-09-12"
DATES+=" 2025-10-13"
DATES+=" 2025-10-14"
DATES+=" 2025-10-15"
#DATES+=" 2025-09-16"
#DATES+=" 2025-09-17"
#DATES+=" 2025-09-18"
#DATES+=" 2025-09-19"
#DATES+=" 2025-09-20"
#DATES+=" 2025-09-21"
#DATES+=" 2025-09-22"
#DATES+=" 2025-09-23"
#DATES+=" 2025-09-24"
#DATES+=" 2025-09-25"
#DATES+=" 2025-09-26"
#DATES+=" 2025-09-27"
#DATES+=" 2025-09-28"
#DATES+=" 2025-09-29"
#DATES+=" 2025-09-30"
#DATES+=" 2025-09-31"
Host=cvtt@cloud21.cvtt.vpn
FileGlob='*.alpaca_1m_bars.db.gz'
BackDaysCount=14
Cmd="get_equity_retrofit_dates ${Host} /works/cvtt/md_archive/equity/alpaca_md/2025/N/NVDA ${FileGlob} ${BackDaysCount}"
echo $Cmd
DATES=$($Cmd)
echo $DATES
exit
for dt in ${DATES}; do
echo $dt
@ -43,6 +25,7 @@ for dt in ${DATES}; do
Cmd+=" -f /works/docker/daily_mktdata/docker-compose.yml up"
Cmd+=" daily_equity_hbar"
echo ${Cmd}
eval ${Cmd}
done