From 4c1c715cddf6af91349aabc12cc57676bf8af3ec Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Tue, 23 Jun 2026 00:12:03 +0000 Subject: [PATCH 1/5] minor --- cron.examples/crontab.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cron.examples/crontab.txt b/cron.examples/crontab.txt index 42c64e3..23088ab 100644 --- a/cron.examples/crontab.txt +++ b/cron.examples/crontab.txt @@ -1,3 +1,4 @@ +# ============================================================ # # ---------- pruning md_archive 10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log @@ -5,4 +6,6 @@ # ---------- U t i l s 15 0 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log # - +# ============================================================ +# ---------- T e s t +# ---------- T e s t From 84882d6c5b8770b9fc698b0212113ac184621a6c Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Tue, 23 Jun 2026 12:23:48 -0400 Subject: [PATCH 2/5] progress --- cron.examples/crontab.txt => cron/example.crontab | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) rename cron.examples/crontab.txt => cron/example.crontab (65%) diff --git a/cron.examples/crontab.txt b/cron/example.crontab similarity index 65% rename from cron.examples/crontab.txt rename to cron/example.crontab index 23088ab..26e26c6 100644 --- a/cron.examples/crontab.txt +++ b/cron/example.crontab @@ -4,7 +4,10 @@ 10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log # # ---------- U t i l s -15 0 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log +# ---------- archiving logs +15 6 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log +# ---------- moving archives to NAS +15 7 * * * /works/utils/move_archives.sh -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).move_archives.log # # ============================================================ # ---------- T e s t From 0452f622a0829414b9beb249993350ad5b634da4 Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Tue, 23 Jun 2026 16:58:26 +0000 Subject: [PATCH 3/5] progress --- move_archives.sh | 73 ++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/move_archives.sh b/move_archives.sh index 8081984..163fbbf 100755 --- a/move_archives.sh +++ b/move_archives.sh @@ -3,9 +3,9 @@ function usage { echo -n "Usage: ${0}" - echo -n " -H " echo -n " [ -A (default /works/archive)]" - echo -n " [-D (default: '2 days ago')]" + echo -n " [ -T ]" + echo -n " [ -D (default: '2 days ago')]" echo exit 1 } @@ -16,20 +16,24 @@ echo Starting $0 $* ArchiveDir=/works/archive DateCriteria="2 days ago" FromHost=$(hostname -s) +TargetRoot=hl-storage.cvtt.vpn:/works/archive # ---- D e f a u l t s # ---------------- cmdline -while getopts "A:H:D:" opt; do +while getopts "A:T:D:h" opt; do case ${opt} in A ) ArchiveDir=$OPTARG ;; - H ) - FromHost=$OPTARG + T ) + TargetRoot=$OPTARG ;; D ) DateCriteria=$OPTARG ;; + h ) + usage + ;; \? ) echo "Invalid option: -$OPTARG" >&2 usage @@ -42,50 +46,33 @@ while getopts "A:H:D:" opt; do done # ---------------- cmdline - -if [ "${FromHost}" == "" ] -then - usage -fi - -TargetHost=cloud21.cvtt.vpn -TargetRootDir=/opt/store/cvtt/archive - Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S') echo "Looking for log files older than ${DateCriteria} in ${ArchiveDir}" -Cmd="find ${ArchiveDir}/" -Cmd="${Cmd} '('" -Cmd="${Cmd} -name '*.log'" -Cmd="${Cmd} -o -name '*.log.*'" -Cmd="${Cmd} -o -name '*.logs.*'" -Cmd="${Cmd} -o -name '*.tgz'" -Cmd="${Cmd} ')'" -Cmd="${Cmd} -type f" -Cmd="${Cmd} -not -newermt \"${Oldest}\"" -echo ${Cmd} -files=$(eval ${Cmd}) -if [ "$files" == "" ] -then +# 1. First, check if any matching files exist using a quick find check +if ! find "${ArchiveDir}" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then echo "No files found older than ${Oldest} in ${ArchiveDir}" -else - Target="${TargetHost}:${TargetRootDir}/${FromHost}/" - echo "Moving files to ${Target}:" - echo ----------------- - for f in ${files} - do - echo ${f} - done - - Cmd="rsync -ahvv" - Cmd="${Cmd} --remove-source-files" - Cmd="${Cmd} $files" - Cmd="${Cmd} ${Target}" - echo ${Cmd} -# exit - eval ${Cmd} + echo "Done ${0} ${*}" + exit 0 fi + +Target="${TargetRoot}/${FromHost}/" +echo "Moving files to ${Target}:" +echo "-----------------" + +# 2. Safely pipe find into rsync using relative paths from the ArchiveDir base +# This completely avoids "Argument list too long" errors and handles spaces perfectly +set -x +find "${ArchiveDir}" -type f -not -newermt "${Oldest}" -printf "%P\0" | \ + rsync -ahvv \ + --remove-source-files \ + --mkpath \ + --from0 \ + --files-from=- \ + "${ArchiveDir}/" \ + "${Target}" +{ set +x; } 2>/dev/null echo Done ${0} ${*} From 15799cd4f7e01cce64d35b2a5d53f865eba8185b Mon Sep 17 00:00:00 2001 From: Cryptoval2 Date: Tue, 23 Jun 2026 17:00:33 +0000 Subject: [PATCH 4/5] progress --- cron/example.crontab | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cron/example.crontab b/cron/example.crontab index 26e26c6..a313954 100644 --- a/cron/example.crontab +++ b/cron/example.crontab @@ -4,10 +4,12 @@ 10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log # # ---------- U t i l s +# # ---------- archiving logs 15 6 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log +# # ---------- moving archives to NAS -15 7 * * * /works/utils/move_archives.sh -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).move_archives.log +15 7 * * * /works/utils/move_archives.sh -D 'week ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).move_archives.log # # ============================================================ # ---------- T e s t From f152b7c358416e0cddd17edf38a40600e175dde3 Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Tue, 23 Jun 2026 19:51:34 +0000 Subject: [PATCH 5/5] progress --- move_archives.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/move_archives.sh b/move_archives.sh index 163fbbf..73e61d4 100755 --- a/move_archives.sh +++ b/move_archives.sh @@ -52,7 +52,7 @@ echo "Looking for log files older than ${DateCriteria} in ${ArchiveDir}" # 1. First, check if any matching files exist using a quick find check -if ! find "${ArchiveDir}" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then +if ! find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then echo "No files found older than ${Oldest} in ${ArchiveDir}" echo "Done ${0} ${*}" exit 0 @@ -65,8 +65,9 @@ echo "-----------------" # 2. Safely pipe find into rsync using relative paths from the ArchiveDir base # This completely avoids "Argument list too long" errors and handles spaces perfectly set -x -find "${ArchiveDir}" -type f -not -newermt "${Oldest}" -printf "%P\0" | \ +find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -printf "%P\0" | \ rsync -ahvv \ + -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ --remove-source-files \ --mkpath \ --from0 \