Merge branch 'master' of https://gitea.cvtt.net/prod/utils

cron.examples/crontab.txt

@@ -1,8 +0,0 @@
-#
-# ---------- pruning md_archive
-10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log
-#
-# ---------- U t i l s
-15 0 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log
-#
-

cron/example.crontab

@@ -0,0 +1,16 @@
+# ============================================================
+#
+# ---------- pruning md_archive
+10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log
+#
+# ---------- U t i l s
+#
+# ---------- archiving logs
+15 6 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log
+#
+# ---------- moving archives to NAS
+15 7 * * * /works/utils/move_archives.sh  -D 'week ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).move_archives.log
+#
+# ============================================================
+# ---------- T e s t
+# ---------- T e s t

move_archives.sh

@@ -3,9 +3,9 @@
 
 function usage {
     echo -n "Usage: ${0}"
-    echo -n " -H <host_label>"
     echo -n " [ -A <archive_dir> (default /works/archive)]"
-    echo -n " [-D <older than time criteria> (default: '2 days ago')]"
+    echo -n " [ -T <target (default hl-storage.cvtt.vpn:/works/archive)>]"
+    echo -n " [ -D <older than time criteria> (default: '2 days ago')]"
     echo
     exit 1
 }
@@ -16,20 +16,24 @@ echo Starting $0 $*
 ArchiveDir=/works/archive
 DateCriteria="2 days ago"
 FromHost=$(hostname -s)
+TargetRoot=hl-storage.cvtt.vpn:/works/archive
 # ---- D e f a u l t s
 
 # ---------------- cmdline
-while getopts "A:H:D:" opt; do
+while getopts "A:T:D:h" opt; do
   case ${opt} in
     A )
       ArchiveDir=$OPTARG
       ;;
-    H )
-      FromHost=$OPTARG
+    T )
+      TargetRoot=$OPTARG
       ;;
     D )
       DateCriteria=$OPTARG
       ;;
+    h )
+      usage
+      ;;
     \? )
       echo "Invalid option: -$OPTARG" >&2
       usage
@@ -42,50 +46,34 @@ while getopts "A:H:D:" opt; do
 done
 # ---------------- cmdline
 
-
-if [ "${FromHost}" == "" ]
-then
-    usage
-fi
-
-TargetHost=cloud21.cvtt.vpn
-TargetRootDir=/opt/store/cvtt/archive
-
 Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S')
 
 echo "Looking for log files older than ${DateCriteria} in ${ArchiveDir}"
-Cmd="find ${ArchiveDir}/"
-Cmd="${Cmd} '('"
-Cmd="${Cmd} -name '*.log'"
-Cmd="${Cmd} -o -name '*.log.*'"
-Cmd="${Cmd} -o -name '*.logs.*'"
-Cmd="${Cmd} -o -name '*.tgz'"
-Cmd="${Cmd} ')'"
-Cmd="${Cmd} -type f"
-Cmd="${Cmd} -not -newermt \"${Oldest}\""
 
-echo ${Cmd}
-files=$(eval ${Cmd})
 
-if [ "$files" == "" ]
-then
+# 1. First, check if any matching files exist using a quick find check
+if ! find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then
     echo "No files found older than ${Oldest} in ${ArchiveDir}"
-else
-    Target="${TargetHost}:${TargetRootDir}/${FromHost}/"
-    echo "Moving files to ${Target}:"
-    echo -----------------
-    for f in ${files}
-    do
-        echo ${f}
-    done
-
-    Cmd="rsync -ahvv"
-    Cmd="${Cmd} --remove-source-files"
-    Cmd="${Cmd} $files"
-    Cmd="${Cmd} ${Target}"
-    echo ${Cmd}
-# exit
-    eval ${Cmd}
+    echo "Done ${0} ${*}"
+    exit 0
 fi
+
+Target="${TargetRoot}/${FromHost}/"
+echo "Moving files to ${Target}:"
+echo "-----------------"
+
+# 2. Safely pipe find into rsync using relative paths from the ArchiveDir base
+# This completely avoids "Argument list too long" errors and handles spaces perfectly
+set -x
+find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -printf "%P\0" | \
+    rsync -ahvv \
+          -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+          --remove-source-files \
+          --mkpath \
+          --from0 \
+          --files-from=- \
+          "${ArchiveDir}/" \
+          "${Target}"
+{ set +x; } 2>/dev/null
 echo Done ${0} ${*}