diff --git a/cron.examples/crontab.txt b/cron.examples/crontab.txt deleted file mode 100644 index 42c64e3..0000000 --- a/cron.examples/crontab.txt +++ /dev/null @@ -1,8 +0,0 @@ -# -# ---------- pruning md_archive -10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log -# -# ---------- U t i l s -15 0 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log -# - diff --git a/cron/example.crontab b/cron/example.crontab new file mode 100644 index 0000000..a313954 --- /dev/null +++ b/cron/example.crontab @@ -0,0 +1,16 @@ +# ============================================================ +# +# ---------- pruning md_archive +10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log +# +# ---------- U t i l s +# +# ---------- archiving logs +15 6 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log +# +# ---------- moving archives to NAS +15 7 * * * /works/utils/move_archives.sh -D 'week ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).move_archives.log +# +# ============================================================ +# ---------- T e s t +# ---------- T e s t diff --git a/move_archives.sh b/move_archives.sh index 8081984..73e61d4 100755 --- a/move_archives.sh +++ b/move_archives.sh @@ -3,9 +3,9 @@ function usage { echo -n "Usage: ${0}" - echo -n " -H " echo -n " [ -A (default /works/archive)]" - echo -n " [-D (default: '2 days ago')]" + echo -n " [ -T ]" + echo -n " [ -D (default: '2 days ago')]" echo exit 1 } @@ -16,20 +16,24 @@ echo Starting $0 $* ArchiveDir=/works/archive DateCriteria="2 days ago" FromHost=$(hostname -s) +TargetRoot=hl-storage.cvtt.vpn:/works/archive # ---- D e f a u l t s # ---------------- cmdline -while getopts "A:H:D:" opt; do +while getopts "A:T:D:h" opt; do case ${opt} in A ) ArchiveDir=$OPTARG ;; - H ) - FromHost=$OPTARG + T ) + TargetRoot=$OPTARG ;; D ) DateCriteria=$OPTARG ;; + h ) + usage + ;; \? ) echo "Invalid option: -$OPTARG" >&2 usage @@ -42,50 +46,34 @@ while getopts "A:H:D:" opt; do done # ---------------- cmdline - -if [ "${FromHost}" == "" ] -then - usage -fi - -TargetHost=cloud21.cvtt.vpn -TargetRootDir=/opt/store/cvtt/archive - Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S') echo "Looking for log files older than ${DateCriteria} in ${ArchiveDir}" -Cmd="find ${ArchiveDir}/" -Cmd="${Cmd} '('" -Cmd="${Cmd} -name '*.log'" -Cmd="${Cmd} -o -name '*.log.*'" -Cmd="${Cmd} -o -name '*.logs.*'" -Cmd="${Cmd} -o -name '*.tgz'" -Cmd="${Cmd} ')'" -Cmd="${Cmd} -type f" -Cmd="${Cmd} -not -newermt \"${Oldest}\"" -echo ${Cmd} -files=$(eval ${Cmd}) -if [ "$files" == "" ] -then +# 1. First, check if any matching files exist using a quick find check +if ! find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then echo "No files found older than ${Oldest} in ${ArchiveDir}" -else - Target="${TargetHost}:${TargetRootDir}/${FromHost}/" - echo "Moving files to ${Target}:" - echo ----------------- - for f in ${files} - do - echo ${f} - done - - Cmd="rsync -ahvv" - Cmd="${Cmd} --remove-source-files" - Cmd="${Cmd} $files" - Cmd="${Cmd} ${Target}" - echo ${Cmd} -# exit - eval ${Cmd} + echo "Done ${0} ${*}" + exit 0 fi + +Target="${TargetRoot}/${FromHost}/" +echo "Moving files to ${Target}:" +echo "-----------------" + +# 2. Safely pipe find into rsync using relative paths from the ArchiveDir base +# This completely avoids "Argument list too long" errors and handles spaces perfectly +set -x +find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -printf "%P\0" | \ + rsync -ahvv \ + -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + --remove-source-files \ + --mkpath \ + --from0 \ + --files-from=- \ + "${ArchiveDir}/" \ + "${Target}" +{ set +x; } 2>/dev/null echo Done ${0} ${*}