Merge branch 'master' of https://gitea.cvtt.net/prod/utils

cron/example.crontab

@@ -4,10 +4,12 @@
 10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log
 #
 # ---------- U t i l s
+#
 # ---------- archiving logs
 15 6 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log
+#
 # ---------- moving archives to NAS
-15 7 * * * /works/utils/move_archives.sh  -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).move_archives.log
+15 7 * * * /works/utils/move_archives.sh  -D 'week ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).move_archives.log
 #
 # ============================================================
 # ---------- T e s t

move_archives.sh

@@ -52,7 +52,7 @@ echo "Looking for log files older than ${DateCriteria} in ${ArchiveDir}"
 
 
 # 1. First, check if any matching files exist using a quick find check
-if ! find "${ArchiveDir}" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then
+if ! find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then
     echo "No files found older than ${Oldest} in ${ArchiveDir}"
     echo "Done ${0} ${*}"
     exit 0
@@ -65,8 +65,9 @@ echo "-----------------"
 # 2. Safely pipe find into rsync using relative paths from the ArchiveDir base
 # This completely avoids "Argument list too long" errors and handles spaces perfectly
 set -x
-find "${ArchiveDir}" -type f -not -newermt "${Oldest}" -printf "%P\0" | \
+find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -printf "%P\0" | \
     rsync -ahvv \
+          -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
           --remove-source-files \
           --mkpath \
           --from0 \

qcow2/backup_vm.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+echo Starting $0 $*
+
+VmName=${1}
+
+# ---------------------------------------------------
+# Cronjob requires explicit connection to local server
+# ---------------------------------------------------
+VirshCmd="/usr/bin/virsh -c qemu:///system"
+
+if  [ "${VmName}" == "" ]
+then
+    echo "Usage: $0 <VM Domain Name>"
+    exit 1
+fi
+
+Today=$(date +%Y%m%d)
+
+BackupDir=/localdisk/backup/qcow2
+GuestQcow2Files=$(${VirshCmd} domblklist ${VmName} | grep qcow2 | awk '{print $2}')
+
+Cmd="mkdir -p ${BackupDir}"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="${VirshCmd} suspend ${VmName}"
+echo ${Cmd} && eval ${Cmd}
+
+for qcow2_file in ${GuestQcow2Files}
+do
+    Cmd="cp ${qcow2_file} ${BackupDir}/${Today}.$(basename ${qcow2_file})"
+    echo ${Cmd} && eval ${Cmd}
+done
+
+Cmd="${VirshCmd} resume ${VmName}"
+echo ${Cmd} && eval ${Cmd}
+
+echo "$0 Done."
+
+

qcow2/clean_shrunk.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+print_and_run() {
+    local cmd="$*"
+    printf '➜ %s\n' "$cmd"
+    # pipefail applies to the subshell; change/remove if not desired
+    bash -o pipefail -c "$cmd"
+}
+
+
+SHRUNK="/opt/vm_drive/shrunk"
+LINKS="/opt/vm_drive/*.qcow2"
+ ACTION="echo Would remove"   # default: dry-run
+
+DRYRUN=0
+if [[ "$1" == "--dry-run" ]]; then
+    DRYRUN=1
+    shift
+fi
+
+find "$SHRUNK" -maxdepth 1 -type f -printf '%f\n' \
+    | grep -vxFf <(readlink -f $LINKS | xargs -n1 basename) \
+    | while read -r f; do
+        if (( DRYRUN )); then
+          echo "Would remove: $SHRUNK/$f"
+        else
+          echo "Removing: $SHRUNK/$f"
+          rm -v -- "$SHRUNK/$f"
+        fi
+      done

qcow2/compress_qcow2.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+if [ "$(id -u)" -ne 0 ]; then
+    echo "This script must be run as root." >&2
+    exit 1
+fi
+
+# ------ Settings
+Qcow2Dir=/localdisk/backup/qcow2
+ShrunkDir=/opt/vm_drive/shrunk
+TmpDir=/localdisk/tmp
+
+mkdir -p ${ShrunkDir}
+mkdir -p ${TmpDir}
+# ------ Settings
+
+Qcow2Files=$(find ${Qcow2Dir} -type f -name "*.qcow2" -mtime -1 -print)
+echo "Files to compress:"
+echo ${Qcow2Files}
+
+for file in ${Qcow2Files}
+do 
+    echo "Compressing file ${file}..."
+    SparseFile="${ShrunkDir}/$(basename ${file}).sparse"
+    ShrunkFile="${ShrunkDir}/$(basename ${file}).shrunk"
+    if [ -f ${ShrunkFile} ]
+    then
+	echo Shrunk file ${ShrunkFile} exists. Skipping...
+        continue
+    fi
+    Cmd="/usr/bin/virt-sparsify"
+    Cmd="${Cmd} --check-tmpdir=continue"
+    Cmd="${Cmd} --tmp=${TmpDir}"
+    Cmd="${Cmd} -q"
+    Cmd="${Cmd} ${file} ${SparseFile}"
+    Cmd="${Cmd} && /usr/bin/qemu-img convert"
+    Cmd="${Cmd} -c"
+    Cmd="${Cmd} -O qcow2"
+    Cmd="${Cmd} ${SparseFile} ${ShrunkFile}"
+    echo ${Cmd}
+    eval ${Cmd}
+
+    if [ ${?} -ne 0 ]
+    then
+        echo "Compression failed. Removing ${ShrunkFile}"
+        rm ${ShrunkFile}
+    else
+        filesz=$(stat -c %s ${file})
+        cfilesz=$(stat -c %s ${ShrunkFile})
+
+        ratio=$((cfilesz * 100 / filesz))
+        echo "Compression is successful. Compression ratio is ${ratio}%"
+        rm ${SparseFile}
+
+        ## ***************************************************************************
+        ##
+        ## Switching symlink will mask latest changes for future backups
+        ## if VM was not restarted.
+        ## This must be done manually before restarting - pick the latest shrunk
+        ## symlink it and restart VM
+        ##
+        ## ***************************************************************************
+
+        # GuestFile="${ShrunkFile#*.}"
+        # GuestFile="${GuestFile%.shrunk}"
+        # Cmd="ln -snf ${ShrunkFile} /opt/vm_drive/${GuestFile}"
+        # echo ${Cmd}
+        # eval ${Cmd}
+        ##
+        Cmd="chown oleg:oleg ${ShrunkFile}"
+        echo ${Cmd}
+        eval ${Cmd}
+    fi
+done
+    
+echo "Done $0 $*"
+

qcow2/compress_qcow2_file.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ============ R e q u i r e s  package guestfs-tools
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "This script must be run as root." >&2
+  exit 1
+fi
+
+# ------ Settings
+TmpDir=/localdisk/tmp
+
+# ------ Settings
+
+mkdir -p ${TmpDir}
+file=${1}
+out_file=${2}
+
+SparseFile="${out_file}.sparse"
+ShrunkFile="${out_file}.shrunk"
+
+echo "Compressing file ${file}..."
+Cmd="/usr/bin/virt-sparsify"
+Cmd="${Cmd} --check-tmpdir=continue"
+Cmd="${Cmd} --tmp=${TmpDir}"
+Cmd="${Cmd} -q"
+Cmd="${Cmd} ${file} ${SparseFile}"
+Cmd="${Cmd} && /usr/bin/qemu-img convert"
+Cmd="${Cmd} -c"
+Cmd="${Cmd} -O qcow2"
+Cmd="${Cmd} ${SparseFile} ${ShrunkFile}"
+echo ${Cmd}
+eval ${Cmd}
+
+if [ ${?} -ne 0 ]
+then
+    echo "Compression failed. Removing ${ShrunkFile}"
+    rm ${ShrunkFile}
+else
+    filesz=$(stat -c %s ${file})
+    cfilesz=$(stat -c %s ${ShrunkFile})
+
+    ratio=$((cfilesz * 100 / filesz))
+    echo "Compression is successful. Compression ratio is ${ratio}%"
+    rm ${SparseFile}
+fi
+    
+echo "Done $0 $*"
+

qcow2/cron/example.oleg.crontab

@@ -0,0 +1,19 @@
+#===========================================================================
+# -------- T e s t
+# -------- T e s t
+#===========================================================================
+10 2 * * MON,FRI /works/admin/backup_vm.sh desktop 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.desktop.log 2>&1
+10 4 * * MON,FRI /works/admin/backup_vm.sh home-fin 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.home-fin.log 2>&1
+#
+10 2 * * TUE /works/admin/backup_vm.sh allbright-dev 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.allbright-dev.log 2>&1
+10 4 * * TUE,FRI /works/admin/backup_vm.sh develop-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1
+#
+10 2 * * WED,SAT /works/admin/backup_vm.sh cvtt-admin 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.cvtt-admin.log 2>&1
+10 4 * * WED,SAT /works/admin/backup_vm.sh edu-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.edu-desktop-02.log 2>&1
+#
+10 2 * * THU,SUN /works/admin/backup_vm.sh develop-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1
+#---------------------- Utils
+23 23 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D "1 day ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).archive_logs.works.log 
+10 1 * * * /works/utils/prune_data.sh -d /opt/vm_drive/shrunk -D "2 weeks ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_shrunk.log 
+10 2 * * * /works/utils/prune_data.sh -d /localdisk/backup/qcow2 -D "2 months ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_backup.log 
+

qcow2/cron/example.root.crontab

@@ -0,0 +1,8 @@
+#===========================================================================
+# -------- T e s t
+# -------- T e s t
+#===========================================================================
+10 0 * * * /works/admin/compress_qcow2.sh  2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.compress_qcow2.log 2>&1
+###
+* * * * *   chown -R oleg:oleg /works 2>&1
+#

qcow2/download_shrunk.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -o pipefail
+
+echo "*** START $0 ${*}"
+
+# ------- Settings
+MAX_AGE_DAYS=1
+
+SourceHost=hl-storage.cvtt.vpn
+SourceDir=/works/qcow2/shrunk
+
+DownloadTgtDir=/opt/vm_drive/shrunk
+
+# ------- Settings
+
+download_latest() {
+    local ssh_opts=(
+        -o StrictHostKeyChecking=accept-new
+    )
+
+    local rsync_rsh="ssh -o StrictHostKeyChecking=accept-new"
+
+    local FindCmd=(
+        ssh
+        "${ssh_opts[@]}"
+        "$SourceHost"
+        find "$SourceDir"
+        -type f
+        "-mtime -${MAX_AGE_DAYS}"
+        "-printf '%P\0'"
+    )
+
+    local RsyncCmd=(
+        rsync 
+        -avh 
+        --progress 
+        --partial 
+        --append-verify 
+        -n
+        --from0
+        --files-from=-
+        "-e $rsync_rsh"
+        "${SourceHost}:${SourceDir}/"
+        "${DownloadTgtDir}/"
+    )
+
+    echo "-------------------"
+    echo "Running:"
+    set -x
+    "${FindCmd[@]}" | "${RsyncCmd[@]}"
+    { set +x; } 2>/dev/null
+}
+
+echo "-------------------"
+echo "Downloading files from ${SourceHost}:${SourceDir}"
+download_latest 
+echo "-------------------"
+echo "Download completed"
+
+echo "*** DONE: ${0} ${*}"

qcow2/sparsify_qcow2.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+set -o pipefail
+
+echo "*** START $0 ${*}"
+
+# ------- Settings
+MAX_AGE_DAYS=1
+
+SourceHost=office-oleg-02.cvtt.vpn
+SourceDir=/localdisk/backup/qcow2
+
+DownloadTgtDir=/mnt/host_usb3/qcow2
+
+ShrunkDir=/mnt/host_usb3/qcow2/shrunk
+TmpDir=/mnt/host_usb3/tmp
+# ------- Settings
+
+download_latest() {
+    local ssh_opts=(
+        -o StrictHostKeyChecking=accept-new
+    )
+
+    local rsync_rsh="ssh -o StrictHostKeyChecking=accept-new"
+
+    local FindCmd=(
+        ssh
+        "${ssh_opts[@]}"
+        "$SourceHost"
+        find "$SourceDir"
+        -type f
+        "-mtime -${MAX_AGE_DAYS}"
+        "-printf '%P\0'"
+    )
+
+    local RsyncCmd=(
+        rsync 
+        # -n
+        -avh 
+        --progress 
+        --partial 
+        --append-verify 
+        --from0
+        --files-from=-
+        "-e $rsync_rsh"
+        "${SourceHost}:${SourceDir}/"
+        "${DownloadTgtDir}/"
+    )
+
+    echo "-------------------"
+    echo "Running:"
+    set -x
+    "${FindCmd[@]}" | "${RsyncCmd[@]}"
+    { set +x; } 2>/dev/null
+}
+
+compress_qcow2_file() {
+    local file=${1}
+    local out_file=${2}
+
+    mkdir -p "${ShrunkDir}"
+    mkdir -p "${TmpDir}"
+
+    local ShrunkFile="${out_file}.shrunk"
+
+    echo "Sparsifying file ${file}..."
+    local Cmd="/usr/bin/virt-sparsify"
+    Cmd+=" --check-tmpdir=fail"
+    Cmd+=" --tmp=${TmpDir}"
+    Cmd+=" --compress"
+    if [[ ! -t 1 ]]; then
+        Cmd+=" -q"
+    fi
+    Cmd+=" ${file} ${ShrunkFile}"
+    echo "${Cmd}"
+    ${Cmd}
+    
+    if [ ${?} -ne 0 ] ; then
+        echo "Compression failed. Removing ${ShrunkFile}"
+        rm "${ShrunkFile}"
+        return 255
+    else
+        filesz=$(stat -c %s "${file}")
+        if [ "${filesz}" == "0" ]; then
+            return 254
+        fi
+        cfilesz=$(stat -c %s "${ShrunkFile}")
+
+        ratio=$((cfilesz * 100 / filesz))
+        echo "Compression is successful. Compression ratio is ${ratio}%"
+    fi
+    echo "*** DONE: $file -> $ShrunkFile"
+    return 0
+}
+
+
+echo "-------------------"
+echo "Downloading files from ${SourceHost}:${SourceDir}"
+download_latest 
+# exit
+echo "-------------------"
+echo "Download completed"
+
+for qcow2_file in "${DownloadTgtDir}"/*.qcow2; do
+    shrunk_file="${ShrunkDir}/$(basename "${qcow2_file}")"
+    compress_qcow2_file "${qcow2_file}" "${shrunk_file}"
+    if [ ${?} -ne 0 ] ; then
+        echo "ERROR: File ${qcow2_file} compression FAILED"
+    else
+        echo "SUCCESS: File ${qcow2_file} is compressed to ${shrunk_file}"
+        echo "Removing file ${qcow2_file} ..."
+        rm  "${qcow2_file}"
+    fi    
+done
+
+echo "*** DONE $0 ${*}"