progress

qcow2/backup_vm.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+echo Starting $0 $*
+
+VmName=${1}
+
+# ---------------------------------------------------
+# Cronjob requires explicit connection to local server
+# ---------------------------------------------------
+VirshCmd="/usr/bin/virsh -c qemu:///system"
+
+if  [ "${VmName}" == "" ]
+then
+    echo "Usage: $0 <VM Domain Name>"
+    exit 1
+fi
+
+Today=$(date +%Y%m%d)
+
+BackupDir=/localdisk/backup/qcow2
+GuestQcow2Files=$(${VirshCmd} domblklist ${VmName} | grep qcow2 | awk '{print $2}')
+
+Cmd="mkdir -p ${BackupDir}"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="${VirshCmd} suspend ${VmName}"
+echo ${Cmd} && eval ${Cmd}
+
+for qcow2_file in ${GuestQcow2Files}
+do
+    Cmd="cp ${qcow2_file} ${BackupDir}/${Today}.$(basename ${qcow2_file})"
+    echo ${Cmd} && eval ${Cmd}
+done
+
+Cmd="${VirshCmd} resume ${VmName}"
+echo ${Cmd} && eval ${Cmd}
+
+echo "$0 Done."
+
+

qcow2/clean_shrunk.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+print_and_run() {
+    local cmd="$*"
+    printf '➜ %s\n' "$cmd"
+    # pipefail applies to the subshell; change/remove if not desired
+    bash -o pipefail -c "$cmd"
+}
+
+
+SHRUNK="/opt/vm_drive/shrunk"
+LINKS="/opt/vm_drive/*.qcow2"
+ ACTION="echo Would remove"   # default: dry-run
+
+DRYRUN=0
+if [[ "$1" == "--dry-run" ]]; then
+    DRYRUN=1
+    shift
+fi
+
+find "$SHRUNK" -maxdepth 1 -type f -printf '%f\n' \
+    | grep -vxFf <(readlink -f $LINKS | xargs -n1 basename) \
+    | while read -r f; do
+        if (( DRYRUN )); then
+          echo "Would remove: $SHRUNK/$f"
+        else
+          echo "Removing: $SHRUNK/$f"
+          rm -v -- "$SHRUNK/$f"
+        fi
+      done

qcow2/compress_qcow2.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+if [ "$(id -u)" -ne 0 ]; then
+    echo "This script must be run as root." >&2
+    exit 1
+fi
+
+# ------ Settings
+Qcow2Dir=/localdisk/backup/qcow2
+ShrunkDir=/opt/vm_drive/shrunk
+TmpDir=/localdisk/tmp
+
+mkdir -p ${ShrunkDir}
+mkdir -p ${TmpDir}
+# ------ Settings
+
+Qcow2Files=$(find ${Qcow2Dir} -type f -name "*.qcow2" -mtime -1 -print)
+echo "Files to compress:"
+echo ${Qcow2Files}
+
+for file in ${Qcow2Files}
+do 
+    echo "Compressing file ${file}..."
+    SparseFile="${ShrunkDir}/$(basename ${file}).sparse"
+    ShrunkFile="${ShrunkDir}/$(basename ${file}).shrunk"
+    if [ -f ${ShrunkFile} ]
+    then
+	echo Shrunk file ${ShrunkFile} exists. Skipping...
+        continue
+    fi
+    Cmd="/usr/bin/virt-sparsify"
+    Cmd="${Cmd} --check-tmpdir=continue"
+    Cmd="${Cmd} --tmp=${TmpDir}"
+    Cmd="${Cmd} -q"
+    Cmd="${Cmd} ${file} ${SparseFile}"
+    Cmd="${Cmd} && /usr/bin/qemu-img convert"
+    Cmd="${Cmd} -c"
+    Cmd="${Cmd} -O qcow2"
+    Cmd="${Cmd} ${SparseFile} ${ShrunkFile}"
+    echo ${Cmd}
+    eval ${Cmd}
+
+    if [ ${?} -ne 0 ]
+    then
+        echo "Compression failed. Removing ${ShrunkFile}"
+        rm ${ShrunkFile}
+    else
+        filesz=$(stat -c %s ${file})
+        cfilesz=$(stat -c %s ${ShrunkFile})
+
+        ratio=$((cfilesz * 100 / filesz))
+        echo "Compression is successful. Compression ratio is ${ratio}%"
+        rm ${SparseFile}
+
+        ## ***************************************************************************
+        ##
+        ## Switching symlink will mask latest changes for future backups
+        ## if VM was not restarted.
+        ## This must be done manually before restarting - pick the latest shrunk
+        ## symlink it and restart VM
+        ##
+        ## ***************************************************************************
+
+        # GuestFile="${ShrunkFile#*.}"
+        # GuestFile="${GuestFile%.shrunk}"
+        # Cmd="ln -snf ${ShrunkFile} /opt/vm_drive/${GuestFile}"
+        # echo ${Cmd}
+        # eval ${Cmd}
+        ##
+        Cmd="chown oleg:oleg ${ShrunkFile}"
+        echo ${Cmd}
+        eval ${Cmd}
+    fi
+done
+    
+echo "Done $0 $*"
+

qcow2/compress_qcow2_file.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ============ R e q u i r e s  package guestfs-tools
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "This script must be run as root." >&2
+  exit 1
+fi
+
+# ------ Settings
+TmpDir=/localdisk/tmp
+
+# ------ Settings
+
+mkdir -p ${TmpDir}
+file=${1}
+out_file=${2}
+
+SparseFile="${out_file}.sparse"
+ShrunkFile="${out_file}.shrunk"
+
+echo "Compressing file ${file}..."
+Cmd="/usr/bin/virt-sparsify"
+Cmd="${Cmd} --check-tmpdir=continue"
+Cmd="${Cmd} --tmp=${TmpDir}"
+Cmd="${Cmd} -q"
+Cmd="${Cmd} ${file} ${SparseFile}"
+Cmd="${Cmd} && /usr/bin/qemu-img convert"
+Cmd="${Cmd} -c"
+Cmd="${Cmd} -O qcow2"
+Cmd="${Cmd} ${SparseFile} ${ShrunkFile}"
+echo ${Cmd}
+eval ${Cmd}
+
+if [ ${?} -ne 0 ]
+then
+    echo "Compression failed. Removing ${ShrunkFile}"
+    rm ${ShrunkFile}
+else
+    filesz=$(stat -c %s ${file})
+    cfilesz=$(stat -c %s ${ShrunkFile})
+
+    ratio=$((cfilesz * 100 / filesz))
+    echo "Compression is successful. Compression ratio is ${ratio}%"
+    rm ${SparseFile}
+fi
+    
+echo "Done $0 $*"
+

qcow2/cron/example.oleg.crontab

@@ -0,0 +1,19 @@
+#===========================================================================
+# -------- T e s t
+# -------- T e s t
+#===========================================================================
+10 2 * * MON,FRI /works/admin/backup_vm.sh desktop 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.desktop.log 2>&1
+10 4 * * MON,FRI /works/admin/backup_vm.sh home-fin 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.home-fin.log 2>&1
+#
+10 2 * * TUE /works/admin/backup_vm.sh allbright-dev 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.allbright-dev.log 2>&1
+10 4 * * TUE,FRI /works/admin/backup_vm.sh develop-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1
+#
+10 2 * * WED,SAT /works/admin/backup_vm.sh cvtt-admin 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.cvtt-admin.log 2>&1
+10 4 * * WED,SAT /works/admin/backup_vm.sh edu-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.edu-desktop-02.log 2>&1
+#
+10 2 * * THU,SUN /works/admin/backup_vm.sh develop-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1
+#---------------------- Utils
+23 23 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D "1 day ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).archive_logs.works.log 
+10 1 * * * /works/utils/prune_data.sh -d /opt/vm_drive/shrunk -D "2 weeks ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_shrunk.log 
+10 2 * * * /works/utils/prune_data.sh -d /localdisk/backup/qcow2 -D "2 months ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_backup.log 
+

qcow2/cron/example.root.crontab

@@ -0,0 +1,8 @@
+#===========================================================================
+# -------- T e s t
+# -------- T e s t
+#===========================================================================
+10 0 * * * /works/admin/compress_qcow2.sh  2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.compress_qcow2.log 2>&1
+###
+* * * * *   chown -R oleg:oleg /works 2>&1
+#