From 3385ffb33a90476b2a7fa91f81ffab0e81a0984f Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Wed, 24 Jun 2026 19:52:27 +0000 Subject: [PATCH] progress --- qcow2/.gitignore | 0 qcow2/backup_vm.sh | 39 +++++++++++++++++ qcow2/clean_shrunk.sh | 30 +++++++++++++ qcow2/compress_qcow2.sh | 77 +++++++++++++++++++++++++++++++++ qcow2/compress_qcow2_file.sh | 49 +++++++++++++++++++++ qcow2/cron/example.oleg.crontab | 19 ++++++++ qcow2/cron/example.root.crontab | 8 ++++ 7 files changed, 222 insertions(+) create mode 100644 qcow2/.gitignore create mode 100755 qcow2/backup_vm.sh create mode 100755 qcow2/clean_shrunk.sh create mode 100755 qcow2/compress_qcow2.sh create mode 100755 qcow2/compress_qcow2_file.sh create mode 100644 qcow2/cron/example.oleg.crontab create mode 100644 qcow2/cron/example.root.crontab diff --git a/qcow2/.gitignore b/qcow2/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/qcow2/backup_vm.sh b/qcow2/backup_vm.sh new file mode 100755 index 0000000..1ad62c9 --- /dev/null +++ b/qcow2/backup_vm.sh @@ -0,0 +1,39 @@ +#!/bin/bash +echo Starting $0 $* + +VmName=${1} + +# --------------------------------------------------- +# Cronjob requires explicit connection to local server +# --------------------------------------------------- +VirshCmd="/usr/bin/virsh -c qemu:///system" + +if [ "${VmName}" == "" ] +then + echo "Usage: $0 " + exit 1 +fi + +Today=$(date +%Y%m%d) + +BackupDir=/localdisk/backup/qcow2 +GuestQcow2Files=$(${VirshCmd} domblklist ${VmName} | grep qcow2 | awk '{print $2}') + +Cmd="mkdir -p ${BackupDir}" +echo ${Cmd} && eval ${Cmd} + +Cmd="${VirshCmd} suspend ${VmName}" +echo ${Cmd} && eval ${Cmd} + +for qcow2_file in ${GuestQcow2Files} +do + Cmd="cp ${qcow2_file} ${BackupDir}/${Today}.$(basename ${qcow2_file})" + echo ${Cmd} && eval ${Cmd} +done + +Cmd="${VirshCmd} resume ${VmName}" +echo ${Cmd} && eval ${Cmd} + +echo "$0 Done." + + diff --git a/qcow2/clean_shrunk.sh b/qcow2/clean_shrunk.sh new file mode 100755 index 0000000..7d95e1b --- /dev/null +++ b/qcow2/clean_shrunk.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +print_and_run() { + local cmd="$*" + printf '➜ %s\n' "$cmd" + # pipefail applies to the subshell; change/remove if not desired + bash -o pipefail -c "$cmd" +} + + +SHRUNK="/opt/vm_drive/shrunk" +LINKS="/opt/vm_drive/*.qcow2" + ACTION="echo Would remove" # default: dry-run + +DRYRUN=0 +if [[ "$1" == "--dry-run" ]]; then + DRYRUN=1 + shift +fi + +find "$SHRUNK" -maxdepth 1 -type f -printf '%f\n' \ + | grep -vxFf <(readlink -f $LINKS | xargs -n1 basename) \ + | while read -r f; do + if (( DRYRUN )); then + echo "Would remove: $SHRUNK/$f" + else + echo "Removing: $SHRUNK/$f" + rm -v -- "$SHRUNK/$f" + fi + done diff --git a/qcow2/compress_qcow2.sh b/qcow2/compress_qcow2.sh new file mode 100755 index 0000000..ca07839 --- /dev/null +++ b/qcow2/compress_qcow2.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +# ------ Settings +Qcow2Dir=/localdisk/backup/qcow2 +ShrunkDir=/opt/vm_drive/shrunk +TmpDir=/localdisk/tmp + +mkdir -p ${ShrunkDir} +mkdir -p ${TmpDir} +# ------ Settings + +Qcow2Files=$(find ${Qcow2Dir} -type f -name "*.qcow2" -mtime -1 -print) +echo "Files to compress:" +echo ${Qcow2Files} + +for file in ${Qcow2Files} +do + echo "Compressing file ${file}..." + SparseFile="${ShrunkDir}/$(basename ${file}).sparse" + ShrunkFile="${ShrunkDir}/$(basename ${file}).shrunk" + if [ -f ${ShrunkFile} ] + then + echo Shrunk file ${ShrunkFile} exists. Skipping... + continue + fi + Cmd="/usr/bin/virt-sparsify" + Cmd="${Cmd} --check-tmpdir=continue" + Cmd="${Cmd} --tmp=${TmpDir}" + Cmd="${Cmd} -q" + Cmd="${Cmd} ${file} ${SparseFile}" + Cmd="${Cmd} && /usr/bin/qemu-img convert" + Cmd="${Cmd} -c" + Cmd="${Cmd} -O qcow2" + Cmd="${Cmd} ${SparseFile} ${ShrunkFile}" + echo ${Cmd} + eval ${Cmd} + + if [ ${?} -ne 0 ] + then + echo "Compression failed. Removing ${ShrunkFile}" + rm ${ShrunkFile} + else + filesz=$(stat -c %s ${file}) + cfilesz=$(stat -c %s ${ShrunkFile}) + + ratio=$((cfilesz * 100 / filesz)) + echo "Compression is successful. Compression ratio is ${ratio}%" + rm ${SparseFile} + + ## *************************************************************************** + ## + ## Switching symlink will mask latest changes for future backups + ## if VM was not restarted. + ## This must be done manually before restarting - pick the latest shrunk + ## symlink it and restart VM + ## + ## *************************************************************************** + + # GuestFile="${ShrunkFile#*.}" + # GuestFile="${GuestFile%.shrunk}" + # Cmd="ln -snf ${ShrunkFile} /opt/vm_drive/${GuestFile}" + # echo ${Cmd} + # eval ${Cmd} + ## + Cmd="chown oleg:oleg ${ShrunkFile}" + echo ${Cmd} + eval ${Cmd} + fi +done + +echo "Done $0 $*" + diff --git a/qcow2/compress_qcow2_file.sh b/qcow2/compress_qcow2_file.sh new file mode 100755 index 0000000..5d28173 --- /dev/null +++ b/qcow2/compress_qcow2_file.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# ============ R e q u i r e s package guestfs-tools + +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +# ------ Settings +TmpDir=/localdisk/tmp + +# ------ Settings + +mkdir -p ${TmpDir} +file=${1} +out_file=${2} + +SparseFile="${out_file}.sparse" +ShrunkFile="${out_file}.shrunk" + +echo "Compressing file ${file}..." +Cmd="/usr/bin/virt-sparsify" +Cmd="${Cmd} --check-tmpdir=continue" +Cmd="${Cmd} --tmp=${TmpDir}" +Cmd="${Cmd} -q" +Cmd="${Cmd} ${file} ${SparseFile}" +Cmd="${Cmd} && /usr/bin/qemu-img convert" +Cmd="${Cmd} -c" +Cmd="${Cmd} -O qcow2" +Cmd="${Cmd} ${SparseFile} ${ShrunkFile}" +echo ${Cmd} +eval ${Cmd} + +if [ ${?} -ne 0 ] +then + echo "Compression failed. Removing ${ShrunkFile}" + rm ${ShrunkFile} +else + filesz=$(stat -c %s ${file}) + cfilesz=$(stat -c %s ${ShrunkFile}) + + ratio=$((cfilesz * 100 / filesz)) + echo "Compression is successful. Compression ratio is ${ratio}%" + rm ${SparseFile} +fi + +echo "Done $0 $*" + diff --git a/qcow2/cron/example.oleg.crontab b/qcow2/cron/example.oleg.crontab new file mode 100644 index 0000000..61d7fc2 --- /dev/null +++ b/qcow2/cron/example.oleg.crontab @@ -0,0 +1,19 @@ +#=========================================================================== +# -------- T e s t +# -------- T e s t +#=========================================================================== +10 2 * * MON,FRI /works/admin/backup_vm.sh desktop 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.desktop.log 2>&1 +10 4 * * MON,FRI /works/admin/backup_vm.sh home-fin 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.home-fin.log 2>&1 +# +10 2 * * TUE /works/admin/backup_vm.sh allbright-dev 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.allbright-dev.log 2>&1 +10 4 * * TUE,FRI /works/admin/backup_vm.sh develop-desktop-02 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1 +# +10 2 * * WED,SAT /works/admin/backup_vm.sh cvtt-admin 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.cvtt-admin.log 2>&1 +10 4 * * WED,SAT /works/admin/backup_vm.sh edu-desktop-02 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.edu-desktop-02.log 2>&1 +# +10 2 * * THU,SUN /works/admin/backup_vm.sh develop-desktop-02 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1 +#---------------------- Utils +23 23 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D "1 day ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).archive_logs.works.log +10 1 * * * /works/utils/prune_data.sh -d /opt/vm_drive/shrunk -D "2 weeks ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_shrunk.log +10 2 * * * /works/utils/prune_data.sh -d /localdisk/backup/qcow2 -D "2 months ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_backup.log + diff --git a/qcow2/cron/example.root.crontab b/qcow2/cron/example.root.crontab new file mode 100644 index 0000000..bb042a8 --- /dev/null +++ b/qcow2/cron/example.root.crontab @@ -0,0 +1,8 @@ +#=========================================================================== +# -------- T e s t +# -------- T e s t +#=========================================================================== +10 0 * * * /works/admin/compress_qcow2.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.compress_qcow2.log 2>&1 +### +* * * * * chown -R oleg:oleg /works 2>&1 +#