progress

qcow2/download_shrunk.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -o pipefail
+
+echo "*** START $0 ${*}"
+
+# ------- Settings
+MAX_AGE_DAYS=1
+
+SourceHost=hl-storage.cvtt.vpn
+SourceDir=/works/qcow2/shrunk
+
+DownloadTgtDir=/opt/vm_drive/shrunk
+
+# ------- Settings
+
+download_latest() {
+    local ssh_opts=(
+        -o StrictHostKeyChecking=accept-new
+    )
+
+    local rsync_rsh="ssh -o StrictHostKeyChecking=accept-new"
+
+    local FindCmd=(
+        ssh
+        "${ssh_opts[@]}"
+        "$SourceHost"
+        find "$SourceDir"
+        -type f
+        "-mtime -${MAX_AGE_DAYS}"
+        "-printf '%P\0'"
+    )
+
+    local RsyncCmd=(
+        rsync 
+        -avh 
+        --progress 
+        --partial 
+        --append-verify 
+        -n
+        --from0
+        --files-from=-
+        "-e $rsync_rsh"
+        "${SourceHost}:${SourceDir}/"
+        "${DownloadTgtDir}/"
+    )
+
+    echo "-------------------"
+    echo "Running:"
+    set -x
+    "${FindCmd[@]}" | "${RsyncCmd[@]}"
+    { set +x; } 2>/dev/null
+}
+
+echo "-------------------"
+echo "Downloading files from ${SourceHost}:${SourceDir}"
+download_latest 
+echo "-------------------"
+echo "Download completed"
+
+echo "*** DONE: ${0} ${*}"

qcow2/sparsify_qcow2.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+set -o pipefail
+
+echo "*** START $0 ${*}"
+
+# ------- Settings
+MAX_AGE_DAYS=1
+
+SourceHost=office-oleg-02.cvtt.vpn
+SourceDir=/localdisk/backup/qcow2
+
+DownloadTgtDir=/mnt/host_usb3/qcow2
+
+ShrunkDir=/mnt/host_usb3/qcow2/shrunk
+TmpDir=/mnt/host_usb3/tmp
+# ------- Settings
+
+download_latest() {
+    local ssh_opts=(
+        -o StrictHostKeyChecking=accept-new
+    )
+
+    local rsync_rsh="ssh -o StrictHostKeyChecking=accept-new"
+
+    local FindCmd=(
+        ssh
+        "${ssh_opts[@]}"
+        "$SourceHost"
+        find "$SourceDir"
+        -type f
+        "-mtime -${MAX_AGE_DAYS}"
+        "-printf '%P\0'"
+    )
+
+    local RsyncCmd=(
+        rsync 
+        # -n
+        -avh 
+        --progress 
+        --partial 
+        --append-verify 
+        --from0
+        --files-from=-
+        "-e $rsync_rsh"
+        "${SourceHost}:${SourceDir}/"
+        "${DownloadTgtDir}/"
+    )
+
+    echo "-------------------"
+    echo "Running:"
+    set -x
+    "${FindCmd[@]}" | "${RsyncCmd[@]}"
+    { set +x; } 2>/dev/null
+}
+
+compress_qcow2_file() {
+    local file=${1}
+    local out_file=${2}
+
+    mkdir -p "${ShrunkDir}"
+    mkdir -p "${TmpDir}"
+
+    local ShrunkFile="${out_file}.shrunk"
+
+    echo "Sparsifying file ${file}..."
+    local Cmd="/usr/bin/virt-sparsify"
+    Cmd+=" --check-tmpdir=fail"
+    Cmd+=" --tmp=${TmpDir}"
+    Cmd+=" --compress"
+    if [[ ! -t 1 ]]; then
+        Cmd+=" -q"
+    fi
+    Cmd+=" ${file} ${ShrunkFile}"
+    echo "${Cmd}"
+    ${Cmd}
+    
+    if [ ${?} -ne 0 ] ; then
+        echo "Compression failed. Removing ${ShrunkFile}"
+        rm "${ShrunkFile}"
+        return 255
+    else
+        filesz=$(stat -c %s "${file}")
+        if [ "${filesz}" == "0" ]; then
+            return 254
+        fi
+        cfilesz=$(stat -c %s "${ShrunkFile}")
+
+        ratio=$((cfilesz * 100 / filesz))
+        echo "Compression is successful. Compression ratio is ${ratio}%"
+    fi
+    echo "*** DONE: $file -> $ShrunkFile"
+    return 0
+}
+
+
+echo "-------------------"
+echo "Downloading files from ${SourceHost}:${SourceDir}"
+download_latest 
+# exit
+echo "-------------------"
+echo "Download completed"
+
+for qcow2_file in "${DownloadTgtDir}"/*.qcow2; do
+    shrunk_file="${ShrunkDir}/$(basename "${qcow2_file}")"
+    compress_qcow2_file "${qcow2_file}" "${shrunk_file}"
+    if [ ${?} -ne 0 ] ; then
+        echo "ERROR: File ${qcow2_file} compression FAILED"
+    else
+        echo "SUCCESS: File ${qcow2_file} is compressed to ${shrunk_file}"
+        echo "Removing file ${qcow2_file} ..."
+        rm  "${qcow2_file}"
+    fi    
+done
+
+echo "*** DONE $0 ${*}"