Merge branch 'master' of https://gitea.cvtt.net/prod/utils

archive_logs.sh

@@ -16,9 +16,10 @@ LogDir=${1}
 LogArchiveDir=/works/archive/logs
 DateCriteria="2 days ago"
 ExtraWildCard=
+AddHostname=
 
 # ---------------- cmdline
-while getopts "A:L:D:W:" opt; do
+while getopts "A:L:D:W:H" opt; do
   case ${opt} in
     A )
       LogArchiveDir=$OPTARG
@@ -32,6 +33,9 @@ while getopts "A:L:D:W:" opt; do
     W )
       ExtraWildCard=$OPTARG
       ;;
+    H )
+      AddHostname=Y
+      ;;
     \? )
       echo "Invalid option: -$OPTARG" >&2
       usage
@@ -74,19 +78,22 @@ if [ "$files" == "" ]
 then
     echo "No files found older than ${Oldest} in ${LogDir}"
 else
-    echo Archiving files:
-    echo -----------------
-    echo ${files}
-
     oldest_file=$(ls -t ${files}  | tail -1)
     youngest_file=$(ls -tr ${files}  | tail -1)
 
     tstmp1=$(stat --printf '%y' ${oldest_file})
     tstmp2=$(stat --printf '%y' ${youngest_file})
     tmsig="$(date -d "${tstmp1}" +"%Y%m%d_%H%M%S")-$(date -d "${tstmp2}" +"%Y%m%d_%H%M%S")"
-    echo $tmsig
 
-    Cmd="tar zcvf ${LogArchiveDir}/${tmsig}.logs.tgz ${files}"
+    TarFile="${LogArchiveDir}/${tmsig}.logs.tgz"
+    if [ "${AddHostname}" == "Y" ] ; then
+        TarFile="${LogArchiveDir}/${tmsig}.$(hostname -s).logs.tgz"
+    fi
+    echo Archiving to ${TarFile} files:
+    echo -----------------
+    echo ${files}
+
+    Cmd="tar zcvf ${TarFile} ${files}"
     echo $Cmd
     eval $Cmd
 

backup/backup_by_json.sh

@@ -139,6 +139,9 @@ main() {
         exit 1
     fi
 
+    echo "---------- ${config_file}"
+    cat ${config_file}
+    echo "----------"
     local dt
     dt=$(date +%Y%m%d)
 
@@ -183,5 +186,6 @@ if [[ $# -lt 1 ]]; then
     exit 1
 fi
 
+echo "[$(date)] Starting ${0} ${*} ..." 
 main "${Config}"
-
+echo "[$(date)] Done ${0} ${*}" 

backup/examples/cvtt_md_archive-2.json

@@ -0,0 +1,18 @@
+{
+    "Name" : "cvtt_md_archive_2",
+    "Host" : "",
+
+    "BackupHost" : "",
+    "BackupRootDir" : "/mnt/usb2",
+
+    "Filesystems" : ["/works/cvtt/md_archive" ],
+
+    "RsyncOptions" : ["-arxv"],
+
+    "Exclude" : [],
+
+    "DailySnapshots" : false,
+    "PruneBackupDays" : 0
+
+}
+

backup/examples/cvtt_md_archive.json

@@ -0,0 +1,18 @@
+{
+    "Name" : "cvtt_md_archive",
+    "Host" : "",
+
+    "BackupHost" : "",
+    "BackupRootDir" : "/mnt/usb1",
+
+    "Filesystems" : ["/works/cvtt/md_archive" ],
+
+    "RsyncOptions" : ["-arxv"],
+
+    "Exclude" : [],
+
+    "DailySnapshots" : false,
+    "PruneBackupDays" : 0
+
+}
+

backup/examples/hs01_local_backup.json

@@ -0,0 +1,18 @@
+{
+    "Name" : "hs01",
+    "Host" : "",
+
+    "BackupHost" : "",
+    "BackupRootDir" : "/backup/hs01",
+
+    "Filesystems" : ["/etc", "/opt", "/archive", "/data", "/works" ],
+
+    "RsyncOptions" : ["-arxv", "--delete", "--relative" ],
+
+    "Exclude" : [],
+
+    "DailySnapshots" : false,
+    "PruneBackupDays" : 0
+
+}
+

backup/examples/hs01_to_hbkp01_backup.json

@@ -0,0 +1,17 @@
+{
+    "Name" : "hs02_to_hbkp01",
+    "Host" : "hs01.cvtt.vpn",
+
+    "BackupHost" : "",
+    "BackupRootDir" : "/backup/hs01",
+
+    "Filesystems" : ["/backup" ],
+
+    "RsyncOptions" : ["-arxv", "--delete", "--relative" ],
+
+    "Exclude" : [],
+
+    "DailySnapshots" : true,
+    "PruneBackupDays" : 90
+
+}

backup/examples/oo02-guests-backup.json

@@ -0,0 +1,18 @@
+{
+    "Name" : "office-oleg-02_guests",
+    "Host" : "office-oleg-02.cvtt.vpn",
+
+    "BackupHost" : "",
+    "BackupRootDir" : "/backup/office-oleg-02/guests",
+
+    "Filesystems" : ["/localdisk/guests/" ],
+
+    "RsyncOptions" : ["-arxv", "--delete", "--copy-links"],
+
+    "Exclude" : [],
+
+    "DailySnapshots" : false,
+    "PruneBackupDays" : 0
+
+}
+

backup/examples/oo02-qcow2-backup.json

@@ -0,0 +1,18 @@
+{
+    "Name" : "office-oleg-02_qcow2",
+    "Host" : "office-oleg-02.cvtt.vpn",
+
+    "BackupHost" : "",
+    "BackupRootDir" : "/backup/office-oleg-02/qcow2",
+
+    "Filesystems" : ["/localdisk/backup/qcow2/" ],
+
+    "RsyncOptions" : ["-arxv", "--delete", "--copy-links"],
+
+    "Exclude" : [],
+
+    "DailySnapshots" : false,
+    "PruneBackupDays" : 2
+
+}
+

cron/example.crontab

@@ -0,0 +1,16 @@
+# ============================================================
+#
+# ---------- pruning md_archive
+10 3 * * * /works/utils/prune_data.sh -d /works/cvtt/md_archive -D '6 months ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date '+\%Y\%m\%d').prune_md_archive.log
+#
+# ---------- U t i l s
+#
+# ---------- archiving logs
+15 6 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D 'week ago' -H 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).archive_logs.log
+#
+# ---------- moving archives to NAS
+15 7 * * * /works/utils/move_archives.sh  -D 'week ago' 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).move_archives.log
+#
+# ============================================================
+# ---------- T e s t
+# ---------- T e s t

move_archives.sh

@@ -3,9 +3,9 @@
 
 function usage {
     echo -n "Usage: ${0}"
-    echo -n " -H <host_label>"
     echo -n " [ -A <archive_dir> (default /works/archive)]"
-    echo -n " [-D <older than time criteria> (default: '2 days ago')]"
+    echo -n " [ -T <target (default hl-storage.cvtt.vpn:/works/archive)>]"
+    echo -n " [ -D <older than time criteria> (default: '2 days ago')]"
     echo
     exit 1
 }
@@ -16,20 +16,24 @@ echo Starting $0 $*
 ArchiveDir=/works/archive
 DateCriteria="2 days ago"
 FromHost=$(hostname -s)
+TargetRoot=hl-storage.cvtt.vpn:/works/archive
 # ---- D e f a u l t s
 
 # ---------------- cmdline
-while getopts "A:H:D:" opt; do
+while getopts "A:T:D:h" opt; do
   case ${opt} in
     A )
       ArchiveDir=$OPTARG
       ;;
-    H )
+    T )
-      FromHost=$OPTARG
+      TargetRoot=$OPTARG
       ;;
     D )
       DateCriteria=$OPTARG
       ;;
+    h )
+      usage
+      ;;
     \? )
       echo "Invalid option: -$OPTARG" >&2
       usage
@@ -42,50 +46,34 @@ while getopts "A:H:D:" opt; do
 done
 # ---------------- cmdline
 
-
-if [ "${FromHost}" == "" ]
-then
-    usage
-fi
-
-TargetHost=cloud21.cvtt.vpn
-TargetRootDir=/opt/store/cvtt/archive
-
 Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S')
 
 echo "Looking for log files older than ${DateCriteria} in ${ArchiveDir}"
-Cmd="find ${ArchiveDir}/"
-Cmd="${Cmd} '('"
-Cmd="${Cmd} -name '*.log'"
-Cmd="${Cmd} -o -name '*.log.*'"
-Cmd="${Cmd} -o -name '*.logs.*'"
-Cmd="${Cmd} -o -name '*.tgz'"
-Cmd="${Cmd} ')'"
-Cmd="${Cmd} -type f"
-Cmd="${Cmd} -not -newermt \"${Oldest}\""
 
-echo ${Cmd}
-files=$(eval ${Cmd})
 
-if [ "$files" == "" ]
+# 1. First, check if any matching files exist using a quick find check
-then
+if ! find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -print -quit | grep -q .; then
     echo "No files found older than ${Oldest} in ${ArchiveDir}"
-else
+    echo "Done ${0} ${*}"
-    Target="${TargetHost}:${TargetRootDir}/${FromHost}/"
+    exit 0
-    echo "Moving files to ${Target}:"
-    echo -----------------
-    for f in ${files}
-    do
-        echo ${f}
-    done
-
-    Cmd="rsync -ahvv"
-    Cmd="${Cmd} --remove-source-files"
-    Cmd="${Cmd} $files"
-    Cmd="${Cmd} ${Target}"
-    echo ${Cmd}
-# exit
-    eval ${Cmd}
 fi
+
+Target="${TargetRoot}/${FromHost}/"
+echo "Moving files to ${Target}:"
+echo "-----------------"
+
+# 2. Safely pipe find into rsync using relative paths from the ArchiveDir base
+# This completely avoids "Argument list too long" errors and handles spaces perfectly
+set -x
+find "${ArchiveDir}/" -type f -not -newermt "${Oldest}" -printf "%P\0" | \
+    rsync -ahvv \
+          -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+          --remove-source-files \
+          --mkpath \
+          --from0 \
+          --files-from=- \
+          "${ArchiveDir}/" \
+          "${Target}"
+{ set +x; } 2>/dev/null
 echo Done ${0} ${*}
 

qcow2/backup_vm.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+echo Starting $0 $*
+
+VmName=${1}
+
+# ---------------------------------------------------
+# Cronjob requires explicit connection to local server
+# ---------------------------------------------------
+VirshCmd="/usr/bin/virsh -c qemu:///system"
+
+if  [ "${VmName}" == "" ]
+then
+    echo "Usage: $0 <VM Domain Name>"
+    exit 1
+fi
+
+Today=$(date +%Y%m%d)
+
+BackupDir=/localdisk/backup/qcow2
+GuestQcow2Files=$(${VirshCmd} domblklist ${VmName} | grep qcow2 | awk '{print $2}')
+
+Cmd="mkdir -p ${BackupDir}"
+echo ${Cmd} && eval ${Cmd}
+
+Cmd="${VirshCmd} suspend ${VmName}"
+echo ${Cmd} && eval ${Cmd}
+
+for qcow2_file in ${GuestQcow2Files}
+do
+    Cmd="cp ${qcow2_file} ${BackupDir}/${Today}.$(basename ${qcow2_file})"
+    echo ${Cmd} && eval ${Cmd}
+done
+
+Cmd="${VirshCmd} resume ${VmName}"
+echo ${Cmd} && eval ${Cmd}
+
+echo "$0 Done."
+
+

qcow2/clean_shrunk.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+print_and_run() {
+    local cmd="$*"
+    printf '➜ %s\n' "$cmd"
+    # pipefail applies to the subshell; change/remove if not desired
+    bash -o pipefail -c "$cmd"
+}
+
+
+SHRUNK="/opt/vm_drive/shrunk"
+LINKS="/opt/vm_drive/*.qcow2"
+ ACTION="echo Would remove"   # default: dry-run
+
+DRYRUN=0
+if [[ "$1" == "--dry-run" ]]; then
+    DRYRUN=1
+    shift
+fi
+
+find "$SHRUNK" -maxdepth 1 -type f -printf '%f\n' \
+    | grep -vxFf <(readlink -f $LINKS | xargs -n1 basename) \
+    | while read -r f; do
+        if (( DRYRUN )); then
+          echo "Would remove: $SHRUNK/$f"
+        else
+          echo "Removing: $SHRUNK/$f"
+          rm -v -- "$SHRUNK/$f"
+        fi
+      done

qcow2/compress_qcow2.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+if [ "$(id -u)" -ne 0 ]; then
+    echo "This script must be run as root." >&2
+    exit 1
+fi
+
+# ------ Settings
+Qcow2Dir=/localdisk/backup/qcow2
+ShrunkDir=/opt/vm_drive/shrunk
+TmpDir=/localdisk/tmp
+
+mkdir -p ${ShrunkDir}
+mkdir -p ${TmpDir}
+# ------ Settings
+
+Qcow2Files=$(find ${Qcow2Dir} -type f -name "*.qcow2" -mtime -1 -print)
+echo "Files to compress:"
+echo ${Qcow2Files}
+
+for file in ${Qcow2Files}
+do 
+    echo "Compressing file ${file}..."
+    SparseFile="${ShrunkDir}/$(basename ${file}).sparse"
+    ShrunkFile="${ShrunkDir}/$(basename ${file}).shrunk"
+    if [ -f ${ShrunkFile} ]
+    then
+	echo Shrunk file ${ShrunkFile} exists. Skipping...
+        continue
+    fi
+    Cmd="/usr/bin/virt-sparsify"
+    Cmd="${Cmd} --check-tmpdir=continue"
+    Cmd="${Cmd} --tmp=${TmpDir}"
+    Cmd="${Cmd} -q"
+    Cmd="${Cmd} ${file} ${SparseFile}"
+    Cmd="${Cmd} && /usr/bin/qemu-img convert"
+    Cmd="${Cmd} -c"
+    Cmd="${Cmd} -O qcow2"
+    Cmd="${Cmd} ${SparseFile} ${ShrunkFile}"
+    echo ${Cmd}
+    eval ${Cmd}
+
+    if [ ${?} -ne 0 ]
+    then
+        echo "Compression failed. Removing ${ShrunkFile}"
+        rm ${ShrunkFile}
+    else
+        filesz=$(stat -c %s ${file})
+        cfilesz=$(stat -c %s ${ShrunkFile})
+
+        ratio=$((cfilesz * 100 / filesz))
+        echo "Compression is successful. Compression ratio is ${ratio}%"
+        rm ${SparseFile}
+
+        ## ***************************************************************************
+        ##
+        ## Switching symlink will mask latest changes for future backups
+        ## if VM was not restarted.
+        ## This must be done manually before restarting - pick the latest shrunk
+        ## symlink it and restart VM
+        ##
+        ## ***************************************************************************
+
+        # GuestFile="${ShrunkFile#*.}"
+        # GuestFile="${GuestFile%.shrunk}"
+        # Cmd="ln -snf ${ShrunkFile} /opt/vm_drive/${GuestFile}"
+        # echo ${Cmd}
+        # eval ${Cmd}
+        ##
+        Cmd="chown oleg:oleg ${ShrunkFile}"
+        echo ${Cmd}
+        eval ${Cmd}
+    fi
+done
+    
+echo "Done $0 $*"
+

qcow2/compress_qcow2_file.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# ============ R e q u i r e s  package guestfs-tools
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "This script must be run as root." >&2
+  exit 1
+fi
+
+# ------ Settings
+TmpDir=/localdisk/tmp
+
+# ------ Settings
+
+mkdir -p ${TmpDir}
+file=${1}
+out_file=${2}
+
+SparseFile="${out_file}.sparse"
+ShrunkFile="${out_file}.shrunk"
+
+echo "Compressing file ${file}..."
+Cmd="/usr/bin/virt-sparsify"
+Cmd="${Cmd} --check-tmpdir=continue"
+Cmd="${Cmd} --tmp=${TmpDir}"
+Cmd="${Cmd} -q"
+Cmd="${Cmd} ${file} ${SparseFile}"
+Cmd="${Cmd} && /usr/bin/qemu-img convert"
+Cmd="${Cmd} -c"
+Cmd="${Cmd} -O qcow2"
+Cmd="${Cmd} ${SparseFile} ${ShrunkFile}"
+echo ${Cmd}
+eval ${Cmd}
+
+if [ ${?} -ne 0 ]
+then
+    echo "Compression failed. Removing ${ShrunkFile}"
+    rm ${ShrunkFile}
+else
+    filesz=$(stat -c %s ${file})
+    cfilesz=$(stat -c %s ${ShrunkFile})
+
+    ratio=$((cfilesz * 100 / filesz))
+    echo "Compression is successful. Compression ratio is ${ratio}%"
+    rm ${SparseFile}
+fi
+    
+echo "Done $0 $*"
+

qcow2/cron/example.oleg.crontab

@@ -0,0 +1,19 @@
+#===========================================================================
+# -------- T e s t
+# -------- T e s t
+#===========================================================================
+10 2 * * MON,FRI /works/admin/backup_vm.sh desktop 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.desktop.log 2>&1
+10 4 * * MON,FRI /works/admin/backup_vm.sh home-fin 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.home-fin.log 2>&1
+#
+10 2 * * TUE /works/admin/backup_vm.sh allbright-dev 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.allbright-dev.log 2>&1
+10 4 * * TUE,FRI /works/admin/backup_vm.sh develop-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1
+#
+10 2 * * WED,SAT /works/admin/backup_vm.sh cvtt-admin 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.cvtt-admin.log 2>&1
+10 4 * * WED,SAT /works/admin/backup_vm.sh edu-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.edu-desktop-02.log 2>&1
+#
+10 2 * * THU,SUN /works/admin/backup_vm.sh develop-desktop-02 2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.develop-desktop-02.log 2>&1
+#---------------------- Utils
+23 23 * * * /works/utils/archive_logs.sh -L /works/logs -A /works/archive/logs -D "1 day ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).archive_logs.works.log 
+10 1 * * * /works/utils/prune_data.sh -d /opt/vm_drive/shrunk -D "2 weeks ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_shrunk.log 
+10 2 * * * /works/utils/prune_data.sh -d /localdisk/backup/qcow2 -D "2 months ago" 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /works/logs/$(date +\%Y\%m\%d).prune_qcow2_backup.log 
+

qcow2/cron/example.root.crontab

@@ -0,0 +1,8 @@
+#===========================================================================
+# -------- T e s t
+# -------- T e s t
+#===========================================================================
+10 0 * * * /works/admin/compress_qcow2.sh  2>&1 |  /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]' >> /works/logs/$(date +\%Y\%m\%d).backup_vm.compress_qcow2.log 2>&1
+###
+* * * * *   chown -R oleg:oleg /works 2>&1
+#

qcow2/download_shrunk.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -o pipefail
+
+echo "*** START $0 ${*}"
+
+# ------- Settings
+MAX_AGE_DAYS=1
+
+SourceHost=hl-storage.cvtt.vpn
+SourceDir=/works/qcow2/shrunk
+
+DownloadTgtDir=/opt/vm_drive/shrunk
+
+# ------- Settings
+
+download_latest() {
+    local ssh_opts=(
+        -o StrictHostKeyChecking=accept-new
+    )
+
+    local rsync_rsh="ssh -o StrictHostKeyChecking=accept-new"
+
+    local FindCmd=(
+        ssh
+        "${ssh_opts[@]}"
+        "$SourceHost"
+        find "$SourceDir"
+        -type f
+        "-mtime -${MAX_AGE_DAYS}"
+        "-printf '%P\0'"
+    )
+
+    local RsyncCmd=(
+        rsync 
+        -avh 
+        --progress 
+        --partial 
+        --append-verify 
+        -n
+        --from0
+        --files-from=-
+        "-e $rsync_rsh"
+        "${SourceHost}:${SourceDir}/"
+        "${DownloadTgtDir}/"
+    )
+
+    echo "-------------------"
+    echo "Running:"
+    set -x
+    "${FindCmd[@]}" | "${RsyncCmd[@]}"
+    { set +x; } 2>/dev/null
+}
+
+echo "-------------------"
+echo "Downloading files from ${SourceHost}:${SourceDir}"
+download_latest 
+echo "-------------------"
+echo "Download completed"
+
+echo "*** DONE: ${0} ${*}"

qcow2/sparsify_qcow2.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+set -o pipefail
+
+echo "*** START $0 ${*}"
+
+# ------- Settings
+MAX_AGE_DAYS=1
+
+SourceHost=office-oleg-02.cvtt.vpn
+SourceDir=/localdisk/backup/qcow2
+
+DownloadTgtDir=/mnt/host_usb3/qcow2
+
+ShrunkDir=/mnt/host_usb3/qcow2/shrunk
+TmpDir=/mnt/host_usb3/tmp
+# ------- Settings
+
+download_latest() {
+    local ssh_opts=(
+        -o StrictHostKeyChecking=accept-new
+    )
+
+    local rsync_rsh="ssh -o StrictHostKeyChecking=accept-new"
+
+    local FindCmd=(
+        ssh
+        "${ssh_opts[@]}"
+        "$SourceHost"
+        find "$SourceDir"
+        -type f
+        "-mtime -${MAX_AGE_DAYS}"
+        "-printf '%P\0'"
+    )
+
+    local RsyncCmd=(
+        rsync 
+        # -n
+        -avh 
+        --progress 
+        --partial 
+        --append-verify 
+        --from0
+        --files-from=-
+        "-e $rsync_rsh"
+        "${SourceHost}:${SourceDir}/"
+        "${DownloadTgtDir}/"
+    )
+
+    echo "-------------------"
+    echo "Running:"
+    set -x
+    "${FindCmd[@]}" | "${RsyncCmd[@]}"
+    { set +x; } 2>/dev/null
+}
+
+compress_qcow2_file() {
+    local file=${1}
+    local out_file=${2}
+
+    mkdir -p "${ShrunkDir}"
+    mkdir -p "${TmpDir}"
+
+    local ShrunkFile="${out_file}.shrunk"
+
+    echo "Sparsifying file ${file}..."
+    local Cmd="/usr/bin/virt-sparsify"
+    Cmd+=" --check-tmpdir=fail"
+    Cmd+=" --tmp=${TmpDir}"
+    Cmd+=" --compress"
+    if [[ ! -t 1 ]]; then
+        Cmd+=" -q"
+    fi
+    Cmd+=" ${file} ${ShrunkFile}"
+    echo "${Cmd}"
+    ${Cmd}
+    
+    if [ ${?} -ne 0 ] ; then
+        echo "Compression failed. Removing ${ShrunkFile}"
+        rm "${ShrunkFile}"
+        return 255
+    else
+        filesz=$(stat -c %s "${file}")
+        if [ "${filesz}" == "0" ]; then
+            return 254
+        fi
+        cfilesz=$(stat -c %s "${ShrunkFile}")
+
+        ratio=$((cfilesz * 100 / filesz))
+        echo "Compression is successful. Compression ratio is ${ratio}%"
+    fi
+    echo "*** DONE: $file -> $ShrunkFile"
+    return 0
+}
+
+
+echo "-------------------"
+echo "Downloading files from ${SourceHost}:${SourceDir}"
+download_latest 
+# exit
+echo "-------------------"
+echo "Download completed"
+
+for qcow2_file in "${DownloadTgtDir}"/*.qcow2; do
+    shrunk_file="${ShrunkDir}/$(basename "${qcow2_file}")"
+    compress_qcow2_file "${qcow2_file}" "${shrunk_file}"
+    if [ ${?} -ne 0 ] ; then
+        echo "ERROR: File ${qcow2_file} compression FAILED"
+    else
+        echo "SUCCESS: File ${qcow2_file} is compressed to ${shrunk_file}"
+        echo "Removing file ${qcow2_file} ..."
+        rm  "${qcow2_file}"
+    fi    
+done
+
+echo "*** DONE $0 ${*}"

send_mmost.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+channel_name=${1}
+if [ "" == "${channel_name}" ]
+then
+    echo "Usage: ${0} <channel_name>"
+	exit 1
+fi
+
+url=$(jq -r '.MATTERMOST.url' ${HOME}/.creds)
+team=$(jq -r '.MATTERMOST.team' ${HOME}/.creds)
+bearer=$(jq -r '.MATTERMOST.bearer' ${HOME}/.creds)
+
+#lowercase channel
+channel_name=$(echo "${channel_name}" | awk '{print tolower($0)}')
+# space to '-'
+channel_name=${channel_name//" "/"-"}
+
+get_channel_url="${url}/api/v4/teams/name/${team}/channels/name/${channel_name}"
+
+Cmd="curl -s"
+Cmd="${Cmd} -X GET"
+Cmd="${Cmd} -H \"Content-Type: Application/JSON\""
+Cmd="${Cmd} -H \"Authorization: Bearer ${bearer}\""
+Cmd="${Cmd} ${get_channel_url}"
+Cmd="${Cmd} | jq -r '.id'"
+
+channel_id=$(eval ${Cmd})
+
+input="$(cat)"
+
+body="{"
+body="${body}\"channel_id\":\"${channel_id}\""
+body="${body}, \"message\":\"${input}\""
+body="${body}}"
+
+# replace new line with literal \n
+body=$(echo "${body}" |awk '{printf "%s\\n", $0}')
+
+post_url="${url}/api/v4/posts"
+
+Cmd="curl -s"
+Cmd="${Cmd} --request POST"
+Cmd="${Cmd} --header \"Content-Type: Application/JSON\""
+Cmd="${Cmd} --header \"Authorization: Bearer ${bearer}\""
+Cmd="${Cmd} --data @-"
+Cmd="${Cmd} ${post_url}"
+
+(echo "${body}" | eval ${Cmd}) > /dev/null
+