Merge branch 'master' of https://gitea.cvtt.net/prod/utils

archive_logs.sh

@@ -5,6 +5,7 @@ function usage {
     echo -n " -L <log directory>"
     echo -n " [ -A <archive_logs_dir> (default /works/archive/logs)]"
     echo -n " [-D <older than time criteria> (default: '2 days ago')]"
+    echo -n " [-W <extra wild card> (default: '*.log.*')]"
     echo
     exit 1
 }
@@ -14,10 +15,10 @@ echo Starting $0 $*
 LogDir=${1}
 LogArchiveDir=/works/archive/logs
 DateCriteria="2 days ago"
-
+ExtraWildCard=
 
 # ---------------- cmdline
-while getopts "A:L:D:" opt; do
+while getopts "A:L:D:W:" opt; do
   case ${opt} in
     A )
       LogArchiveDir=$OPTARG
@@ -28,6 +29,9 @@ while getopts "A:L:D:" opt; do
     D )
       DateCriteria=$OPTARG
       ;;
+    W )
+      ExtraWildCard=$OPTARG
+      ;;
     \? )
       echo "Invalid option: -$OPTARG" >&2
       usage
@@ -53,7 +57,16 @@ echo "Looking for log files older than '${DateCriteria}' in ${LogDir}"
 
 Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S')
 
-Cmd="find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')'  -type f -not -newermt \"${Oldest}\""
+Cmd="find ${LogDir}/"
+Cmd+=" '('"
+Cmd+=" -name '*.log'"
+Cmd+=" -o -name '*.log.*'"
+if [ "${ExtraWildCard}" != "" ]; then
+  Cmd+=" -o -name '${ExtraWildCard}'"
+fi
+Cmd+=" ')'"
+Cmd+=" -type f"
+Cmd+=" -not -newermt \"${Oldest}\""
 echo $Cmd
 
 files=$(eval ${Cmd})

backup/backup-git.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SRC_ROOT="/works/git/"
+DEST_USER="cvtt"
+DEST_HOST="hs01.cvtt.vpn"
+DEST_BASE="/backup/git/snapshots"
+TODAY="$(date +%F)"
+DEST_TODAY="${DEST_BASE}/${TODAY}"
+SSH_OPTS="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+
+COMMON=
+COMMON+="-avHAX"
+COMMON+=" --numeric-ids"
+COMMON+=" --delete"
+COMMON+=" --partial"
+COMMON+=" --delay-updates"
+COMMON+=" --no-inc-recursive"
+COMMON+=" --mkpath"
+COMMON+=" -e '${SSH_OPTS}'"
+COMMON+=" --exclude '**/*.lock'"
+COMMON+=" --exclude '**/hooks/*.sample'"
+
+# Two-phase to avoid refs racing
+PHASE1=
+PHASE1+="${COMMON}"
+PHASE1+=" --exclude '**/refs/**'"
+PHASE1+=" --exclude '**/packed-refs'"
+PHASE1+=" --exclude '**/HEAD'"
+
+PHASE2=
+PHASE2+="${COMMON}"
+PHASE2+=" --include '**/refs/**'"
+PHASE2+=" --include '**/packed-refs'"
+PHASE2+=" --include '**/HEAD'"
+PHASE2+=" --include '*/'"
+PHASE2+=" --exclude '*'"
+
+TARGET="${DEST_USER}@${DEST_HOST}:${DEST_TODAY%/}/"
+
+echo "== Phase 1 =="
+Cmd="rsync ${PHASE1} ${SRC_ROOT} ${TARGET}"
+echo ${Cmd} && eval ${Cmd}
+
+echo "== Phase 2 =="
+Cmd="rsync ${PHASE2} ${SRC_ROOT} ${TARGET}"
+echo ${Cmd} && eval ${Cmd}
+
+echo "== Consistency re-check =="
+echo ${Cmd} && eval ${Cmd}
+
+echo "Snapshot completed: $DEST_TODAY"
+

backup/backup_by_json.sh

@@ -0,0 +1,187 @@
+#!/bin/bash
+# Script to create and run rsync command based on JSON configuration
+
+: <<'COMMENT'
+Config File Example (local backup for hs01
+==========================================
+{
+    "Name" : "hs01",
+    "Host" : "",
+
+    "BackupHost" : "",
+    "BackupRootDir" : "/backup/hs01",
+
+    "Filesystems" : ["/etc", "/opt", "/archive", "/data", "/works" ],
+
+    "RsyncOptions" : ["-arxv", "--delete", "--relative" ],
+
+    "Exclude" : ["*var/tmp*"
+                 , "*var/lib/ntp/proc*"
+                 , "*.cache*"
+                 , "*root/.mozilla*"
+                 , "*var/cache*"
+                 , "*var/spool*"
+                 , "*/.kde4/*"
+                 , "/tmp"
+               ],
+
+    "DailySnapshots" : false,
+    "PruneBackupDays" : 1
+
+}
+COMMENT
+
+
+usage() {
+    echo "Usage: ${Script} <json_config_file>"
+    exit 1
+}
+
+
+# Check if jq is installed
+if ! command -v jq &>/dev/null; then
+    echo "Error: jq is required for JSON processing. Install it and try again."
+    exit 1
+fi
+
+# Verify configuration
+verify_config() {
+    local config_file="$1"
+
+    # Check if required directories and files exist
+    local backup_root_dir
+    backup_root_dir=$(jq -r '.BackupRootDir' "$config_file")
+    if [[ ! -d "$backup_root_dir" ]]; then
+        echo "----- Error: BackupRootDir ($backup_root_dir) does not exist."
+        mkdir -p "${backup_root_dir}"
+    fi
+
+}
+
+# Run rsync
+run_rsync() {
+    local cmdlist=("$@")
+    echo "----- Running rsync: ${cmdlist[*]}"
+    "${cmdlist[@]}"
+}
+
+# Prune old backups
+prune_backup_dirs() {
+    local bkp_host="$1"
+    local root_dir="$2"
+    local days="$3"
+
+    echo "----- Looking for files to prune..."
+    local cmd="cd $root_dir && ls -rd 2* | awk -v days=$days '{ if (NR > days) print }'"
+    [[ -n "$bkp_host" ]] && cmd="ssh $bkp_host \"$cmd\""
+
+    echo "----- Command: $cmd"
+    local files
+    files=$(eval "$cmd")
+    if [[ -n "$files" ]]; then
+        echo "----- Found files to prune:"
+        echo "$files"
+        local prune_cmd="rm -rf $(echo "$files" | xargs -I {} echo "$root_dir/{}")"
+        [[ -n "$bkp_host" ]] && prune_cmd="ssh $bkp_host \"$prune_cmd\""
+        echo "----- Executing: $prune_cmd"
+        eval "$prune_cmd"
+    else
+        echo "----- Nothing to prune."
+    fi
+}
+
+# Generate rsync arguments
+get_rsync_arguments() {
+    local config_file="$1"
+    local dt="$2"
+    local fs="$3"
+
+    local bkp_root_dir
+    bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file")
+    local daily_snapshots
+    daily_snapshots=$(jq -r '.DailySnapshots' "$config_file")
+    local rsync_options
+    rsync_options=$(jq -r '.RsyncOptions[]' "$config_file")
+    local exclude_list
+    exclude_list=$(jq -r '.Exclude[]' "$config_file")
+
+    local bkp_dir="$bkp_root_dir"
+    [[ "$daily_snapshots" == "true" ]] && bkp_dir="$bkp_root_dir/$dt"
+
+    local cmdlist=("rsync" $rsync_options)
+    for excl in $exclude_list; do
+        cmdlist+=("--exclude=$excl")
+    done
+    [[ "$daily_snapshots" == "true" ]] && cmdlist+=("--link-dest=$bkp_root_dir/current")
+
+    local src_host
+    src_host=$(jq -r '.Host' "$config_file")
+    [[ -n "$src_host" ]] && src_host="$src_host:"
+
+    local tgt_host
+    tgt_host=$(jq -r '.BackupHost' "$config_file")
+    [[ -n "$tgt_host" ]] && tgt_host="$tgt_host:"
+
+    cmdlist+=("${src_host}${fs}" "${tgt_host}${bkp_dir}/")
+
+    echo "${cmdlist[@]}"
+}
+
+# Main function
+main() {
+    local config_file="$1"
+    if [[ ! -f "$config_file" ]]; then
+        echo "----- Error: Configuration file $config_file does not exist."
+        exit 1
+    fi
+
+    if ! verify_config "$config_file"; then
+        exit 1
+    fi
+
+    local dt
+    dt=$(date +%Y%m%d)
+
+    local filesystems
+    filesystems=$(jq -r '.Filesystems[]' "$config_file")
+    for fs in $filesystems; do
+        echo "----- Processing filesystem: $fs"
+        local cmdlist
+        cmdlist=($(get_rsync_arguments "$config_file" "$dt" "$fs"))
+        run_rsync "${cmdlist[@]}"
+    done
+
+    local daily_snapshots
+    daily_snapshots=$(jq -r '.DailySnapshots' "$config_file")
+    if [[ "$daily_snapshots" == "true" ]]; then
+        local bkp_root_dir
+        bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file")
+        local bkp_host
+        bkp_host=$(jq -r '.BackupHost' "$config_file")
+        local cmd="ln -snf $bkp_root_dir/$dt $bkp_root_dir/current"
+        [[ -n "$bkp_host" ]] && cmd="ssh $bkp_host \"$cmd\""
+        echo "----- Switching current symlink: $cmd"
+        eval "$cmd"
+
+        local prune_days
+        prune_days=$(jq -r '.PruneBackupDays' "$config_file")
+        if [[ "$prune_days" -gt 0 ]]; then
+            local bkp_root_dir
+            bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file")
+            local bkp_host
+            bkp_host=$(jq -r '.BackupHost' "$config_file")
+            prune_backup_dirs "$bkp_host" "$bkp_root_dir" "$prune_days"
+        fi
+    fi
+}
+
+# Entry point
+Script=${0}
+Config=${1}
+if [[ $# -lt 1 ]]; then
+    usage
+    exit 1
+fi
+
+main "${Config}"
+

backup/pull_backup.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+### E x a m p l e
+#     -S cvtt@cloud23.cvtt.vpn:/works/docker/infisical/backup/
+#     -T /backup/cloud23/infisical/
+#     -e "--remove-source-files"
+
+usage() {
+  echo "Usage: $0 -S <source> -T  <target_dir> [-e <extra rsync args>]"
+  exit 1
+}
+
+# ---------------- Default
+AddRsyncArgs=
+
+# ---------------- cmdline
+while getopts "S:T:e:" opt; do
+  case ${opt} in
+    S )
+      Source=$OPTARG
+      ;;
+    T )
+      TargetDir=$OPTARG
+      ;;
+    e )
+      AddRsyncArgs=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+# ---------------- cmdline
+
+if [ "${Source}" == "" ]
+then
+    usage
+fi
+
+if [ "${TargetDir}" == "" ]
+then
+    usage
+fi
+
+Cmd="mkdir -p ${TargetDir}"
+echo ${Cmd} && eval ${Cmd} || exit
+
+Cmd="rsync"
+Cmd+=" -ahv"
+Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+Cmd+=" ${AddRsyncArgs}"
+Cmd+=" ${Source}"
+Cmd+=" ${TargetDir}"
+echo ${Cmd}
+eval ${Cmd}

prune_data.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: ${0}"
+    echo -n " -d <root_dir>"
+    echo -n " -D <older than time criteria> (example: '2 days ago')"
+    echo
+    exit 1
+}
+
+echo Starting $0 $*
+
+
+# ---------------- cmdline
+while getopts "d:D:" opt; do
+  case ${opt} in
+    d )
+      RootDir=$OPTARG
+      ;;
+    D )
+      DateCriteria=$OPTARG
+      ;;
+    \? )
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      ;;
+    : )
+      echo "Option -$OPTARG requires an argument." >&2
+      usage
+      ;;
+  esac
+done
+
+if  [ "${RootDir}" == "" ] || [ "${DateCriteria}" == "" ] ; then
+    usage
+fi
+
+declare -A Settings=()
+Settings[Src]=${RootDir}
+Settings[PruneDate]=$(date -d "${DateCriteria}" '+%Y-%m-%d')
+
+src=${Settings[Src]}
+prune_date=${Settings[PruneDate]}
+
+echo Before Pruning....
+df -hT ${src}
+
+echo "Finding files older than ${prune_date}..."
+Cmd="find ${src} -type f ! -newermt \"${prune_date}\""
+echo ${Cmd}
+files=($(eval ${Cmd}))
+echo "Total files to be pruned: ${#files[*]}"
+
+for file in "${files[@]}" ; do
+   Cmd="rm -f ${file}"
+   echo ${Cmd}
+   eval ${Cmd}
+done
+
+echo "Removing empty directories..."
+Cmd="find ${src} -type d -empty -print -delete"
+echo ${Cmd}
+eval ${Cmd}
+
+echo After Pruning....
+df -hT ${src}
+
+echo $0 Done
+