diff --git a/archive_logs.sh b/archive_logs.sh index a9ce3b3..e9bf4a0 100755 --- a/archive_logs.sh +++ b/archive_logs.sh @@ -5,6 +5,7 @@ function usage { echo -n " -L " echo -n " [ -A (default /works/archive/logs)]" echo -n " [-D (default: '2 days ago')]" + echo -n " [-W (default: '*.log.*')]" echo exit 1 } @@ -14,10 +15,10 @@ echo Starting $0 $* LogDir=${1} LogArchiveDir=/works/archive/logs DateCriteria="2 days ago" - +ExtraWildCard= # ---------------- cmdline -while getopts "A:L:D:" opt; do +while getopts "A:L:D:W:" opt; do case ${opt} in A ) LogArchiveDir=$OPTARG @@ -28,6 +29,9 @@ while getopts "A:L:D:" opt; do D ) DateCriteria=$OPTARG ;; + W ) + ExtraWildCard=$OPTARG + ;; \? ) echo "Invalid option: -$OPTARG" >&2 usage @@ -53,7 +57,16 @@ echo "Looking for log files older than '${DateCriteria}' in ${LogDir}" Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S') -Cmd="find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt \"${Oldest}\"" +Cmd="find ${LogDir}/" +Cmd+=" '('" +Cmd+=" -name '*.log'" +Cmd+=" -o -name '*.log.*'" +if [ "${ExtraWildCard}" != "" ]; then + Cmd+=" -o -name '${ExtraWildCard}'" +fi +Cmd+=" ')'" +Cmd+=" -type f" +Cmd+=" -not -newermt \"${Oldest}\"" echo $Cmd files=$(eval ${Cmd}) diff --git a/backup/backup-git.sh b/backup/backup-git.sh new file mode 100755 index 0000000..da6529a --- /dev/null +++ b/backup/backup-git.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SRC_ROOT="/works/git/" +DEST_USER="cvtt" +DEST_HOST="hs01.cvtt.vpn" +DEST_BASE="/backup/git/snapshots" +TODAY="$(date +%F)" +DEST_TODAY="${DEST_BASE}/${TODAY}" +SSH_OPTS="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + +COMMON= +COMMON+="-avHAX" +COMMON+=" --numeric-ids" +COMMON+=" --delete" +COMMON+=" --partial" +COMMON+=" --delay-updates" +COMMON+=" --no-inc-recursive" +COMMON+=" --mkpath" +COMMON+=" -e '${SSH_OPTS}'" +COMMON+=" --exclude '**/*.lock'" +COMMON+=" --exclude '**/hooks/*.sample'" + +# Two-phase to avoid refs racing +PHASE1= +PHASE1+="${COMMON}" +PHASE1+=" --exclude '**/refs/**'" +PHASE1+=" --exclude '**/packed-refs'" +PHASE1+=" --exclude '**/HEAD'" + +PHASE2= +PHASE2+="${COMMON}" +PHASE2+=" --include '**/refs/**'" +PHASE2+=" --include '**/packed-refs'" +PHASE2+=" --include '**/HEAD'" +PHASE2+=" --include '*/'" +PHASE2+=" --exclude '*'" + +TARGET="${DEST_USER}@${DEST_HOST}:${DEST_TODAY%/}/" + +echo "== Phase 1 ==" +Cmd="rsync ${PHASE1} ${SRC_ROOT} ${TARGET}" +echo ${Cmd} && eval ${Cmd} + +echo "== Phase 2 ==" +Cmd="rsync ${PHASE2} ${SRC_ROOT} ${TARGET}" +echo ${Cmd} && eval ${Cmd} + +echo "== Consistency re-check ==" +echo ${Cmd} && eval ${Cmd} + +echo "Snapshot completed: $DEST_TODAY" + diff --git a/backup/backup_by_json.sh b/backup/backup_by_json.sh new file mode 100755 index 0000000..1338ecd --- /dev/null +++ b/backup/backup_by_json.sh @@ -0,0 +1,187 @@ +#!/bin/bash +# Script to create and run rsync command based on JSON configuration + +: <<'COMMENT' +Config File Example (local backup for hs01 +========================================== +{ + "Name" : "hs01", + "Host" : "", + + "BackupHost" : "", + "BackupRootDir" : "/backup/hs01", + + "Filesystems" : ["/etc", "/opt", "/archive", "/data", "/works" ], + + "RsyncOptions" : ["-arxv", "--delete", "--relative" ], + + "Exclude" : ["*var/tmp*" + , "*var/lib/ntp/proc*" + , "*.cache*" + , "*root/.mozilla*" + , "*var/cache*" + , "*var/spool*" + , "*/.kde4/*" + , "/tmp" + ], + + "DailySnapshots" : false, + "PruneBackupDays" : 1 + +} +COMMENT + + +usage() { + echo "Usage: ${Script} " + exit 1 +} + + +# Check if jq is installed +if ! command -v jq &>/dev/null; then + echo "Error: jq is required for JSON processing. Install it and try again." + exit 1 +fi + +# Verify configuration +verify_config() { + local config_file="$1" + + # Check if required directories and files exist + local backup_root_dir + backup_root_dir=$(jq -r '.BackupRootDir' "$config_file") + if [[ ! -d "$backup_root_dir" ]]; then + echo "----- Error: BackupRootDir ($backup_root_dir) does not exist." + mkdir -p "${backup_root_dir}" + fi + +} + +# Run rsync +run_rsync() { + local cmdlist=("$@") + echo "----- Running rsync: ${cmdlist[*]}" + "${cmdlist[@]}" +} + +# Prune old backups +prune_backup_dirs() { + local bkp_host="$1" + local root_dir="$2" + local days="$3" + + echo "----- Looking for files to prune..." + local cmd="cd $root_dir && ls -rd 2* | awk -v days=$days '{ if (NR > days) print }'" + [[ -n "$bkp_host" ]] && cmd="ssh $bkp_host \"$cmd\"" + + echo "----- Command: $cmd" + local files + files=$(eval "$cmd") + if [[ -n "$files" ]]; then + echo "----- Found files to prune:" + echo "$files" + local prune_cmd="rm -rf $(echo "$files" | xargs -I {} echo "$root_dir/{}")" + [[ -n "$bkp_host" ]] && prune_cmd="ssh $bkp_host \"$prune_cmd\"" + echo "----- Executing: $prune_cmd" + eval "$prune_cmd" + else + echo "----- Nothing to prune." + fi +} + +# Generate rsync arguments +get_rsync_arguments() { + local config_file="$1" + local dt="$2" + local fs="$3" + + local bkp_root_dir + bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file") + local daily_snapshots + daily_snapshots=$(jq -r '.DailySnapshots' "$config_file") + local rsync_options + rsync_options=$(jq -r '.RsyncOptions[]' "$config_file") + local exclude_list + exclude_list=$(jq -r '.Exclude[]' "$config_file") + + local bkp_dir="$bkp_root_dir" + [[ "$daily_snapshots" == "true" ]] && bkp_dir="$bkp_root_dir/$dt" + + local cmdlist=("rsync" $rsync_options) + for excl in $exclude_list; do + cmdlist+=("--exclude=$excl") + done + [[ "$daily_snapshots" == "true" ]] && cmdlist+=("--link-dest=$bkp_root_dir/current") + + local src_host + src_host=$(jq -r '.Host' "$config_file") + [[ -n "$src_host" ]] && src_host="$src_host:" + + local tgt_host + tgt_host=$(jq -r '.BackupHost' "$config_file") + [[ -n "$tgt_host" ]] && tgt_host="$tgt_host:" + + cmdlist+=("${src_host}${fs}" "${tgt_host}${bkp_dir}/") + + echo "${cmdlist[@]}" +} + +# Main function +main() { + local config_file="$1" + if [[ ! -f "$config_file" ]]; then + echo "----- Error: Configuration file $config_file does not exist." + exit 1 + fi + + if ! verify_config "$config_file"; then + exit 1 + fi + + local dt + dt=$(date +%Y%m%d) + + local filesystems + filesystems=$(jq -r '.Filesystems[]' "$config_file") + for fs in $filesystems; do + echo "----- Processing filesystem: $fs" + local cmdlist + cmdlist=($(get_rsync_arguments "$config_file" "$dt" "$fs")) + run_rsync "${cmdlist[@]}" + done + + local daily_snapshots + daily_snapshots=$(jq -r '.DailySnapshots' "$config_file") + if [[ "$daily_snapshots" == "true" ]]; then + local bkp_root_dir + bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file") + local bkp_host + bkp_host=$(jq -r '.BackupHost' "$config_file") + local cmd="ln -snf $bkp_root_dir/$dt $bkp_root_dir/current" + [[ -n "$bkp_host" ]] && cmd="ssh $bkp_host \"$cmd\"" + echo "----- Switching current symlink: $cmd" + eval "$cmd" + + local prune_days + prune_days=$(jq -r '.PruneBackupDays' "$config_file") + if [[ "$prune_days" -gt 0 ]]; then + local bkp_root_dir + bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file") + local bkp_host + bkp_host=$(jq -r '.BackupHost' "$config_file") + prune_backup_dirs "$bkp_host" "$bkp_root_dir" "$prune_days" + fi + fi +} + +# Entry point +Script=${0} +Config=${1} +if [[ $# -lt 1 ]]; then + usage + exit 1 +fi + +main "${Config}" + diff --git a/backup/pull_backup.sh b/backup/pull_backup.sh new file mode 100755 index 0000000..4a8bfd4 --- /dev/null +++ b/backup/pull_backup.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +### E x a m p l e +# -S cvtt@cloud23.cvtt.vpn:/works/docker/infisical/backup/ +# -T /backup/cloud23/infisical/ +# -e "--remove-source-files" + +usage() { + echo "Usage: $0 -S -T [-e ]" + exit 1 +} + +# ---------------- Default +AddRsyncArgs= + +# ---------------- cmdline +while getopts "S:T:e:" opt; do + case ${opt} in + S ) + Source=$OPTARG + ;; + T ) + TargetDir=$OPTARG + ;; + e ) + AddRsyncArgs=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done +# ---------------- cmdline + +if [ "${Source}" == "" ] +then + usage +fi + +if [ "${TargetDir}" == "" ] +then + usage +fi + +Cmd="mkdir -p ${TargetDir}" +echo ${Cmd} && eval ${Cmd} || exit + +Cmd="rsync" +Cmd+=" -ahv" +Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" +Cmd+=" ${AddRsyncArgs}" +Cmd+=" ${Source}" +Cmd+=" ${TargetDir}" +echo ${Cmd} +eval ${Cmd} diff --git a/prune_data.sh b/prune_data.sh new file mode 100755 index 0000000..d5774e5 --- /dev/null +++ b/prune_data.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +usage() { + echo -n "Usage: ${0}" + echo -n " -d " + echo -n " -D (example: '2 days ago')" + echo + exit 1 +} + +echo Starting $0 $* + + +# ---------------- cmdline +while getopts "d:D:" opt; do + case ${opt} in + d ) + RootDir=$OPTARG + ;; + D ) + DateCriteria=$OPTARG + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + usage + ;; + : ) + echo "Option -$OPTARG requires an argument." >&2 + usage + ;; + esac +done + +if [ "${RootDir}" == "" ] || [ "${DateCriteria}" == "" ] ; then + usage +fi + +declare -A Settings=() +Settings[Src]=${RootDir} +Settings[PruneDate]=$(date -d "${DateCriteria}" '+%Y-%m-%d') + +src=${Settings[Src]} +prune_date=${Settings[PruneDate]} + +echo Before Pruning.... +df -hT ${src} + +echo "Finding files older than ${prune_date}..." +Cmd="find ${src} -type f ! -newermt \"${prune_date}\"" +echo ${Cmd} +files=($(eval ${Cmd})) +echo "Total files to be pruned: ${#files[*]}" + +for file in "${files[@]}" ; do + Cmd="rm -f ${file}" + echo ${Cmd} + eval ${Cmd} +done + +echo "Removing empty directories..." +Cmd="find ${src} -type d -empty -print -delete" +echo ${Cmd} +eval ${Cmd} + +echo After Pruning.... +df -hT ${src} + +echo $0 Done +