Merge branch 'master' of https://gitea.cvtt.net/prod/utils
This commit is contained in:
Cryptoval Trading Technologies
commit
54dfeeff52
@ -5,6 +5,7 @@ function usage {
|
||||
echo -n " -L <log directory>"
|
||||
echo -n " [ -A <archive_logs_dir> (default /works/archive/logs)]"
|
||||
echo -n " [-D <older than time criteria> (default: '2 days ago')]"
|
||||
echo -n " [-W <extra wild card> (default: '*.log.*')]"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
@ -14,10 +15,10 @@ echo Starting $0 $*
|
||||
LogDir=${1}
|
||||
LogArchiveDir=/works/archive/logs
|
||||
DateCriteria="2 days ago"
|
||||
|
||||
ExtraWildCard=
|
||||
|
||||
# ---------------- cmdline
|
||||
while getopts "A:L:D:" opt; do
|
||||
while getopts "A:L:D:W:" opt; do
|
||||
case ${opt} in
|
||||
A )
|
||||
LogArchiveDir=$OPTARG
|
||||
@ -28,6 +29,9 @@ while getopts "A:L:D:" opt; do
|
||||
D )
|
||||
DateCriteria=$OPTARG
|
||||
;;
|
||||
W )
|
||||
ExtraWildCard=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
@ -53,7 +57,16 @@ echo "Looking for log files older than '${DateCriteria}' in ${LogDir}"
|
||||
|
||||
Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
Cmd="find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt \"${Oldest}\""
|
||||
Cmd="find ${LogDir}/"
|
||||
Cmd+=" '('"
|
||||
Cmd+=" -name '*.log'"
|
||||
Cmd+=" -o -name '*.log.*'"
|
||||
if [ "${ExtraWildCard}" != "" ]; then
|
||||
Cmd+=" -o -name '${ExtraWildCard}'"
|
||||
fi
|
||||
Cmd+=" ')'"
|
||||
Cmd+=" -type f"
|
||||
Cmd+=" -not -newermt \"${Oldest}\""
|
||||
echo $Cmd
|
||||
|
||||
files=$(eval ${Cmd})
|
||||
|
||||
53
backup/backup-git.sh
Executable file
53
backup/backup-git.sh
Executable file
@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
SRC_ROOT="/works/git/"
|
||||
DEST_USER="cvtt"
|
||||
DEST_HOST="hs01.cvtt.vpn"
|
||||
DEST_BASE="/backup/git/snapshots"
|
||||
TODAY="$(date +%F)"
|
||||
DEST_TODAY="${DEST_BASE}/${TODAY}"
|
||||
SSH_OPTS="ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
||||
|
||||
COMMON=
|
||||
COMMON+="-avHAX"
|
||||
COMMON+=" --numeric-ids"
|
||||
COMMON+=" --delete"
|
||||
COMMON+=" --partial"
|
||||
COMMON+=" --delay-updates"
|
||||
COMMON+=" --no-inc-recursive"
|
||||
COMMON+=" --mkpath"
|
||||
COMMON+=" -e '${SSH_OPTS}'"
|
||||
COMMON+=" --exclude '**/*.lock'"
|
||||
COMMON+=" --exclude '**/hooks/*.sample'"
|
||||
|
||||
# Two-phase to avoid refs racing
|
||||
PHASE1=
|
||||
PHASE1+="${COMMON}"
|
||||
PHASE1+=" --exclude '**/refs/**'"
|
||||
PHASE1+=" --exclude '**/packed-refs'"
|
||||
PHASE1+=" --exclude '**/HEAD'"
|
||||
|
||||
PHASE2=
|
||||
PHASE2+="${COMMON}"
|
||||
PHASE2+=" --include '**/refs/**'"
|
||||
PHASE2+=" --include '**/packed-refs'"
|
||||
PHASE2+=" --include '**/HEAD'"
|
||||
PHASE2+=" --include '*/'"
|
||||
PHASE2+=" --exclude '*'"
|
||||
|
||||
TARGET="${DEST_USER}@${DEST_HOST}:${DEST_TODAY%/}/"
|
||||
|
||||
echo "== Phase 1 =="
|
||||
Cmd="rsync ${PHASE1} ${SRC_ROOT} ${TARGET}"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
echo "== Phase 2 =="
|
||||
Cmd="rsync ${PHASE2} ${SRC_ROOT} ${TARGET}"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
echo "== Consistency re-check =="
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
echo "Snapshot completed: $DEST_TODAY"
|
||||
|
||||
187
backup/backup_by_json.sh
Executable file
187
backup/backup_by_json.sh
Executable file
@ -0,0 +1,187 @@
|
||||
#!/bin/bash
|
||||
# Script to create and run rsync command based on JSON configuration
|
||||
|
||||
: <<'COMMENT'
|
||||
Config File Example (local backup for hs01
|
||||
==========================================
|
||||
{
|
||||
"Name" : "hs01",
|
||||
"Host" : "",
|
||||
|
||||
"BackupHost" : "",
|
||||
"BackupRootDir" : "/backup/hs01",
|
||||
|
||||
"Filesystems" : ["/etc", "/opt", "/archive", "/data", "/works" ],
|
||||
|
||||
"RsyncOptions" : ["-arxv", "--delete", "--relative" ],
|
||||
|
||||
"Exclude" : ["*var/tmp*"
|
||||
, "*var/lib/ntp/proc*"
|
||||
, "*.cache*"
|
||||
, "*root/.mozilla*"
|
||||
, "*var/cache*"
|
||||
, "*var/spool*"
|
||||
, "*/.kde4/*"
|
||||
, "/tmp"
|
||||
],
|
||||
|
||||
"DailySnapshots" : false,
|
||||
"PruneBackupDays" : 1
|
||||
|
||||
}
|
||||
COMMENT
|
||||
|
||||
|
||||
usage() {
|
||||
echo "Usage: ${Script} <json_config_file>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
# Check if jq is installed
|
||||
if ! command -v jq &>/dev/null; then
|
||||
echo "Error: jq is required for JSON processing. Install it and try again."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify configuration
|
||||
verify_config() {
|
||||
local config_file="$1"
|
||||
|
||||
# Check if required directories and files exist
|
||||
local backup_root_dir
|
||||
backup_root_dir=$(jq -r '.BackupRootDir' "$config_file")
|
||||
if [[ ! -d "$backup_root_dir" ]]; then
|
||||
echo "----- Error: BackupRootDir ($backup_root_dir) does not exist."
|
||||
mkdir -p "${backup_root_dir}"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
# Run rsync
|
||||
run_rsync() {
|
||||
local cmdlist=("$@")
|
||||
echo "----- Running rsync: ${cmdlist[*]}"
|
||||
"${cmdlist[@]}"
|
||||
}
|
||||
|
||||
# Prune old backups
|
||||
prune_backup_dirs() {
|
||||
local bkp_host="$1"
|
||||
local root_dir="$2"
|
||||
local days="$3"
|
||||
|
||||
echo "----- Looking for files to prune..."
|
||||
local cmd="cd $root_dir && ls -rd 2* | awk -v days=$days '{ if (NR > days) print }'"
|
||||
[[ -n "$bkp_host" ]] && cmd="ssh $bkp_host \"$cmd\""
|
||||
|
||||
echo "----- Command: $cmd"
|
||||
local files
|
||||
files=$(eval "$cmd")
|
||||
if [[ -n "$files" ]]; then
|
||||
echo "----- Found files to prune:"
|
||||
echo "$files"
|
||||
local prune_cmd="rm -rf $(echo "$files" | xargs -I {} echo "$root_dir/{}")"
|
||||
[[ -n "$bkp_host" ]] && prune_cmd="ssh $bkp_host \"$prune_cmd\""
|
||||
echo "----- Executing: $prune_cmd"
|
||||
eval "$prune_cmd"
|
||||
else
|
||||
echo "----- Nothing to prune."
|
||||
fi
|
||||
}
|
||||
|
||||
# Generate rsync arguments
|
||||
get_rsync_arguments() {
|
||||
local config_file="$1"
|
||||
local dt="$2"
|
||||
local fs="$3"
|
||||
|
||||
local bkp_root_dir
|
||||
bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file")
|
||||
local daily_snapshots
|
||||
daily_snapshots=$(jq -r '.DailySnapshots' "$config_file")
|
||||
local rsync_options
|
||||
rsync_options=$(jq -r '.RsyncOptions[]' "$config_file")
|
||||
local exclude_list
|
||||
exclude_list=$(jq -r '.Exclude[]' "$config_file")
|
||||
|
||||
local bkp_dir="$bkp_root_dir"
|
||||
[[ "$daily_snapshots" == "true" ]] && bkp_dir="$bkp_root_dir/$dt"
|
||||
|
||||
local cmdlist=("rsync" $rsync_options)
|
||||
for excl in $exclude_list; do
|
||||
cmdlist+=("--exclude=$excl")
|
||||
done
|
||||
[[ "$daily_snapshots" == "true" ]] && cmdlist+=("--link-dest=$bkp_root_dir/current")
|
||||
|
||||
local src_host
|
||||
src_host=$(jq -r '.Host' "$config_file")
|
||||
[[ -n "$src_host" ]] && src_host="$src_host:"
|
||||
|
||||
local tgt_host
|
||||
tgt_host=$(jq -r '.BackupHost' "$config_file")
|
||||
[[ -n "$tgt_host" ]] && tgt_host="$tgt_host:"
|
||||
|
||||
cmdlist+=("${src_host}${fs}" "${tgt_host}${bkp_dir}/")
|
||||
|
||||
echo "${cmdlist[@]}"
|
||||
}
|
||||
|
||||
# Main function
|
||||
main() {
|
||||
local config_file="$1"
|
||||
if [[ ! -f "$config_file" ]]; then
|
||||
echo "----- Error: Configuration file $config_file does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! verify_config "$config_file"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local dt
|
||||
dt=$(date +%Y%m%d)
|
||||
|
||||
local filesystems
|
||||
filesystems=$(jq -r '.Filesystems[]' "$config_file")
|
||||
for fs in $filesystems; do
|
||||
echo "----- Processing filesystem: $fs"
|
||||
local cmdlist
|
||||
cmdlist=($(get_rsync_arguments "$config_file" "$dt" "$fs"))
|
||||
run_rsync "${cmdlist[@]}"
|
||||
done
|
||||
|
||||
local daily_snapshots
|
||||
daily_snapshots=$(jq -r '.DailySnapshots' "$config_file")
|
||||
if [[ "$daily_snapshots" == "true" ]]; then
|
||||
local bkp_root_dir
|
||||
bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file")
|
||||
local bkp_host
|
||||
bkp_host=$(jq -r '.BackupHost' "$config_file")
|
||||
local cmd="ln -snf $bkp_root_dir/$dt $bkp_root_dir/current"
|
||||
[[ -n "$bkp_host" ]] && cmd="ssh $bkp_host \"$cmd\""
|
||||
echo "----- Switching current symlink: $cmd"
|
||||
eval "$cmd"
|
||||
|
||||
local prune_days
|
||||
prune_days=$(jq -r '.PruneBackupDays' "$config_file")
|
||||
if [[ "$prune_days" -gt 0 ]]; then
|
||||
local bkp_root_dir
|
||||
bkp_root_dir=$(jq -r '.BackupRootDir' "$config_file")
|
||||
local bkp_host
|
||||
bkp_host=$(jq -r '.BackupHost' "$config_file")
|
||||
prune_backup_dirs "$bkp_host" "$bkp_root_dir" "$prune_days"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Entry point
|
||||
Script=${0}
|
||||
Config=${1}
|
||||
if [[ $# -lt 1 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
main "${Config}"
|
||||
|
||||
60
backup/pull_backup.sh
Executable file
60
backup/pull_backup.sh
Executable file
@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
|
||||
### E x a m p l e
|
||||
# -S cvtt@cloud23.cvtt.vpn:/works/docker/infisical/backup/
|
||||
# -T /backup/cloud23/infisical/
|
||||
# -e "--remove-source-files"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -S <source> -T <target_dir> [-e <extra rsync args>]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ---------------- Default
|
||||
AddRsyncArgs=
|
||||
|
||||
# ---------------- cmdline
|
||||
while getopts "S:T:e:" opt; do
|
||||
case ${opt} in
|
||||
S )
|
||||
Source=$OPTARG
|
||||
;;
|
||||
T )
|
||||
TargetDir=$OPTARG
|
||||
;;
|
||||
e )
|
||||
AddRsyncArgs=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
# ---------------- cmdline
|
||||
|
||||
if [ "${Source}" == "" ]
|
||||
then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ "${TargetDir}" == "" ]
|
||||
then
|
||||
usage
|
||||
fi
|
||||
|
||||
Cmd="mkdir -p ${TargetDir}"
|
||||
echo ${Cmd} && eval ${Cmd} || exit
|
||||
|
||||
Cmd="rsync"
|
||||
Cmd+=" -ahv"
|
||||
Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
|
||||
Cmd+=" ${AddRsyncArgs}"
|
||||
Cmd+=" ${Source}"
|
||||
Cmd+=" ${TargetDir}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
69
prune_data.sh
Executable file
69
prune_data.sh
Executable file
@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
|
||||
usage() {
|
||||
echo -n "Usage: ${0}"
|
||||
echo -n " -d <root_dir>"
|
||||
echo -n " -D <older than time criteria> (example: '2 days ago')"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo Starting $0 $*
|
||||
|
||||
|
||||
# ---------------- cmdline
|
||||
while getopts "d:D:" opt; do
|
||||
case ${opt} in
|
||||
d )
|
||||
RootDir=$OPTARG
|
||||
;;
|
||||
D )
|
||||
DateCriteria=$OPTARG
|
||||
;;
|
||||
\? )
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
;;
|
||||
: )
|
||||
echo "Option -$OPTARG requires an argument." >&2
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${RootDir}" == "" ] || [ "${DateCriteria}" == "" ] ; then
|
||||
usage
|
||||
fi
|
||||
|
||||
declare -A Settings=()
|
||||
Settings[Src]=${RootDir}
|
||||
Settings[PruneDate]=$(date -d "${DateCriteria}" '+%Y-%m-%d')
|
||||
|
||||
src=${Settings[Src]}
|
||||
prune_date=${Settings[PruneDate]}
|
||||
|
||||
echo Before Pruning....
|
||||
df -hT ${src}
|
||||
|
||||
echo "Finding files older than ${prune_date}..."
|
||||
Cmd="find ${src} -type f ! -newermt \"${prune_date}\""
|
||||
echo ${Cmd}
|
||||
files=($(eval ${Cmd}))
|
||||
echo "Total files to be pruned: ${#files[*]}"
|
||||
|
||||
for file in "${files[@]}" ; do
|
||||
Cmd="rm -f ${file}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
|
||||
echo "Removing empty directories..."
|
||||
Cmd="find ${src} -type d -empty -print -delete"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
|
||||
echo After Pruning....
|
||||
df -hT ${src}
|
||||
|
||||
echo $0 Done
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user