Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0e6190f999 | |||
| 180e3f89cf | |||
| 80bbf47755 | |||
| e295aa8dff | |||
| 60d2327b7e | |||
| a0b41cad8e | |||
| b62981a338 | |||
| 553748fdcd | |||
| 5956d9d200 | |||
| 88308ea13e | |||
| 74783ab064 | |||
| c1a4e7b292 | |||
| f9ac5503eb | |||
| 5cc53f8cd8 | |||
| fce17db6ec | |||
| ed43a900a2 | |||
| 9de86bf3e3 | |||
| 4873af1673 | |||
| c2dae20d7b | |||
| 8607db6506 | |||
| 465d4c746e | |||
| 70c7769fd6 | |||
| 2416ca4b0d | |||
| 41d18e0e7f | |||
| e93e1f676f | |||
| 46eead797f | |||
| a6a37edec3 |
@@ -1,47 +0,0 @@
|
||||
[supervisord]
|
||||
environment=PYTHONPATH="/home/cvtt/prod"
|
||||
autostart=true
|
||||
autorestart=unexpected
|
||||
startretries=3
|
||||
logfile=/home/cvtt/prod/logs/supervisord.log
|
||||
|
||||
[supervisorctl]
|
||||
|
||||
[inet_http_server]
|
||||
port = 127.0.0.1:9001
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
[program:bnbfut_rec_cv1]
|
||||
directory=/home/cvtt/prod
|
||||
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||
--active_exchanges=BNBFUT
|
||||
--instruments=PERP-BTC-USDT,PERP-ETH-USDT
|
||||
--db_credentials_key=TSDB_MD_CV1
|
||||
--admin_port=7226
|
||||
--log_level=INFO
|
||||
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CV1.log
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=1
|
||||
startsecs=3
|
||||
user=cvtt
|
||||
|
||||
[program:coinbase_rec_cv1]
|
||||
directory=/home/cvtt/prod
|
||||
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||
--active_exchanges=COINBASE_AT
|
||||
--instruments=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
|
||||
--db_credentials_key=TSDB_MD_CV1
|
||||
--admin_port=7224
|
||||
--log_level=INFO
|
||||
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CV1.log
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=1
|
||||
startsecs=3
|
||||
user=cvtt
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
[supervisord]
|
||||
environment=PYTHONPATH="/home/cvtt/prod"
|
||||
autostart=true
|
||||
autorestart=unexpected
|
||||
startretries=3
|
||||
logfile=/home/cvtt/prod/logs/supervisord.log
|
||||
|
||||
[supervisorctl]
|
||||
|
||||
[inet_http_server]
|
||||
port = 127.0.0.1:9001
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
[program:config_service]
|
||||
directory=/home/cvtt/prod
|
||||
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3.10
|
||||
-u /home/cvtt/prod/cvttpy/apps/utils/config_server.py
|
||||
--port=6789
|
||||
--root=/home/cvtt/prod/config_service/data
|
||||
--log_file=/home/cvtt/prod/logs/%%T.CONFIG_SERVICE.log
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=1
|
||||
startsecs=3
|
||||
user=cvtt
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
[supervisord]
|
||||
environment=PYTHONPATH="/home/cvtt/prod"
|
||||
autostart=true
|
||||
autorestart=unexpected
|
||||
startretries=3
|
||||
logfile=/home/cvtt/prod/logs/supervisord.log
|
||||
|
||||
[supervisorctl]
|
||||
|
||||
[inet_http_server]
|
||||
port = 127.0.0.1:9001
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
# [program:executor]
|
||||
# directory=/home/cvtt/prod
|
||||
# command=/home/cvtt/.pyenv/python3.10-venv/bin/python3
|
||||
# -u /home/cvtt/prod/cvttpy/apps/executor_app.py
|
||||
# --config=http://cloud16.cvtt.vpn:6789/apps/cvtt
|
||||
# --active_exchanges=COINBASE_AT
|
||||
# --admin_port=7220
|
||||
# --log_level=INFO
|
||||
# --log_file=/home/cvtt/prod/logs/%%T.EXECUTOR_APP.log
|
||||
# autostart=true
|
||||
# autorestart=true
|
||||
# startretries=1
|
||||
# startsecs=3
|
||||
# user=cvtt
|
||||
|
||||
#[group:bnbspot_mdrec]
|
||||
#programs=
|
||||
# bnbspot_rec_cloud17,
|
||||
# bnbspot_rec_cv1
|
||||
|
||||
[program:bnbfut_rec_cld19]
|
||||
directory=/home/cvtt/prod
|
||||
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||
--active_exchanges=BNBFUT
|
||||
--instruments=PERP-BTC-USDT,PERP-ETH-USDT
|
||||
--db_credentials_key=TSDB_MD_CLD19
|
||||
--admin_port=7201
|
||||
--log_level=INFO
|
||||
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBFUT.CLD19.log
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=1
|
||||
startsecs=3
|
||||
user=cvtt
|
||||
|
||||
[program:coinbase_rec_cld19]
|
||||
directory=/home/cvtt/prod
|
||||
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||
--active_exchanges=COINBASE_AT
|
||||
--instruments=PAIR-BTC-USD,PAIR-ETH-USD,PAIR-LTC-USD,PAIR-XRP-USD
|
||||
--db_credentials_key=TSDB_MD_CLD19
|
||||
--admin_port=7202
|
||||
--log_level=INFO
|
||||
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.COINBASE.CLD19.log
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=1
|
||||
startsecs=3
|
||||
user=cvtt
|
||||
|
||||
[program:bnbspot_rec_cld19]
|
||||
directory=/home/cvtt/prod
|
||||
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||
--active_exchanges=BNBSPOT
|
||||
--instruments=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
|
||||
--db_credentials_key=TSDB_MD_CLD19
|
||||
--admin_port=7203
|
||||
--log_level=INFO
|
||||
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBSPOT.CLD19.log
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=1
|
||||
startsecs=3
|
||||
user=cvtt
|
||||
|
||||
|
||||
[program:bnbspot_rec_cvttdata]
|
||||
directory=/home/cvtt/prod
|
||||
command=/home/cvtt/.pyenv/python3.10-venv/bin/python3 -u /home/cvtt/prod/cvttpy/apps/md/md_recorder.py
|
||||
--config=http://cloud16.cvtt.vpn:6789/apps/md_recorder
|
||||
--active_exchanges=BNBSPOT
|
||||
--instruments=PAIR-BTC-USDT,PAIR-ETH-USDT,PAIR-LTC-USDT,PAIR-XRP-USDT,PAIR-USDT-USD
|
||||
--db_credentials_key=TSDB_MD_CVTTDATA
|
||||
--admin_port=7222
|
||||
--log_level=INFO
|
||||
--log_file=/home/cvtt/prod/logs/%%T.MD_REC.BNBSPOT.CVTTDATA.log
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=1
|
||||
startsecs=3
|
||||
user=cvtt
|
||||
+1
-1
@@ -1 +1 @@
|
||||
0.5.9
|
||||
0.8.2
|
||||
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
CredsFile=${1}
|
||||
CredKey=${2}
|
||||
|
||||
if [ "${CredsFile}" == "" ] ; then
|
||||
CredsFile="${HOME}/.creds"
|
||||
fi
|
||||
|
||||
if [ "${CredKey}" == "" ] ; then
|
||||
CredKey="ALPACA_SANDBOX"
|
||||
fi
|
||||
|
||||
key=$(jq -r ".[\"${CredKey}\"] | .api_key" ${CredsFile})
|
||||
secret=$(jq -r ".[\"${CredKey}\"] | .secret_key" ${CredsFile})
|
||||
|
||||
Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
|
||||
Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
|
||||
Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
|
||||
Cmd="${Cmd} --header 'accept: application/json'"
|
||||
|
||||
eval ${Cmd}
|
||||
|
||||
# split string into array
|
||||
Instruments=()
|
||||
for Inst in $(eval ${Cmd})
|
||||
do
|
||||
Instruments+=("$Inst")
|
||||
done
|
||||
Executable
+114
@@ -0,0 +1,114 @@
|
||||
#!/bin/bash
|
||||
|
||||
Start=${1}
|
||||
NumJobs=${2}
|
||||
InstListFile=${3}
|
||||
|
||||
export CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
|
||||
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
|
||||
if [ "${Start}" == "" ] ; then
|
||||
Start=$(date -d "yesterday" "+%Y-%m-%d")
|
||||
fi
|
||||
|
||||
if ! is_business_day ${Start}; then
|
||||
echo "${Start} is not business day in US"
|
||||
exit
|
||||
fi
|
||||
|
||||
End=$(date -d "${Start} + 1 day" "+%Y-%m-%d")
|
||||
|
||||
# Exclude today
|
||||
if [ "${End}" == $(date '+%Y-%m-%d') ] ; then
|
||||
End=$(date -d 'yesterday' ''+%Y-%m-%d'')
|
||||
fi
|
||||
|
||||
if [ "${NumJobs}" == "" ] ; then
|
||||
NumJobs=30
|
||||
fi
|
||||
|
||||
echo "Start=${Start} End=${End} NumJobs=${NumJobs}"
|
||||
|
||||
# export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_mkt_data.py
|
||||
|
||||
export PYTHONPATH=/home/cvtt/prod
|
||||
export Python=/home/cvtt/.pyenv/python3.10-venv/bin/python3
|
||||
export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
|
||||
export PyScript=/home/cvtt/prod/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
|
||||
export OutputDir=/home/cvtt/host_drive/alpaca_md # Local
|
||||
export LogDir=/home/cvtt/prod/logs/alpaca_md
|
||||
|
||||
# ----- T E M P
|
||||
# export PYTHONPATH=/home/oleg/develop/cvtt2
|
||||
# export Python=/home/oleg/.pyenv/python3.10-venv/bin/python3
|
||||
# export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
|
||||
# export PyScript=/home/oleg/develop/cvtt2/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
|
||||
# export OutputDir=/home/oleg/develop/cvtt2/tmp # Local
|
||||
# export LogDir=/home/oleg/develop/cvtt2/tmp
|
||||
|
||||
mkdir -p ${LogDir}
|
||||
|
||||
run_proc() {
|
||||
Inst=${1}
|
||||
Start=${2}
|
||||
End=${3}
|
||||
echo "Running for $Inst"
|
||||
|
||||
Cmd="${Python} ${PyScript}"
|
||||
Cmd="${Cmd} --config=${Config}"
|
||||
Cmd="${Cmd} --output_dir=${OutputDir}/${Start:0:4}"
|
||||
Cmd="${Cmd} --instruments=ALPACA:${Inst}"
|
||||
Cmd="${Cmd} --start=${Start}"
|
||||
if [ "${End}" != "" ]; then
|
||||
Cmd="${Cmd} --end=${End}"
|
||||
fi
|
||||
Cmd="${Cmd} --log_file=${LogDir}/${Inst}.log"
|
||||
Cmd="${Cmd} --log_level=WARNING"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
}
|
||||
|
||||
export -f run_proc
|
||||
|
||||
if [ "$InstListFile" == "" ]; then
|
||||
|
||||
key=$(jq -r '.["ALPACA_SANDBOX"] | .api_key' ~/.creds)
|
||||
secret=$(jq -r '.["ALPACA_SANDBOX"] | .secret_key' ~/.creds)
|
||||
|
||||
Cmd="curl -s --request GET --url 'https://paper-api.alpaca.markets/v2/assets?status=active'"
|
||||
Cmd="${Cmd} --header 'APCA-API-KEY-ID: ${key}'"
|
||||
Cmd="${Cmd} --header 'APCA-API-SECRET-KEY: ${secret}'"
|
||||
Cmd="${Cmd} --header 'accept: application/json'"
|
||||
Cmd="${Cmd} | jq '.[] | select(.class == \"us_equity\" and .exchange != \"OTC\") | .symbol'"
|
||||
Cmd="${Cmd} | sed 's/\"//g'"
|
||||
Cmd="${Cmd} | sed 's/^/STOCK-/'"
|
||||
else
|
||||
Cmd="cat ${InstListFile}"
|
||||
fi
|
||||
|
||||
# split string into array
|
||||
Instruments=()
|
||||
for Inst in $(eval ${Cmd})
|
||||
do
|
||||
Instruments+=("$Inst")
|
||||
done
|
||||
|
||||
slice_size=500 # 10K symbols parallel cannot handle
|
||||
for ((ii=0; ii <${#Instruments[@]}; ii+=slice_size)); do
|
||||
InstSlice=("${Instruments[@]:ii:slice_size}")
|
||||
|
||||
parallel -j ${NumJobs} run_proc {} ${Start} ${End} ::: "${InstSlice[@]}"
|
||||
done
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
|
||||
is_business_day() {
|
||||
dt=${1}
|
||||
|
||||
CalendarURL=http://cloud16.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
||||
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||
if [ -n "${open_time}" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
export -f is_business_day
|
||||
|
||||
# Dates are in descending order
|
||||
FinishedDateFile=${HOME}/prod/run/alpaca_md_last_date
|
||||
Script=${HOME}/prod/run/alpaca_md.sh
|
||||
|
||||
last_date=$(cat ${FinishedDateFile})
|
||||
|
||||
if [ "${last_date}" == "" ]; then
|
||||
echo "process is currently on"
|
||||
exit
|
||||
fi
|
||||
|
||||
while true; do
|
||||
new_date=$(date -d "${last_date} - 1 day" "+%Y-%m-%d")
|
||||
if is_business_day ${new_date}; then
|
||||
break
|
||||
else
|
||||
last_date=${new_date}
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
echo Collecting data for ${new_date}
|
||||
|
||||
Cmd="${Script} ${new_date}"
|
||||
|
||||
echo "${0} Starting at $(date)"
|
||||
|
||||
echo > ${FinishedDateFile}
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
echo $new_date > ${FinishedDateFile}
|
||||
|
||||
echo "${0} Done at $(date)"
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# ---------------- Settings
|
||||
declare -A git_repo_arr
|
||||
|
||||
git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
|
||||
git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
|
||||
git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
|
||||
|
||||
default_project=cvttpy
|
||||
default_branch=master
|
||||
@@ -63,6 +64,13 @@ else
|
||||
fi
|
||||
|
||||
tmp_dir=$(mktemp -d)
|
||||
function cleanup {
|
||||
cd ${HOME}
|
||||
rm -rf ${tmp_dir}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
prj_dir="${tmp_dir}/${prj}"
|
||||
|
||||
cmd_arr=()
|
||||
@@ -146,10 +154,7 @@ confirm
|
||||
|
||||
for cmd in "${cmd_arr[@]}"
|
||||
do
|
||||
echo ${cmd} && eval ${cmd}
|
||||
pwd && echo ${cmd} && eval ${cmd}
|
||||
done
|
||||
|
||||
cd ${HOME}
|
||||
rm -rf ${tmp_dir}
|
||||
|
||||
echo $0 Done
|
||||
echo "$0 Done ${project} ${release_version}"
|
||||
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
date=${1}
|
||||
if [ "${date}" == "" ] ; then
|
||||
date="yesterday"
|
||||
fi
|
||||
|
||||
SourceHost=cryptoval3.cvtt.vpn
|
||||
SourceUser=cvtt
|
||||
SourceRootDir=/localdisk/cvtt/archive/md_archive
|
||||
SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
|
||||
SourceFileZip="${SourceFile}.gz"
|
||||
SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
|
||||
|
||||
TargetDir="/opt/jupyter_gpu/data/crypto_md"
|
||||
TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
|
||||
TargetFilePath="${TargetDir}/${TargetFile}"
|
||||
|
||||
echo ${SourceFile}
|
||||
tmp_dir=$(mktemp -d)
|
||||
|
||||
function cleanup {
|
||||
cd ${HOME}
|
||||
rm -rf ${tmp_dir}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
function download_file {
|
||||
|
||||
Cmd="rsync"
|
||||
Cmd="${Cmd} -ahv"
|
||||
if tty -s; then
|
||||
Cmd="${Cmd} --progress=info2"
|
||||
fi
|
||||
Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
ls -lh ${tmp_dir}
|
||||
Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
ls -lh ${tmp_dir}
|
||||
|
||||
rm -f ${TargetFilePath}
|
||||
touch ${TargetFilePath}
|
||||
|
||||
for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
|
||||
do
|
||||
Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
done
|
||||
chmod 600 ${TargetFilePath}
|
||||
ls -lh ${TargetFilePath}
|
||||
|
||||
|
||||
}
|
||||
download_file
|
||||
@@ -15,7 +15,6 @@ Sender=${RootDir}/ops/utils/send_mmost.sh
|
||||
|
||||
Hosts=
|
||||
Hosts="${Hosts} cloud11.cvtt.vpn"
|
||||
Hosts="${Hosts} cloud14.cvtt.vpn"
|
||||
Hosts="${Hosts} cloud15.cvtt.vpn"
|
||||
Hosts="${Hosts} cloud16.cvtt.vpn"
|
||||
Hosts="${Hosts} cloud17.cvtt.vpn"
|
||||
@@ -23,26 +22,43 @@ Hosts="${Hosts} cloud21.cvtt.vpn"
|
||||
Hosts="${Hosts} cloud22.cryptovaltrading.com"
|
||||
|
||||
Hosts="${Hosts} cvttdata.cvtt.vpn"
|
||||
Hosts="${Hosts} cryptoval1.cvtt.vpn"
|
||||
Hosts="${Hosts} cryptoval2.cvtt.vpn"
|
||||
Hosts="${Hosts} cryptoval3.cvtt.vpn"
|
||||
|
||||
Hosts="${Hosts} homestore.cvtt.vpn"
|
||||
Hosts="${Hosts} nsbackup.cvtt.vpn"
|
||||
Hosts="${Hosts} nsbackup.sheynin.home"
|
||||
Hosts="${Hosts} dtvmhost.cvtt.vpn"
|
||||
Hosts="${Hosts} ops-server.cvtt.vpn"
|
||||
|
||||
Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
|
||||
Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
|
||||
Hosts="${Hosts} cvtt-prod-03.cvtt.vpn"
|
||||
|
||||
Metrics=()
|
||||
TempFiles=
|
||||
|
||||
function cleanup {
|
||||
if [ "" != "${TempFiles}" ]; then
|
||||
rm -f ${TempFiles}
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
function new_tempfile {
|
||||
tmpfile=$(mktemp)
|
||||
TempFiles="${TempFiles} ${tmpfile}"
|
||||
}
|
||||
|
||||
function space_alert() {
|
||||
ALERT_USAGE=75%
|
||||
for ln in "${Measurements[@]}"
|
||||
for metric in "${Metrics[@]}"
|
||||
do
|
||||
IFS=$' '; args=($ln); unset IFS
|
||||
IFS=$' '; args=($metric); unset IFS
|
||||
host=${args[0]}
|
||||
fs=${args[1]}
|
||||
space_used=${args[2]}
|
||||
|
||||
if [ ${space_used%?} -ge ${ALERT_USAGE%?} ]; then
|
||||
echo ":red_circle: Filesystem **${host}:${fs}** is using **${space_used}** :red_circle:"
|
||||
fi
|
||||
@@ -53,6 +69,7 @@ function storage_check() {
|
||||
|
||||
result_lines=()
|
||||
|
||||
declare -a SingleMeas
|
||||
for host in ${Hosts}
|
||||
do
|
||||
if [[ "${host}" == *"cryptovaltrading.com" ]]; then
|
||||
@@ -71,7 +88,12 @@ function storage_check() {
|
||||
do
|
||||
IFS=$' '; args=($ln); unset IFS
|
||||
res="${args[5]}| **${host}** | ***${args[6]}*** | *${args[5]}* |"
|
||||
|
||||
result_lines+=("$res")
|
||||
|
||||
fs=${args[6]}
|
||||
used=${args[5]}
|
||||
Metrics+=("$host $fs $used")
|
||||
done
|
||||
unset IFS
|
||||
done
|
||||
@@ -81,8 +103,8 @@ function storage_check() {
|
||||
done
|
||||
}
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
tmpfile2=$(mktemp)
|
||||
tmpfile=$(new_tempfile)
|
||||
tmpfile2=$(new_tempfile)
|
||||
storage_check > ${tmpfile2}
|
||||
|
||||
echo "## :card_file_box: STORAGE HEALTH CHECK" >> ${tmpfile}
|
||||
@@ -91,10 +113,9 @@ echo "| host | filesystem | usage |" >> ${tmpfile}
|
||||
echo "| --- | --- | --- |" >> ${tmpfile}
|
||||
cat ${tmpfile2} | sort -h -r | awk -F'%' '{printf "%s%%%s\n",$2,$3}' >> ${tmpfile}
|
||||
cat ${tmpfile} | ${Sender} ${StatusChannel}
|
||||
rm ${tmpfile} ${tmpfile2}
|
||||
|
||||
Measurements=()
|
||||
tmpfile=$(mktemp)
|
||||
tmpfile=$(new_tempfile)
|
||||
space_alert > ${tmpfile}
|
||||
cat ${tmpfile}
|
||||
if [ -s ${tmpfile} ]
|
||||
@@ -103,5 +124,4 @@ then
|
||||
else
|
||||
echo File ${tmpfile} is empty
|
||||
fi
|
||||
rm ${tmpfile}
|
||||
|
||||
|
||||
+22
-19
@@ -1,5 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
|
||||
# ----- Settings
|
||||
LocalSoftwareDir=${HOME}/software/cvtt2
|
||||
|
||||
ProdDir=${HOME}/prod
|
||||
|
||||
ReleaseHosts=("cloud21.cvtt.vpn")
|
||||
ReleasePorts=("22")
|
||||
ReleaseUsers=("cvttdist")
|
||||
ReleaseDir=("/home/cvttdist/software/cvtt2")
|
||||
# ----- Settings
|
||||
|
||||
function usage() {
|
||||
echo "Usage: ${0} <project> <version>"
|
||||
exit 1
|
||||
@@ -15,22 +27,9 @@ fi
|
||||
|
||||
if [ "${Version}" == "" ]
|
||||
then
|
||||
usage
|
||||
Version=latest
|
||||
fi
|
||||
|
||||
# ----- Settings
|
||||
LocalSoftwareDir=${HOME}/software
|
||||
|
||||
ProdDir=${HOME}/prod
|
||||
|
||||
ReleaseHosts=("cloud21.cvtt.vpn")
|
||||
ReleasePorts=("22")
|
||||
ReleaseUsers=("cvttdist")
|
||||
ReleaseDir=("/home/cvttdist/software/cvtt2")
|
||||
# ----- Settings
|
||||
|
||||
Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
|
||||
|
||||
function rsync_load_version() {
|
||||
for idx in "${!ReleaseHosts[@]}"
|
||||
do
|
||||
@@ -39,6 +38,11 @@ function rsync_load_version() {
|
||||
user=${ReleaseUsers[${idx}]}
|
||||
rel_dir=${ReleaseDir[${idx}]}
|
||||
|
||||
if [ "${Version}" == "latest" ]; then
|
||||
echo "Checking for latest version of ${Project} on ${user}@${host}:${rel_dir}"
|
||||
Version=$(ssh -q -p ${port} ${user}@${host} "ls -tr ${rel_dir}/${Project} | tail -1" )
|
||||
echo "Latest version is ${Version}"
|
||||
fi
|
||||
echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
|
||||
if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
|
||||
then
|
||||
@@ -65,18 +69,17 @@ function rsync_load_version() {
|
||||
}
|
||||
|
||||
mkdir -p ${LocalSoftwareDir}
|
||||
mkdir -p ${ProdDir}
|
||||
|
||||
# exists and not empty
|
||||
if [[ ! -d "${Location}" ]] ; then
|
||||
rsync_load_version
|
||||
fi
|
||||
|
||||
Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
|
||||
|
||||
Cmd="cd ${ProdDir}"
|
||||
|
||||
Cmd="${Cmd} && rm -rf ${Project}"
|
||||
Cmd="${Cmd} && ln -snf ${Location} ${Project}"
|
||||
|
||||
echo ${Cmd}
|
||||
eval ${Cmd}
|
||||
echo ${Cmd} && eval ${Cmd}
|
||||
|
||||
echo "Done: $0 $*"
|
||||
|
||||
@@ -26,7 +26,7 @@ Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d')
|
||||
|
||||
echo "Looking for log files older than ${DateCriteria} in ${LogDir}"
|
||||
|
||||
LogArchiveDir=/home/cvtt/prod/archive/logs
|
||||
LogArchiveDir=${HOME}/prod/archive/logs
|
||||
mkdir -p ${LogArchiveDir}
|
||||
|
||||
echo "find ${LogDir}/ '(' -name '*.log' -o -name '*.log.*' ')' -type f -not -newermt ${Oldest})"
|
||||
|
||||
Reference in New Issue
Block a user