progress

release_version.txt

@@ -1 +1 @@
-0.4.9
+0.5.8

research/load_histdata.sh

@@ -14,7 +14,7 @@ function load_zipfile {
 }
 
 # ---------------------------------------------------
-root_dir=/home/cvtt/prod/eqt_hist_md
+root_dir=/opt/store/cvtt/eqt_hist_md
 
 log_dir=/home/cvtt/prod/logs
 url_root="https://firstratedata.com/datafile/89CgWiwc60CylOIEFRDTCg"

research/prune_eqt_histdata.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # ---------------------------------------------------
-root_dir=/home/cvtt/prod/eqt_hist_md
+root_dir=/opt/store/cvtt/eqt_hist_md
 
 prune_criteria="6 month ago"
 

scripts/archive_yesterday_md.sh

@@ -11,7 +11,7 @@ then
     CredKey=TSDB_MD_CVTTDATA_RO
 elif [ "${host}" == "cloud21.cryptovaltrading.com" ]
 then
-    ArchiveRootDir=/opt/cvtt/archive/md_archive
+    ArchiveRootDir=/opt/store/cvtt/archive/md_archive
     CredKey=TSDB_MD_CLD21_RO
 else
     echo "Unknown host ${host}. ${0} Aborted."

scripts/build_release.sh

@@ -0,0 +1,148 @@
+#!/bin/bash
+
+# ---------------- Settings
+declare -A git_repo_arr
+
+git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
+git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
+
+default_project=cvttpy
+default_branch=master
+
+
+dist_root=/home/cvttdist/software/cvtt2
+dist_user=cvttdist
+dist_host="cloud21.cvtt.vpn"
+dist_ssh_port="22"
+
+interactive=Y
+
+# cmdline
+prj=${1:-${default_project}}
+brnch=${2:-${default_branch}}
+if [ "${3}" == "D" ]; then
+    interactive=N
+fi
+
+function confirm {
+    if [ "${interactive}" == "Y" ]; then
+        echo "--------------------------------"
+        echo -n "Press <Enter> to continue" && read
+    fi
+}
+
+if [ "${interactive}" == "Y" ]; then
+    echo -n "Enter project [${prj}]: "
+    read project
+    if [ "${project}" == "" ]
+    then
+        project=${prj}
+    fi
+else
+    project=${prj}
+fi
+
+repo=${git_repo_arr[${project}]}
+if [ -z ${repo} ]; then
+    echo "ERROR: Project repository for ${project} not found"
+    exit -1
+fi
+echo "Project repo: ${repo}"
+
+if [ "${interactive}" == "Y" ]; then
+    echo -n "Enter branch to build release from [${brnch}]: "
+    read branch
+    if [ "${branch}" == "" ]
+    then
+        branch=${brnch}
+    fi
+else
+    branch=${brnch}
+fi
+
+tmp_dir=$(mktemp -d)
+prj_dir="${tmp_dir}/${prj}"
+
+cmd_arr=()
+Cmd="git clone ${repo} ${prj_dir}"
+cmd_arr+=("${Cmd}")
+
+Cmd="cd ${prj_dir}"
+cmd_arr+=("${Cmd}")
+
+if [ "${interactive}" == "Y" ]; then
+    echo "------------------------------------"
+    echo "The following commands will execute:"
+    echo "------------------------------------"
+    for cmd in "${cmd_arr[@]}"
+    do
+        echo ${cmd}
+    done
+fi
+
+confirm
+
+for cmd in "${cmd_arr[@]}"
+do
+    echo ${cmd} && eval ${cmd}
+done
+
+Cmd="git checkout ${branch}"
+echo ${Cmd} && eval ${Cmd}
+if [ "${?}" != "0" ]; then
+    echo "ERROR: Branch ${branch} is not found"
+    cd ${HOME} && rm -rf ${tmp_dir}
+    exit -1
+fi
+
+
+release_version=$(cat release_version.txt)
+
+
+echo "--------------------------------"
+echo "Release version: ${release_version}"
+
+confirm
+
+version_tag="v${release_version}"
+version_comment="${version_tag}_${project}_${branch}_$(date +%Y%m%d)"
+
+cmd_arr=()
+Cmd="git tag -a ${version_tag} -m ${version_comment}"
+cmd_arr+=("${Cmd}")
+
+Cmd="git push origin --tags"
+cmd_arr+=("${Cmd}")
+
+Cmd="rm -rf .git"
+cmd_arr+=("${Cmd}")
+
+dist_path="${dist_root}/${project}/${release_version}"
+
+Cmd="rsync -avzh"
+Cmd="${Cmd} --rsync-path=\"mkdir -p ${dist_path} && rsync\""
+Cmd="${Cmd} -e \"ssh -p ${dist_ssh_port}\""
+Cmd="${Cmd} ../${project} ${dist_user}@${dist_host}:${dist_path}/"
+cmd_arr+=("${Cmd}")
+
+if [ "${interactive}" == "Y" ]; then
+    echo "------------------------------------"
+    echo "The following commands will execute:"
+    echo "------------------------------------"
+    for cmd in "${cmd_arr[@]}"
+    do
+        echo ${cmd}
+    done
+fi
+
+confirm
+
+for cmd in "${cmd_arr[@]}"
+do
+    echo ${cmd} && eval ${cmd}
+done
+
+cd ${HOME}
+rm -rf ${tmp_dir}
+
+echo $0 Done

scripts/healthcheck/storage_health_check.sh

@@ -19,8 +19,8 @@ Hosts="${Hosts} cloud14.cvtt.vpn"
 Hosts="${Hosts} cloud15.cvtt.vpn"
 Hosts="${Hosts} cloud16.cvtt.vpn"
 Hosts="${Hosts} cloud17.cvtt.vpn"
-Hosts="${Hosts} cloud19.cvtt.vpn"
+Hosts="${Hosts} cloud21.cvtt.vpn"
-Hosts="${Hosts} cloudstore.cvtt.vpn"
+Hosts="${Hosts} cloud22.cryptovaltrading.com"
 
 Hosts="${Hosts} cvttdata.cvtt.vpn"
 Hosts="${Hosts} cryptoval1.cvtt.vpn"
@@ -32,6 +32,9 @@ Hosts="${Hosts} nsbackup.cvtt.vpn"
 Hosts="${Hosts} dtvmhost.cvtt.vpn"
 Hosts="${Hosts} ops-server.cvtt.vpn"
 
+Hosts="${Hosts} cvtt-prod-01.cvtt.vpn"
+Hosts="${Hosts} cvtt-prod-02.cvtt.vpn"
+
 function space_alert() {
     ALERT_USAGE=75%
     for ln in "${Measurements[@]}"
@@ -52,7 +55,12 @@ function storage_check() {
 
     for host in ${Hosts}
     do
-        Cmd="ssh $host"
+        if [[ "${host}" == *"cryptovaltrading.com" ]]; then
+            port=7822
+        else
+            port=22
+        fi
+        Cmd="ssh -p ${port} $host"
         Cmd="${Cmd} eval \"df -hTl"
         Cmd="${Cmd} -x squashfs"
         Cmd="${Cmd} | grep -v tmpfs"

scripts/set_version.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+function usage() {
+    echo "Usage: ${0} <project> <version>"
+    exit 1
+
+}
+Project=${1}
+Version=${2}
+
+if [ "${Project}" == "" ]
+then
+    usage
+fi
+
+if [ "${Version}" == "" ]
+then
+    usage
+fi
+
+# ----- Settings
+LocalSoftwareDir=${HOME}/software
+
+ProdDir=${HOME}/prod
+
+ReleaseHosts=("cloud21.cvtt.vpn")
+ReleasePorts=("22")
+ReleaseUsers=("cvttdist")
+ReleaseDir=("/home/cvttdist/software/cvtt2")
+# ----- Settings
+
+Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
+
+function rsync_load_version() {
+    for idx in "${!ReleaseHosts[@]}"
+    do
+        host=${ReleaseHosts[${idx}]}
+        port=${ReleasePorts[${idx}]}
+        user=${ReleaseUsers[${idx}]}
+        rel_dir=${ReleaseDir[${idx}]}
+
+        echo "Checking ${user}@${host} for ${rel_dir}/${Project}/${Version} ..."
+        if ssh -q -p ${port} ${user}@${host} "test -d ${rel_dir}/${Project}/${Version}"
+        then
+            echo "Directory found..."
+            rsync_cmd="rsync -ahvv -e \"ssh -p ${port}\""
+            rsync_cmd="${rsync_cmd} ${user}@${host}:${rel_dir}/${Project}/${Version}"
+            rsync_cmd="${rsync_cmd} ${LocalSoftwareDir}/${Project}/"
+            echo ${rsync_cmd}
+            eval ${rsync_cmd}
+            status=$?
+            if [ ${status} -eq 0 ]
+            then
+                echo "Loading successful..."
+                break
+            fi
+        else
+            echo "Not Found ${rel_dir}/${Project}/${Version} on ${user}@${host}"
+        fi
+    done
+    if [[ ! -d ${LocalSoftwareDir}/${Project} ]] ; then
+        echo ERROR loading software
+        exit 1
+    fi
+}
+
+mkdir -p ${LocalSoftwareDir}
+
+# exists and not empty
+if [[ ! -d "${Location}" ]] ; then
+    rsync_load_version
+fi
+
+Cmd="cd ${ProdDir}"
+
+Cmd="${Cmd} && rm -rf ${Project}"
+Cmd="${Cmd} && ln -snf ${Location} ${Project}"
+
+echo ${Cmd}
+eval ${Cmd}
+
+echo "Done: $0 $*"

utils/move_archives.sh

@@ -30,7 +30,8 @@ fi
 
 DateCriteria="${Days} days ago"
 
-TargeHost=cloudstore.cvtt.vpn
+TargetHost=cloud21.cvtt.vpn
+TargetRootDir=/opt/store/cvtt/archive
 
 Oldest=$(date -d "${DateCriteria}" '+%Y-%m-%d')
 Now=$(date '+%Y%m%d_%H%M%S')
@@ -53,7 +54,7 @@ if [ "$files" == "" ]
 then
     echo "No files found older than ${Oldest} in ${ArchiveDir}"
 else
-    Target="${TargetHost}:/home/cvtt/Archive/${FromHost}/"
+    Target="${TargetHost}:${TargetRootDir}/${FromHost}/"
     echo "Moving files to ${Target}:"
     echo -----------------
     for f in ${files}
@@ -66,7 +67,7 @@ else
     Cmd="${Cmd} $files"
     Cmd="${Cmd} ${Target}"
     echo ${Cmd}
-    # exit
+# exit
     eval ${Cmd}
 fi
 echo Done ${0} ${*}

utils/send_mmost.sh

@@ -11,6 +11,10 @@ url=$(jq -r '.MATTERMOST.url' ${HOME}/.creds)
 team=$(jq -r '.MATTERMOST.team' ${HOME}/.creds)
 bearer=$(jq -r '.MATTERMOST.bearer' ${HOME}/.creds)
 
+#lowercase channel
+channel_name=$(echo "${channel_name}" | awk '{print tolower($0)}')
+# space to '-'
+channel_name=${channel_name//" "/"-"}
 
 get_channel_url="${url}/api/v4/teams/name/${team}/channels/name/${channel_name}"