build/build_release.sh

@@ -7,7 +7,6 @@ git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
 git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
 git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
 git_repo_arr[cvtt-rust]=git@cloud21.cvtt.vpn:/opt/store/git/cvtt2/cvtt-rust.git
-git_repo_arr[docker_dev]=git@cloud21.cvtt.vpn:/opt/store/git/docker_dev.git
 
 dist_root=/home/cvttdist/software/cvtt2
 dist_user=cvttdist
@@ -170,21 +169,24 @@ cmd_arr+=("${Cmd}")
 SourceLoc=../${project}
 if [ "${project}" == "cvtt-rust" ]; then
 
-    cmd_arr+=("cd ${prj_dir}")
+    Cmd="cd ${dist_root}/${project}/"
-
+    cmd_arr+=("${Cmd}")
-    release_dir=${prj_dir}/dist/${project}
+    # cmd_arr+=("cargo build")
-
+    cmd_arr+=("cargo build --release")
-    cmd_arr+=("mkdir -p ${release_dir}")
+    cmd_arr+=("mkdir -p ./dist/release")
-    cmd_arr+=("cp release_version.txt ${release_dir}")
+    pushd ${dist_root}/${project}/
-
+    rust_bins=$(rust_binaries)
-    jq_cmd="jq '.packages[] | select(.targets[].kind[] == \"bin\") | .name'"
+    rust_libs=$(rust_libraries)
-    apps=$(cargo metadata --no-deps --format-version 1 --manifest-path=${prj_dir}/Cargo.toml | eval $jq_cmd | uniq)
+    popd
-    for app in ${apps[@]}; do
+    for bin in ${rust_bins[@]}; do
-        app=${app//\"/} # remove quotes
+        cmd_arr+=("cp target/release/${bin} ./dist/release")
-        cmd_arr+=("cargo install --root ${release_dir} --path ${prj_dir}/apps/${app}")
     done
+    for lb in ${rust_libs[@]}; do
+        cmd_arr+=("cp target/release/${lb} ./dist/release")
+    done
+    cmd_arr+=("cp release_version.txt ./dist/release")
 
-    SourceLoc=${release_dir}
+    SourceLoc=../${project}/dist/release
 fi
 
 dist_path="${dist_root}/${project}/${release_version}"

build/docker_images_builder.sh

@@ -1,198 +0,0 @@
-#!/bin/bash
-
-usage() {
-    echo -n "Usage: ${0}  <RootDir>"
-    echo
-    exit 1
-}
-# --- Settings
-RootDir=${1}
-if [ -z "${RootDir}" ] ; then
-    usage
-fi
-
-RegistryService=cloud21.cvtt.vpn:5500
-RegistryProtocol=http
-
-ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/docker_images
-
-
-ReleaseHost="cloud21.cvtt.vpn"
-ReleasePort="22"
-ReleaseUser="cvttdist"
-ReleaseDir="/home/cvttdist/software/cvtt2"
-
-DIMAGES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
-
-
-SetVersionScript=${HOME}/prod/set_version.sh
-
-
-# --------------- D E B U G
-
-mkdir -p ${RootDir}
-
-build_docker_image() {
-    title=${1}
-    image_name=${2}
-    docker_dev_path=${3}
-    project=${4}
-
-    echo "Building ${title}..."
-    echo "ImageName=${image_name}"
-    echo "DockerDevPath=${docker_dev_path}"
-    echo "BasedOnDist=${project}"
-
-    # Get Latest DIST version
-    echo "Checking for latest version of ${project} on ${ReleaseUser}@${ReleaseHost}:${ReleaseDir}"
-    image_version=$(get_latest_dist_version ${project})
-    if [ -z "${image_version}" ]; then
-        echo "No latest version found for ${project}"
-        return
-    fi
-    echo "Latest version is ${image_version}"
-
-    if image_exists "${image_name}" "${image_version}"; then
-        echo "Image $image_name:$image_version exists in the registry. Building is not required."
-        return
-    fi
-    echo "Image $image_name:$image_version does not exist in the registry."
-    echo "getting lastest version of ${project} on ${ReleaseUser}@${ReleaseHost}:${ReleaseDir}"
-
-    Cmd="${SetVersionScript} ${project} ${image_version}"
-    echo ${Cmd}
-    eval ${Cmd}
-
-    convert_symlink_to_dir ${project}
-
-    DockerDir=./docker_dev/${docker_dev_path}
-    echo "Building ${title}... on ${DockerDev}"
-
-    Cmd="docker build"
-    Cmd+=" -t ${image_name}"
-    Cmd+=" -t ${image_name}:latest"
-    Cmd+=" -t ${image_name}:${image_version}"
-    Cmd+=" -f ${DockerDir}/Dockerfile ${RootDir}"
-    echo ${Cmd}
-    eval ${Cmd} 1>&2 || exit
-
-    Cmd="docker tag"
-    Cmd+=" ${image_name}:latest"
-    Cmd+=" ${RegistryService}/${image_name}:latest"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    Cmd="docker tag"
-    Cmd+=" ${image_name}:${image_version}"
-    Cmd+=" ${RegistryService}/${image_name}:${image_version}"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    Cmd="docker push"
-    Cmd+=" ${RegistryService}/${image_name}:latest"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    Cmd="docker push"
-    Cmd+=" ${RegistryService}/${image_name}:${image_version}"
-    echo ${Cmd}
-    eval ${Cmd} || exit
-
-    echo "Removing the copy of ${project} from ${RootDir}"
-    Cmd="rm -rf ${RootDir}/${project}"
-    echo ${Cmd}
-    eval ${Cmd}
-
-    print_all_reg_images ${project}
-}
-
-convert_symlink_to_dir() {
-    project=${1}
-    if [ -L ${RootDir}/${project} ]; then
-        echo "Converting symlink ${RootDir}/${project} to directory"
-
-        Cmd="mv ${RootDir}/${project} ${RootDir}/${project}.symlink"
-        echo ${Cmd} && eval ${Cmd}
-
-        src_path=$(readlink -f ${RootDir}/${project}.symlink)
-
-        Cmd="cp -r ${src_path} ${RootDir}/${project}"
-        echo ${Cmd} && eval ${Cmd}
-    fi
-    if [ -d ${project} ]; then
-        echo "${RootDir}/${project} is a Directory"
-    fi
-    Cmd="rm ${RootDir}/${project}.symlink"
-    echo ${Cmd} && eval ${Cmd}
-}
-
-image_exists() {
-    image_name=${1}
-    version=${2}
-    response=$(curl -s -o /dev/null -w "%{http_code}" "${RegistryProtocol}://${RegistryService}/v2/${image_name}/manifests/${version}" -H "Accept: application/vnd.docker.distribution.manifest.v2+json")
-    if [ "$response" -eq 200 ]; then
-        return 0
-    else
-        return 1
-    fi
-}
-
-print_all_reg_images() {
-    project=${1}
-
-    repositories=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/_catalog" | jq -r '.repositories[]')
-
-    for repo in $repositories; do
-        # Fetch all tags for the repository
-        tags=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/$repo/tags/list" | jq -r '.tags[]')
-
-        # List each tag
-        for tag in $tags; do
-            if [ "${project}" == "" ];then
-                echo "$repo:$tag"
-            else
-                echo "$repo:$tag" | grep ${project}
-            fi
-        done
-    done
-}
-
-get_latest_dist_version() {
-    project=${1}
-
-    Version=$(ssh -q -p ${ReleasePort} ${ReleaseUser}@${ReleaseHost} "ls -tr ${ReleaseDir}/${project} | tail -1" )
-    echo "${Version}"
-}
-
-echo RootDir=${RootDir}
-
-cd ${RootDir}
-
-echo "Getting latest version of docker_dev"
-Cmd="${SetVersionScript} docker_dev latest"
-echo ${Cmd}
-eval ${Cmd}
-
-convert_symlink_to_dir docker_dev
-
-Jobs=()
-while IFS= read -r item; do
-    Jobs+=("$item")
-done < <(echo "${DIMAGES_CONFIG}" | jq -c 'to_entries[]')
-
-for item in "${Jobs[@]}"; do
-    # Extract key (name) and values
-    title=$(echo "$item" | jq -r '.key')
-    image_name=$(echo "$item" | jq -r '.value.image_name')
-    docker_dev_path=$(echo "$item" | jq -r '.value.docker_dev_path')
-    project=$(echo "$item" | jq -r '.value.based_on_project')
-
-    build_docker_image "$title" "$image_name" "$docker_dev_path" "$project"
-done
-
-Cmd="rm -rf ${RootDir}/docker_dev"
-echo ${Cmd}
-eval ${Cmd}
-
-#===============================================================================
-#===============================================================================

build/list_docker_images.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-usage() {
-    echo -n "Usage: ${0}  <grep_filter>"
-    echo
-    exit 1
-}
-# --- Settings
-RegistryService=cloud21.cvtt.vpn:5500
-RegistryProtocol=http
-
-print_all_reg_images() {
-    project=${1}
-
-    repositories=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/_catalog" | jq -r '.repositories[]')
-
-    for repo in $repositories; do
-        # Fetch all tags for the repository
-        tags=$(curl -s "${RegistryProtocol}://${RegistryService}/v2/$repo/tags/list" | jq -r '.tags[]')
-
-        # List each tag
-        echo "REPO: $repo"
-        for tag in $tags; do
-            if [ "${project}" == "" ];then
-                echo "$repo:$tag"
-            else
-                echo "$repo:$tag" | grep ${project}
-            fi
-        done
-    done
-}
-
-print_all_reg_images ${1}

docker/research/crypto_exch_stats.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-Cmd="docker run"
-Cmd+=" -d"
-Cmd+=" --rm"
-Cmd+=" --pull=always"
-Cmd+=" --network=host"
-Cmd+=" --name=crypto_exch_stats"
-Cmd+=" --volume=${HOME}/prod/data:/app/data"
-Cmd+=" --volume=${HOME}/prod/logs:/logs"
-Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest"
-
-echo ${Cmd}
-eval ${Cmd}
-# Cmd+=" cloud21.cvtt.vpn:5500/relative_liquidity:latest"
-

release_version.txt

@@ -1 +1 @@
-1.8.6,crypto_exch_stats docker run
+1.7.9,build for cvtt-rust

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -7,6 +7,7 @@ usage() {
 }
 
 RootDir="${HOME}/prod"
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
 
 export PYTHONPATH=${RootDir}
 
@@ -14,33 +15,28 @@ StatusChannel="MD-Status"
 Sender=${RootDir}/ops/utils/send_mmost.sh
 
 # ----- For DEBUGGING
-# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+# Sender=cat
-# Sender=cat #### D E B U G
+# StatusChannel=
-# StatusChannel=  #### D E B U G
 
 run_checklist() {
 
 
     yr=$(date -d 'yesterday' '+%Y')
-    CheckSymbols="A/AAPL N/NVDA M/META"
-    declare -A Commands
-    Commands["homestore"]=""
-    for sym in ${CheckSymbols}; do
-        Commands["homestore"]+="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
-    done
-    Commands["homestore"]+="echo"
 
-    Commands["cloud21"]=""
+    declare -A Commands
-    for sym in ${CheckSymbols}; do
+    Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
-        Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
+    Commands["homestore"]+="; echo"
-    done
+    Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
-    Commands["cloud21"]+="echo"
+
+    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
+    Commands["cloud21"]+="; echo"
+    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
 
     Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
 
     for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
-        echo "------- ${name}"
         Cmd=${Commands[${name}]}
+        echo "------- ${name}"
         echo ${Cmd}
         eval ${Cmd}
     done

scripts/set_version.sh

@@ -4,7 +4,7 @@
 # ----- Settings
 LocalSoftwareDir=${HOME}/software/cvtt2
 
-ProdDir=$(pwd)
+ProdDir=${HOME}/prod
 
 ReleaseHosts=("cloud21.cvtt.vpn")
 ReleasePorts=("22")
@@ -77,9 +77,9 @@ rsync_load_version
 Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
 
 Cmd="cd ${ProdDir}"
-Cmd+=" && rm -rf ${Project}"
+Cmd="${Cmd} && rm -rf ${Project}"
-Cmd+=" && ln -snf ${Location} ${Project}"
+Cmd="${Cmd} && ln -snf ${Location} ${Project}"
 
 echo ${Cmd} && eval ${Cmd}
 
-echo "Done: $0 ${*}"
+echo "Done: $0 $*"