fix

build/build_release.sh

@@ -7,6 +7,7 @@ git_repo_arr[cvttpy]=git@cloud21.cvtt.vpn:/opt/store/git/cvttpy.git
 git_repo_arr[ops]=git@cloud21.cvtt.vpn:/opt/store/git/ops.git
 git_repo_arr[research]=git@cloud21.cvtt.vpn:/opt/store/git/research.git
 git_repo_arr[cvtt-rust]=git@cloud21.cvtt.vpn:/opt/store/git/cvtt2/cvtt-rust.git
+git_repo_arr[docker_dev]=git@cloud21.cvtt.vpn:/opt/store/git/docker_dev.git
 
 dist_root=/home/cvttdist/software/cvtt2
 dist_user=cvttdist
@@ -169,24 +170,21 @@ cmd_arr+=("${Cmd}")
 SourceLoc=../${project}
 if [ "${project}" == "cvtt-rust" ]; then
 
-    Cmd="cd ${dist_root}/${project}/"
-    cmd_arr+=("${Cmd}")
-    # cmd_arr+=("cargo build")
-    cmd_arr+=("cargo build --release")
-    cmd_arr+=("mkdir -p ./dist/release")
-    pushd ${dist_root}/${project}/
-    rust_bins=$(rust_binaries)
-    rust_libs=$(rust_libraries)
-    popd
-    for bin in ${rust_bins[@]}; do
-        cmd_arr+=("cp target/release/${bin} ./dist/release")
-    done
-    for lb in ${rust_libs[@]}; do
-        cmd_arr+=("cp target/release/${lb} ./dist/release")
-    done
-    cmd_arr+=("cp release_version.txt ./dist/release")
+    cmd_arr+=("cd ${prj_dir}")
 
-    SourceLoc=../${project}/dist/release
+    release_dir=${prj_dir}/dist/${project}
+
+    cmd_arr+=("mkdir -p ${release_dir}")
+    cmd_arr+=("cp release_version.txt ${release_dir}")
+
+    jq_cmd="jq '.packages[] | select(.targets[].kind[] == \"bin\") | .name'"
+    apps=$(cargo metadata --no-deps --format-version 1 --manifest-path=${prj_dir}/Cargo.toml | eval $jq_cmd | uniq)
+    for app in ${apps[@]}; do
+        app=${app//\"/} # remove quotes
+        cmd_arr+=("cargo install --root ${release_dir} --path ${prj_dir}/apps/${app}")
+    done
+
+    SourceLoc=${release_dir}
 fi
 
 dist_path="${dist_root}/${project}/${release_version}"

build/docker_images_builder.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+usage() {
+    echo -n "Usage: ${0}  <RootDir>"
+    echo
+    exit 1
+}
+# --- Settings
+RootDir=${1}
+if [ -z "${RootDir}" ] ; then
+    usage
+fi
+
+RegistryService=cloud21.cvtt.vpn:5500
+RegistryProtocol=http
+
+ConfigUrl=http://cloud23.cvtt.vpn:6789/admin/docker_images
+
+
+ReleaseHost="cloud21.cvtt.vpn"
+ReleasePort="22"
+ReleaseUser="cvttdist"
+ReleaseDir="/home/cvttdist/software/cvtt2"
+
+DIMAGES_CONFIG=$(curl -s ${ConfigUrl} | ${HOME}/bin/hjson -j)
+
+# RootDir=/home/oleg/prod
+SetVersionScript=/home/oleg/prod/ops/scripts/set_version.sh
+
+
+# --------------- D E B U G
+
+mkdir -p ${RootDir}
+
+build_docker_image() {
+    title=${1}
+    image_name=${2}
+    docker_dev=${3}
+    project=${4}
+
+    echo "Building ${title}..."
+    echo "ImageName=${image_name}"
+    echo "DockerDev=${docker_dev}"
+    echo "BasedOnDist=${project}"
+
+    # Get Latest DIST version
+    echo "Checking for latest version of ${project} on ${ReleaseUser}@${ReleaseHost}:${ReleaseDir}"
+    image_version=$(get_latest_dist_version ${project})
+    if [ -z "${image_version}" ]; then
+        echo "No latest version found for ${project}"
+        exit 1
+    fi
+    echo "Latest version is ${image_version}"
+
+    if image_exists "${image_name}" "${image_version}"; then
+        echo "Image $image_name:$image_version exists in the registry. Building is not required."
+        return
+    fi
+    echo "Image $image_name:$image_version does not exist in the registry."
+    echo "getting lastest version of ${project} on ${ReleaseUser}@${ReleaseHost}:${ReleaseDir}"
+    ${SetVersionScript} ${project} ${image_version}
+
+}
+
+image_exists() {
+    image_name=${1}
+    version=${2}
+    response=$(curl -s -o /dev/null -w "%{http_code}" "${RegistryProtocol}://${RegistryService}/v2/${image_name}/manifests/${version}" -H "Accept: application/vnd.docker.distribution.manifest.v2+json")
+    if [ "$response" -eq 200 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+get_latest_dist_version() {
+    project=${1}
+
+    Version=$(ssh -q -p ${ReleasePort} ${ReleaseUser}@${ReleaseHost} "ls -tr ${ReleaseDir}/${project} | tail -1" )
+    echo "${Version}"
+}
+
+echo RootDir=${RootDir}
+
+cd ${RootDir}
+echo "Getting latest version of docker_dev"
+Cmd="${SetVersionScript} docker_dev latest"
+echo ${Cmd}
+eval ${Cmd}
+
+echo "${DIMAGES_CONFIG}" | jq -c 'to_entries[]' | while IFS= read -r item; do
+    # Extract key (name) and values
+    title=$(echo "$item" | jq -r '.key')
+    image_name=$(echo "$item" | jq -r '.value.image_name')
+    docker_dev=$(echo "$item" | jq -r '.value.docker_dev')
+    project=$(echo "$item" | jq -r '.value.based_on_dist')
+    build_docker_image "$title" "$image_name" "$docker_dev" "$project"
+done
+
+exit
+#===============================================================================
+#===============================================================================
+
+DockerDir=$(realpath $(dirname ${0})/${AppPath})
+
+AppName=$(basename ${AppPath})
+
+ImageName=${AppName}
+ImageDir=${HOME}/docker_images
+mkdir -p ${ImageDir}
+
+Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
+
+cd ${RootDir}
+
+Cmd="docker build -t ${ImageName} -t ${ImageName}:latest -t ${ImageName}:${Version} -f ${DockerDir}/Dockerfile ."
+echo ${Cmd}
+eval ${Cmd} || exit
+
+Cmd="docker tag ${ImageName}:latest ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd} || exit
+
+Cmd="docker tag ${ImageName}:${Version} ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd} || exit
+
+Cmd="docker push ${RegistryService}/${ImageName}:latest"
+echo ${Cmd}
+eval ${Cmd} || exit
+
+Cmd="docker push ${RegistryService}/${ImageName}:${Version}"
+echo ${Cmd}
+eval ${Cmd} || exit
+
+

release_version.txt

@@ -1 +1 @@
-1.7.9,build for cvtt-rust
+1.8.3,changes to build scripts 

scripts/mkt_data/checklists/eqty_md_chcklst.sh

@@ -7,7 +7,6 @@ usage() {
 }
 
 RootDir="${HOME}/prod"
-# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
 
 export PYTHONPATH=${RootDir}
 
@@ -15,28 +14,33 @@ StatusChannel="MD-Status"
 Sender=${RootDir}/ops/utils/send_mmost.sh
 
 # ----- For DEBUGGING
-# Sender=cat
-# StatusChannel=
+# RootDir=/home/oleg/develop/cvtt2  ###### D E B U G
+# Sender=cat #### D E B U G
+# StatusChannel=  #### D E B U G
 
 run_checklist() {
 
 
     yr=$(date -d 'yesterday' '+%Y')
-
+    CheckSymbols="A/AAPL N/NVDA M/META"
     declare -A Commands
-    Commands["homestore"]="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
-    Commands["homestore"]+="; echo"
-    Commands["homestore"]+="; ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+    Commands["homestore"]=""
+    for sym in ${CheckSymbols}; do
+        Commands["homestore"]+="ssh cvtt@homestore.cvtt.vpn ls -l /works/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
+    done
+    Commands["homestore"]+="echo"
 
-    Commands["cloud21"]="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/A/AAPL/${yr} | tail -3"
-    Commands["cloud21"]+="; echo"
-    Commands["cloud21"]+="; ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/N/NVDA/${yr} | tail -3"
+    Commands["cloud21"]=""
+    for sym in ${CheckSymbols}; do
+        Commands["cloud21"]+="ssh cvtt@cloud21.cvtt.vpn ls -l /opt/store/cvtt/md_archive/equity/alpaca_md/${yr}/${sym} | tail -3; "
+    done
+    Commands["cloud21"]+="echo"
 
     Commands["gpushnik"]="ssh oleg@gpushnik.cvtt.vpn 'ls -l /opt/jupyter_gpu/data/eqty_md | tail -10'"
 
     for name in $(printf "%s\n" "${!Commands[@]}" | sort); do
-        Cmd=${Commands[${name}]}
         echo "------- ${name}"
+        Cmd=${Commands[${name}]}
         echo ${Cmd}
         eval ${Cmd}
     done

scripts/set_version.sh

@@ -4,7 +4,7 @@
 # ----- Settings
 LocalSoftwareDir=${HOME}/software/cvtt2
 
-ProdDir=${HOME}/prod
+ProdDir=$(pwd)
 
 ReleaseHosts=("cloud21.cvtt.vpn")
 ReleasePorts=("22")
@@ -77,9 +77,9 @@ rsync_load_version
 Location="${LocalSoftwareDir}/${Project}/${Version}/${Project}"
 
 Cmd="cd ${ProdDir}"
-Cmd="${Cmd} && rm -rf ${Project}"
-Cmd="${Cmd} && ln -snf ${Location} ${Project}"
+Cmd+=" && rm -rf ${Project}"
+Cmd+=" && ln -snf ${Location} ${Project}"
 
 echo ${Cmd} && eval ${Cmd}
 
-echo "Done: $0 $*"
+echo "Done: $0 ${*}"