From 97ef506a90fe19d137c2a8235c80fba134bb9bef Mon Sep 17 00:00:00 2001
From: Oleg Sheynin <osheynin@gmail.com>
Date: Mon, 20 Nov 2023 18:27:17 -0500
Subject: [PATCH] progress

---
 cronjobs/cvtt/{cloud16 => cloud16.cron}       |  0
 cronjobs/cvtt/{cloud17 => cloud17.cron}       |  0
 cronjobs/cvtt/{cloudstore => cloudstore.cron} |  5 ++--
 cronjobs/cvtt/{cryptoval3 => cryptoval3.cron} |  0
 research/load_histdata.sh                     | 27 ++++++++++---------
 research/prune_eqt_histdata.sh                | 26 ++++++++++++++++++
 6 files changed, 43 insertions(+), 15 deletions(-)
 rename cronjobs/cvtt/{cloud16 => cloud16.cron} (100%)
 rename cronjobs/cvtt/{cloud17 => cloud17.cron} (100%)
 rename cronjobs/cvtt/{cloudstore => cloudstore.cron} (74%)
 rename cronjobs/cvtt/{cryptoval3 => cryptoval3.cron} (100%)
 create mode 100755 research/prune_eqt_histdata.sh

diff --git a/cronjobs/cvtt/cloud16 b/cronjobs/cvtt/cloud16.cron
similarity index 100%
rename from cronjobs/cvtt/cloud16
rename to cronjobs/cvtt/cloud16.cron
diff --git a/cronjobs/cvtt/cloud17 b/cronjobs/cvtt/cloud17.cron
similarity index 100%
rename from cronjobs/cvtt/cloud17
rename to cronjobs/cvtt/cloud17.cron
diff --git a/cronjobs/cvtt/cloudstore b/cronjobs/cvtt/cloudstore.cron
similarity index 74%
rename from cronjobs/cvtt/cloudstore
rename to cronjobs/cvtt/cloudstore.cron
index 706fb4e..04849ad 100644
--- a/cronjobs/cvtt/cloudstore
+++ b/cronjobs/cvtt/cloudstore.cron
@@ -1,6 +1,7 @@
-20 5 * * SAT /home/cvtt/prod/run/load_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cron.eqt_hist_full_list.log
+20 5 * * SAT /home/cvtt/prod/run/load_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(date +\
+20 6 * * SAT /home/cvtt/prod/run/prune_eqt_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(da
 #
-0 6 * * SUN /home/cvtt/prod/run/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d).cronjob_log_archive.log
+0 6 * * SUN /home/cvtt/prod/run/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/p
 #
 # T E S T
 #----------------------------------------------------------------------
diff --git a/cronjobs/cvtt/cryptoval3 b/cronjobs/cvtt/cryptoval3.cron
similarity index 100%
rename from cronjobs/cvtt/cryptoval3
rename to cronjobs/cvtt/cryptoval3.cron
diff --git a/research/load_histdata.sh b/research/load_histdata.sh
index 25140d6..4b2289e 100755
--- a/research/load_histdata.sh
+++ b/research/load_histdata.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+function load_zipfile {
+    key=${1}
+    letter_pair=${link_letter_pairs[$key]}
+    out_file="${out_dir}/${letter_pair}_tickers.zip"
+    url=${url_root}/${key}
+    log_file="${log_dir}/${today}.tickers_download.log"
+
+    cmd="wget --no-verbose -O ${out_file} ${url} |tee -a ${log_file} 2>&1"
+    echo ${cmd} | tee -a  ${log_file}
+    eval ${cmd}
+
+}
+
+# ---------------------------------------------------
 root_dir=/home/cvtt/prod/eqt_hist_md
 
 log_dir=/home/cvtt/prod/logs
@@ -24,19 +38,6 @@ link_letter_pairs=(
         # [14033]=week_update
 )
 
-function load_zipfile {
-    key=${1}
-    letter_pair=${link_letter_pairs[$key]}
-    out_file="${out_dir}/${letter_pair}_tickers.zip"
-    url=${url_root}/${key}
-    log_file="${log_dir}/${today}.tickers_download.log"
-
-    cmd="wget --no-verbose -O ${out_file} ${url} |tee -a ${log_file} 2>&1"
-    echo ${cmd} | tee -a  ${log_file}
-    eval ${cmd}
-
-}
-
 today=$(date '+%Y%m%d')
 out_dir=${root_dir}/${today}
 
diff --git a/research/prune_eqt_histdata.sh b/research/prune_eqt_histdata.sh
new file mode 100755
index 0000000..92332bd
--- /dev/null
+++ b/research/prune_eqt_histdata.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# ---------------------------------------------------
+root_dir=/home/cvtt/prod/eqt_hist_md
+
+prune_criteria="6 month ago"
+
+oldest=$(date -d "${prune_criteria}" '+%Y-%m-%d')
+
+
+find_cmd="find ${root_dir}/ '(' -name '*.zip' ')'  -type f -not -newermt ${oldest}"
+echo ${find_cmd}
+files=$(eval $find_cmd)
+
+
+if [ "$files" == "" ]
+then
+    echo "No .zip files found older than ${prune_criteria} (${oldest}) in ${root_dir}"
+else
+    duf ${root_dir}
+    echo "Removing files older than ${prune_criteria} (${oldest})"
+    echo $files
+    rm $files
+    duf ${root_dir}
+fi
+
+echo "${0} Done...."
\ No newline at end of file