progress

cronjobs/cvtt/cloudstore.cron

@@ -1,6 +1,7 @@
-20 5 * * SAT /home/cvtt/prod/run/load_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(date +\%Y\%m\%d_\%H\%M\%S).cron.eqt_hist_full_list.log
+20 5 * * SAT /home/cvtt/prod/run/load_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(date +\
+20 6 * * SAT /home/cvtt/prod/run/prune_eqt_histdata.sh 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  >> /home/cvtt/prod/logs/$(da
 #
-0 6 * * SUN /home/cvtt/prod/run/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/prod/logs/$(date +\%Y\%m\%d).cronjob_log_archive.log
+0 6 * * SUN /home/cvtt/prod/run/archive_logs.sh /home/cvtt/prod/logs 2>&1 | /usr/bin/ts '[\%Y-\%m-\%d \%H:\%M:\%S]'  > /home/cvtt/p
 #
 # T E S T
 #----------------------------------------------------------------------

research/load_histdata.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
 
+function load_zipfile {
+    key=${1}
+    letter_pair=${link_letter_pairs[$key]}
+    out_file="${out_dir}/${letter_pair}_tickers.zip"
+    url=${url_root}/${key}
+    log_file="${log_dir}/${today}.tickers_download.log"
+
+    cmd="wget --no-verbose -O ${out_file} ${url} |tee -a ${log_file} 2>&1"
+    echo ${cmd} | tee -a  ${log_file}
+    eval ${cmd}
+
+}
+
+# ---------------------------------------------------
 root_dir=/home/cvtt/prod/eqt_hist_md
 
 log_dir=/home/cvtt/prod/logs
@@ -24,19 +38,6 @@ link_letter_pairs=(
         # [14033]=week_update
 )
 
-function load_zipfile {
-    key=${1}
-    letter_pair=${link_letter_pairs[$key]}
-    out_file="${out_dir}/${letter_pair}_tickers.zip"
-    url=${url_root}/${key}
-    log_file="${log_dir}/${today}.tickers_download.log"
-
-    cmd="wget --no-verbose -O ${out_file} ${url} |tee -a ${log_file} 2>&1"
-    echo ${cmd} | tee -a  ${log_file}
-    eval ${cmd}
-
-}
-
 today=$(date '+%Y%m%d')
 out_dir=${root_dir}/${today}
 

research/prune_eqt_histdata.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# ---------------------------------------------------
+root_dir=/home/cvtt/prod/eqt_hist_md
+
+prune_criteria="6 month ago"
+
+oldest=$(date -d "${prune_criteria}" '+%Y-%m-%d')
+
+
+find_cmd="find ${root_dir}/ '(' -name '*.zip' ')'  -type f -not -newermt ${oldest}"
+echo ${find_cmd}
+files=$(eval $find_cmd)
+
+
+if [ "$files" == "" ]
+then
+    echo "No .zip files found older than ${prune_criteria} (${oldest}) in ${root_dir}"
+else
+    duf ${root_dir}
+    echo "Removing files older than ${prune_criteria} (${oldest})"
+    echo $files
+    rm $files
+    duf ${root_dir}
+fi
+
+echo "${0} Done...."