progress

2024-08-05 22:41:52 -04:00
parent 90fa146190
commit df761027ec
4 changed files with 127 additions and 1 deletions
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+function load_zipfile {
+    key=${1}
+    letter_pair=${link_letter_pairs[$key]}
+    out_file="${out_dir}/${letter_pair}_tickers.zip"
+    url=${url_root}/${key}
+    log_file="${log_dir}/${today}.tickers_download.log"
+
+    cmd="wget --no-verbose -O ${out_file} ${url} |tee -a ${log_file} 2>&1"
+    echo ${cmd} | tee -a  ${log_file}
+    eval ${cmd}
+
+}
+
+# ---------------------------------------------------
+root_dir=/opt/store/cvtt/eqt_hist_md
+
+log_dir=/home/cvtt/prod/logs
+url_root="https://firstratedata.com/datafile/89CgWiwc60CylOIEFRDTCg"
+
+mkdir -p ${log_dir}
+
+link_letter_pairs=(
+        [13580]=AB
+        [13585]=CD
+        [13590]=EF
+        [13595]=GH
+        [13600]=IJ
+        [13605]=KL
+        [12509]=MN
+        [13610]=OP
+        [13615]=QR
+        [13620]=ST
+        [13625]=UV
+        [13630]=WX
+        [13635]=YZ
+        # [14033]=week_update
+)
+
+today=$(date '+%Y%m%d')
+out_dir=${root_dir}/${today}
+
+mkdir -p ${out_dir}
+
+keys=${*}
+
+if [ "${keys}" == "" ]
+then
+    keys="${!link_letter_pairs[@]}"
+fi
+
+echo "keys=${keys}"
+for key in ${keys}
+do
+  load_zipfile ${key}
+done
+
+echo "${0} Done...." | tee -a ${log_file}
@@ -0,0 +1,26 @@
+#!/bin/bash
+# ---------------------------------------------------
+root_dir=/opt/store/cvtt/eqt_hist_md
+
+prune_criteria="6 month ago"
+
+oldest=$(date -d "${prune_criteria}" '+%Y-%m-%d')
+
+
+find_cmd="find ${root_dir}/ '(' -name '*.zip' ')'  -type f -not -newermt ${oldest}"
+echo ${find_cmd}
+files=$(eval $find_cmd)
+
+
+if [ "$files" == "" ]
+then
+    echo "No .zip files found older than ${prune_criteria} (${oldest}) in ${root_dir}"
+else
+    duf ${root_dir}
+    echo "Removing files older than ${prune_criteria} (${oldest})"
+    echo $files
+    rm $files
+    duf ${root_dir}
+fi
+
+echo "${0} Done...."