60 lines
1.4 KiB
Bash
Executable File
60 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
date=${1}
|
|
if [ "${date}" == "" ] ; then
|
|
date="yesterday"
|
|
fi
|
|
|
|
SourceHost=cloud21.cvtt.vpn
|
|
SourceUser=cvtt
|
|
SourceRootDir=/opt/store/cvtt/md_archive/crypto/cloud21
|
|
#SourceRootDir=/opt/store/cvtt/md_archive/crypto/cvttdata
|
|
|
|
SourceFile=$(date -d ${date} "+%Y%m%d.mktdata.db")
|
|
SourceFileZip="${SourceFile}.gz"
|
|
SourceFilePath=$(date -d ${date} "+${SourceRootDir}/%Y/%m/${SourceFileZip}")
|
|
|
|
TargetDir="/opt/jupyter_gpu/data/crypto_md"
|
|
TargetFile=$(date -d ${date} "+%Y%m%d.mktdata.ohlcv.db")
|
|
TargetFilePath="${TargetDir}/${TargetFile}"
|
|
|
|
echo ${SourceFile}
|
|
tmp_dir=$(mktemp -d)
|
|
|
|
function cleanup {
|
|
cd ${HOME}
|
|
rm -rf ${tmp_dir}
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
function download_file {
|
|
|
|
Cmd="rsync"
|
|
Cmd="${Cmd} -ahv"
|
|
if tty -s; then
|
|
Cmd="${Cmd} --progress=info2"
|
|
fi
|
|
Cmd="${Cmd} ${SourceUser}@${SourceHost}:${SourceFilePath} ${tmp_dir}/"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
ls -lh ${tmp_dir}
|
|
Cmd="gunzip ${tmp_dir}/${SourceFileZip}"
|
|
echo ${Cmd} && eval ${Cmd}
|
|
ls -lh ${tmp_dir}
|
|
|
|
rm -f ${TargetFilePath}
|
|
touch ${TargetFilePath}
|
|
|
|
for table in bnbfut_ohlcv_1min bnbspot_ohlcv_1min coinbase_ohlcv_1min
|
|
do
|
|
Cmd="sqlite3 ${tmp_dir}/${SourceFile} \".dump ${table}\" | sqlite3 ${TargetFilePath}"
|
|
echo ${Cmd}
|
|
eval ${Cmd}
|
|
done
|
|
chmod 600 ${TargetFilePath}
|
|
ls -lh ${TargetFilePath}
|
|
|
|
|
|
}
|
|
download_file
|