Compare commits
12 Commits
034cbc7679
...
v0.0.5
| Author | SHA1 | Date | |
|---|---|---|---|
| ffd0ba5469 | |||
| 129221b086 | |||
| 8032688acb | |||
| dbeef87d20 | |||
| cad2327608 | |||
| f3dcfef3e0 | |||
| 8aebd52571 | |||
| ae2b5069d7 | |||
| d5017b1b14 | |||
| 81503d2e2c | |||
| 991c7a52b0 | |||
| acdcaecd8d |
@@ -1,14 +1,14 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
function usage {
|
function usage {
|
||||||
echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <version>"
|
echo "Usage: ${0} <app_path (e.g. market_data/md_recorder_monitor)> <project>"
|
||||||
exit
|
exit
|
||||||
}
|
}
|
||||||
|
|
||||||
AppPath=${1}
|
AppPath=${1}
|
||||||
CvttpyVersion=${2}
|
Project=${2}
|
||||||
|
|
||||||
if [ "${AppPath}" == "" ]
|
if [ "${Project}" == "" ]
|
||||||
then
|
then
|
||||||
usage
|
usage
|
||||||
fi
|
fi
|
||||||
@@ -24,7 +24,9 @@ ImageName=${AppName}
|
|||||||
ImageDir=${HOME}/docker_images
|
ImageDir=${HOME}/docker_images
|
||||||
mkdir -p ${ImageDir}
|
mkdir -p ${ImageDir}
|
||||||
|
|
||||||
Version=$(cat ${RootDir}/cvttpy/release_version.txt | awk -F, '{print $1}')
|
|
||||||
|
Version=$(cat ${RootDir}/${Project}/release_version.txt | awk -F, '{print $1}')
|
||||||
|
# Version=${ProjectVersion}
|
||||||
|
|
||||||
cd ${RootDir}
|
cd ${RootDir}
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -3,7 +3,7 @@
|
|||||||
# --- Settings
|
# --- Settings
|
||||||
export PYTHONPATH=/
|
export PYTHONPATH=/
|
||||||
export Python=python3.12
|
export Python=python3.12
|
||||||
export Config=http://cloud16.cvtt.vpn:6789/apps/minimal_md
|
export Config=http://cloud23.cvtt.vpn:6789/apps/minimal_md
|
||||||
export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
|
export PyScript=/cvttpy/exchanges/alpaca/hist_md/hist_md_bars.py
|
||||||
|
|
||||||
export OutputDir=/app/data/alpaca_md # Local
|
export OutputDir=/app/data/alpaca_md # Local
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"__dummy__": null
|
||||||
|
, "ALPACA_SANDBOX": {
|
||||||
|
"api_key": "PKLZSLFZMFMN1R28K9HK"
|
||||||
|
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
FROM linuxcontainers/debian-slim:latest
|
||||||
|
|
||||||
|
ARG FROM_DIR=./docker_dev/market_data/alpaca_hbar
|
||||||
|
|
||||||
|
# Update the package list and install required packages
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
--no-install-recommends \
|
||||||
|
apt-utils \
|
||||||
|
rsync \
|
||||||
|
openssh-client \
|
||||||
|
software-properties-common \
|
||||||
|
curl \
|
||||||
|
jq
|
||||||
|
|
||||||
|
# must be from disted version
|
||||||
|
COPY cvtt-rust/bin/alpaca_md_hbar_loader /
|
||||||
|
|
||||||
|
COPY ${FROM_DIR}/.creds /.creds
|
||||||
|
COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh
|
||||||
|
|
||||||
|
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
|
||||||
|
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
|
||||||
|
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
|
||||||
|
|
||||||
|
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
|
||||||
|
|
||||||
|
|
||||||
|
# Shared Volumes
|
||||||
|
RUN mkdir -p /data
|
||||||
|
RUN mkdir /logs
|
||||||
|
RUN chmod +x /alpaca_hbar.sh
|
||||||
|
|
||||||
|
WORKDIR /
|
||||||
|
SHELL ["/bin/bash", "-c"]
|
||||||
|
ENTRYPOINT [ "/alpaca_hbar.sh" ]
|
||||||
Executable
+110
@@ -0,0 +1,110 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: $0 [-d <YYYYMMDD Date>] [-N <num_symbols>]"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
is_business_day() {
|
||||||
|
dt=${1}
|
||||||
|
|
||||||
|
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||||
|
if [ -n "${open_time}" ]; then
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
}
|
||||||
|
export -f is_business_day
|
||||||
|
|
||||||
|
# --- Settings
|
||||||
|
export Bin=/alpaca_md_hbar_loader
|
||||||
|
export OutputDir=/data/alpaca_md # Local
|
||||||
|
export LogDir=/logs
|
||||||
|
# --- Settings
|
||||||
|
|
||||||
|
while getopts ":d:N:" opt; do
|
||||||
|
case ${opt} in
|
||||||
|
d )
|
||||||
|
Start=$OPTARG
|
||||||
|
;;
|
||||||
|
N )
|
||||||
|
NumSymbols=$OPTARG
|
||||||
|
;;
|
||||||
|
\? )
|
||||||
|
echo "Invalid option: -$OPTARG" >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
: )
|
||||||
|
echo "Option -$OPTARG requires an argument." >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# if [ -z ${Symbols} ] ; then
|
||||||
|
# echo "symbols are not specified"
|
||||||
|
# usage
|
||||||
|
# fi
|
||||||
|
|
||||||
|
mkdir -p ${OutputDir}
|
||||||
|
mkdir -p ${LogDir}
|
||||||
|
|
||||||
|
|
||||||
|
echo "CommandLine: ${*}"
|
||||||
|
|
||||||
|
export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
||||||
|
|
||||||
|
|
||||||
|
if [ -z ${Start} ] ; then
|
||||||
|
echo "start is not specified, getting yesterday..."
|
||||||
|
Start=$(date -d "yesterday" "+%Y-%m-%d")
|
||||||
|
echo "Start=${Start}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
if is_business_day ${Start}; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "${Start} is not business day in US"
|
||||||
|
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
|
echo "Start=${Start}"
|
||||||
|
Dstamp=$(date -d "${Start}" "+%Y%m%d")
|
||||||
|
|
||||||
|
Cmd="${Bin}"
|
||||||
|
Cmd="${Cmd} --credentials_file=/.creds"
|
||||||
|
Cmd="${Cmd} --output_directory=${OutputDir}"
|
||||||
|
Cmd="${Cmd} --start=${Start}"
|
||||||
|
if [ -n "${NumSymbols}" ]; then
|
||||||
|
Cmd="${Cmd} --num_symbols=${NumSymbols}"
|
||||||
|
fi
|
||||||
|
Cmd="${Cmd} > ${LogDir}/${Dstamp}.alpaca_hbar.log 2>&1"
|
||||||
|
|
||||||
|
echo ${Cmd}
|
||||||
|
eval ${Cmd}
|
||||||
|
|
||||||
|
echo "Compressing"
|
||||||
|
for file in $(find ${OutputDir} -type f -name '*db' -print )
|
||||||
|
do
|
||||||
|
echo "Compressing ${file}"
|
||||||
|
gzip ${file}
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
|
Source=/data/
|
||||||
|
Targets=
|
||||||
|
Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
|
||||||
|
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
|
||||||
|
|
||||||
|
|
||||||
|
for tgt in ${Targets}
|
||||||
|
do
|
||||||
|
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
|
||||||
|
echo $Cmd
|
||||||
|
eval $Cmd
|
||||||
|
done
|
||||||
|
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"__dummy__": null
|
||||||
|
, "ALPACA_SANDBOX": {
|
||||||
|
"api_key": "PKLZSLFZMFMN1R28K9HK"
|
||||||
|
, "secret_key": "SKbxwLWJNs4kpn618DgGaopN6x1xzKwLM4Z7aymA"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
FROM linuxcontainers/debian-slim:latest
|
||||||
|
|
||||||
|
ARG FROM_DIR=docker_dev/market_data/alpaca_qat
|
||||||
|
|
||||||
|
# Update the package list and install required packages
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
--no-install-recommends \
|
||||||
|
apt-utils \
|
||||||
|
rsync \
|
||||||
|
openssh-client \
|
||||||
|
software-properties-common \
|
||||||
|
curl \
|
||||||
|
jq
|
||||||
|
|
||||||
|
|
||||||
|
# must be from disted version
|
||||||
|
COPY cvtt-rust/bin/alpaca_md_qat_loader /
|
||||||
|
|
||||||
|
COPY ${FROM_DIR}/.creds /.creds
|
||||||
|
COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh
|
||||||
|
|
||||||
|
COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa
|
||||||
|
COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub
|
||||||
|
COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts
|
||||||
|
|
||||||
|
RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts
|
||||||
|
|
||||||
|
|
||||||
|
# Shared Volumes
|
||||||
|
RUN mkdir -p /data
|
||||||
|
RUN mkdir /logs
|
||||||
|
RUN chmod +x /alpaca_qat.sh
|
||||||
|
|
||||||
|
WORKDIR /
|
||||||
|
SHELL ["/bin/bash", "-c"]
|
||||||
|
ENTRYPOINT [ "/alpaca_qat.sh" ]
|
||||||
|
|
||||||
|
# CMD [ "echo", "alpaca_md_day"]
|
||||||
Executable
+108
@@ -0,0 +1,108 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: $0 [-d <YYYYMMDD Date>] -S <symbols>"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
is_business_day() {
|
||||||
|
dt=${1}
|
||||||
|
|
||||||
|
open_time=$(curl -s "${CalendarURL}&start=${dt}&end=${dt}" | jq '.[] | .open_time')
|
||||||
|
if [ -n "${open_time}" ]; then
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
}
|
||||||
|
export -f is_business_day
|
||||||
|
|
||||||
|
# --- Settings
|
||||||
|
export Bin=/alpaca_md_qat_loader
|
||||||
|
export OutputDir=/data/alpaca_md # Local
|
||||||
|
export LogDir=/logs
|
||||||
|
# --- Settings
|
||||||
|
|
||||||
|
while getopts ":d:S:" opt; do
|
||||||
|
case ${opt} in
|
||||||
|
d )
|
||||||
|
Start=$OPTARG
|
||||||
|
;;
|
||||||
|
S )
|
||||||
|
Symbols=$OPTARG
|
||||||
|
;;
|
||||||
|
\? )
|
||||||
|
echo "Invalid option: -$OPTARG" >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
: )
|
||||||
|
echo "Option -$OPTARG requires an argument." >&2
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z ${Symbols} ] ; then
|
||||||
|
echo "symbols are not specified"
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p ${OutputDir}
|
||||||
|
mkdir -p ${LogDir}
|
||||||
|
|
||||||
|
|
||||||
|
echo "CommandLine: ${*}"
|
||||||
|
|
||||||
|
export CalendarURL=http://cloud23.cvtt.vpn:8000/api/v1/markets/hours?mic=XNYS
|
||||||
|
|
||||||
|
|
||||||
|
if [ -z ${Start} ] ; then
|
||||||
|
echo "start is not specified, getting yesterday..."
|
||||||
|
Start=$(date -d "yesterday" "+%Y-%m-%d")
|
||||||
|
echo "Start=${Start}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
if is_business_day ${Start}; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
echo "${Start} is not business day in US"
|
||||||
|
Start=$(date -d "${Start} - 1 day" "+%Y-%m-%d")
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
|
echo "Start=${Start}"
|
||||||
|
Dstamp=$(date -d "${Start}" "+%Y%m%d")
|
||||||
|
|
||||||
|
Cmd="${Bin}"
|
||||||
|
Cmd="${Cmd} --credentials_file=/.creds"
|
||||||
|
Cmd="${Cmd} --output_directory=${OutputDir}"
|
||||||
|
Cmd="${Cmd} --start=${Start}"
|
||||||
|
Cmd="${Cmd} --symbols=${Symbols}"
|
||||||
|
Cmd="${Cmd} > ${LogDir}/${Dstamp}.alpaca_qat.log 2>&1"
|
||||||
|
|
||||||
|
echo ${Cmd}
|
||||||
|
eval ${Cmd}
|
||||||
|
|
||||||
|
echo "Compressing"
|
||||||
|
for file in $(find ${OutputDir} -type f -name '*db' -print )
|
||||||
|
do
|
||||||
|
echo "Compressing ${file}"
|
||||||
|
gzip ${file}
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
|
Source=/data/
|
||||||
|
Targets=
|
||||||
|
Targets="${Targets} cvtt@homestore.cvtt.vpn:/works/cvtt/md_archive/equity"
|
||||||
|
Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity"
|
||||||
|
|
||||||
|
|
||||||
|
for tgt in ${Targets}
|
||||||
|
do
|
||||||
|
Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}"
|
||||||
|
echo $Cmd
|
||||||
|
eval $Cmd
|
||||||
|
done
|
||||||
|
|
||||||
@@ -52,7 +52,7 @@ export PYTHONPATH=/
|
|||||||
|
|
||||||
Cmd="python3.12"
|
Cmd="python3.12"
|
||||||
Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
|
Cmd="${Cmd} cvttpy/research/utils/archive_ts_md.py"
|
||||||
Cmd="${Cmd} --config=http://cloud16.cvtt.vpn:6789/apps/md_recorder"
|
Cmd="${Cmd} --config=http://cloud23.cvtt.vpn:6789/apps/md_recorder"
|
||||||
Cmd="${Cmd} --db_credentials_key=${CredKey}"
|
Cmd="${Cmd} --db_credentials_key=${CredKey}"
|
||||||
Cmd="${Cmd} --credentials_file=/.creds"
|
Cmd="${Cmd} --credentials_file=/.creds"
|
||||||
Cmd="${Cmd} --date=${Date}"
|
Cmd="${Cmd} --date=${Date}"
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ if ! is_valid "${job}"; then
|
|||||||
usage
|
usage
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ConfigServer=cloud16.cvtt.vpn
|
|
||||||
ConfigServer=cloud23.cvtt.vpn:6789
|
ConfigServer=cloud23.cvtt.vpn:6789
|
||||||
|
|
||||||
Cmd="python3.12"
|
Cmd="python3.12"
|
||||||
@@ -34,42 +33,32 @@ Cmd="${Cmd} cvttpy/apps/md/md_recorder.py"
|
|||||||
Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
|
Cmd="${Cmd} --config=http://${ConfigServer}/apps/md_recorder"
|
||||||
Cmd="${Cmd} --credentials_file=/.creds"
|
Cmd="${Cmd} --credentials_file=/.creds"
|
||||||
Cmd="${Cmd} --compress_log"
|
Cmd="${Cmd} --compress_log"
|
||||||
|
Cmd="${Cmd} --instrument_group=${job}"
|
||||||
|
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.${job}.log"
|
||||||
if [ "${job}" == "BNBFUT_CLD21" ] ; then
|
if [ "${job}" == "BNBFUT_CLD21" ] ; then
|
||||||
Cmd="${Cmd} --active_exchanges=BNBFUT"
|
Cmd="${Cmd} --active_exchanges=BNBFUT"
|
||||||
Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
|
|
||||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
||||||
Cmd="${Cmd} --admin_port=7201"
|
Cmd="${Cmd} --admin_port=7201"
|
||||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CLD21.log"
|
|
||||||
elif [ "${job}" == "COINBASE_CLD21" ] ; then
|
elif [ "${job}" == "COINBASE_CLD21" ] ; then
|
||||||
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
|
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
|
||||||
Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
|
|
||||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
||||||
Cmd="${Cmd} --admin_port=7202"
|
Cmd="${Cmd} --admin_port=7202"
|
||||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CLD21.log"
|
|
||||||
elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
|
elif [ "${job}" == "BNBSPOT_CLD21" ] ; then
|
||||||
Cmd="${Cmd} --active_exchanges=BNBSPOT"
|
Cmd="${Cmd} --active_exchanges=BNBSPOT"
|
||||||
Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
|
|
||||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CLD21"
|
||||||
Cmd="${Cmd} --admin_port=7203"
|
Cmd="${Cmd} --admin_port=7203"
|
||||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CLD21.log"
|
|
||||||
elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
|
elif [ "${job}" == "BNBSPOT_CVTTDATA" ] ; then
|
||||||
Cmd="${Cmd} --active_exchanges=BNBSPOT"
|
Cmd="${Cmd} --active_exchanges=BNBSPOT"
|
||||||
Cmd="${Cmd} --instruments=BNBSPOT:PAIR-BTC-USDT,BNBSPOT:PAIR-ETH-USDT,BNBSPOT:PAIR-LTC-USDT,BNBSPOT:PAIR-XRP-USDT,BNBSPOT:PAIR-USDC-USDT"
|
|
||||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
||||||
Cmd="${Cmd} --admin_port=7204"
|
Cmd="${Cmd} --admin_port=7204"
|
||||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBSPOT.CVTTDATA.log"
|
|
||||||
elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
|
elif [ "${job}" == "BNBFUT_CVTTDATA" ] ; then
|
||||||
Cmd="${Cmd} --active_exchanges=BNBFUT"
|
Cmd="${Cmd} --active_exchanges=BNBFUT"
|
||||||
Cmd="${Cmd} --instruments=BNBFUT:PERP-BTC-USDT,BNBFUT:PERP-ETH-USDT"
|
|
||||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
||||||
Cmd="${Cmd} --admin_port=7205"
|
Cmd="${Cmd} --admin_port=7205"
|
||||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.BNBFUT.CVTTDATA.log"
|
|
||||||
elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
|
elif [ "${job}" == "COINBASE_CVTTDATA" ] ; then
|
||||||
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
|
Cmd="${Cmd} --active_exchanges=COINBASE_AT"
|
||||||
Cmd="${Cmd} --instruments=COINBASE:PAIR-BTC-USD,COINBASE:PAIR-ETH-USD,COINBASE:PAIR-LTC-USD,COINBASE:PAIR-XRP-USD"
|
|
||||||
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
Cmd="${Cmd} --db_credentials_key=TSDB_MD_CVTTDATA"
|
||||||
Cmd="${Cmd} --admin_port=7206"
|
Cmd="${Cmd} --admin_port=7206"
|
||||||
Cmd="${Cmd} --log_file=/logs/%T.MD_REC.COINBASE.CVTTDATA.log"
|
|
||||||
else
|
else
|
||||||
echo "Unrecognized JOB: ${job}"
|
echo "Unrecognized JOB: ${job}"
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -32,7 +32,6 @@ if [ "${2}" != "" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
# ConfigServer=cloud16.cvtt.vpn
|
|
||||||
ConfigServer=cloud23.cvtt.vpn:6789
|
ConfigServer=cloud23.cvtt.vpn:6789
|
||||||
|
|
||||||
Cmd="python3.12"
|
Cmd="python3.12"
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ CMD [ \
|
|||||||
"cvttpy/apps/md/rel_lqdt_server.py", \
|
"cvttpy/apps/md/rel_lqdt_server.py", \
|
||||||
"--port=5678", \
|
"--port=5678", \
|
||||||
"--db_file=/app/data/rel_liquidity.db", \
|
"--db_file=/app/data/rel_liquidity.db", \
|
||||||
"--log_file=/logs/%%T.REL_LIQUIDITY_SVC.log" \
|
"--log_file=/logs/%T.REL_LIQUIDITY_SVC.log" \
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
0.0.5,16->23
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
.git
|
||||||
|
**/__pycache__
|
||||||
|
.pipenv
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
WORKDIR /
|
||||||
|
COPY docker_dev/research/crypto_exch_stats/requirements.txt /
|
||||||
|
|
||||||
|
RUN pip install --upgrade pip --root-user-action=ignore
|
||||||
|
RUN pip install -r /requirements.txt --root-user-action=ignore
|
||||||
|
|
||||||
|
COPY cvttpy /cvttpy
|
||||||
|
|
||||||
|
# Shared Volumes
|
||||||
|
RUN mkdir -p /app/data
|
||||||
|
RUN mkdir /logs
|
||||||
|
|
||||||
|
ENV PYTHONPATH=/
|
||||||
|
CMD [ \
|
||||||
|
"python3.12", \
|
||||||
|
"cvttpy/apps//research/exchange_trading_stats.py", \
|
||||||
|
"--config=http://cloud23.cvtt.vpn:6789/apps/tests/listen_market_data", \
|
||||||
|
"--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,BNBSPOT,KRAKEN", \
|
||||||
|
"--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,BNBSPOT:PAIR-BTC-USDT,KRAKEN:PAIR-BTC-USD", \
|
||||||
|
"--db_file=/app/data/exchange_trading_stats.db", \
|
||||||
|
"--log_level=INFO", \
|
||||||
|
"--log_file=/logs/%T.EXCHANGE_TRADING_STATS.log" \
|
||||||
|
]
|
||||||
|
|
||||||
|
# ---------- No BNBSPOT from USA
|
||||||
|
# "--active_exchanges=OKX,GEMINI,BITSTAMP,COINBASE_AT,KRAKEN", \
|
||||||
|
# "--instruments=OKX:PAIR-BTC-USDT,GEMINI:PAIR-BTC-USD,BITSTAMP:PAIR-BTC-USD,COINBASE:PAIR-BTC-USD,KRAKEN:PAIR-BTC-USD", \
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
aiohttp>=3.7.4.post0
|
||||||
|
nest-asyncio>=1.5.5
|
||||||
|
numpy>=1.24.1
|
||||||
|
hjson>=3.1.0
|
||||||
|
pandas>=1.5.3
|
||||||
|
sortedcontainers>=2.4.0
|
||||||
|
redis>=5.0.8
|
||||||
Executable
+16
@@ -0,0 +1,16 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
Cmd="docker run"
|
||||||
|
Cmd+=" -d"
|
||||||
|
# Cmd+=" --rm"
|
||||||
|
Cmd+=" --pull=always"
|
||||||
|
Cmd+=" --network=host"
|
||||||
|
Cmd+=" --name=crypto_exch_stats"
|
||||||
|
Cmd+=" --volume=${HOME}/prod/data:/app/data"
|
||||||
|
Cmd+=" --volume=${HOME}/prod/logs:/logs"
|
||||||
|
Cmd+=" cloud21.cvtt.vpn:5500/crypto_exch_stats:latest"
|
||||||
|
|
||||||
|
echo ${Cmd}
|
||||||
|
eval ${Cmd}
|
||||||
|
# Cmd+=" cloud21.cvtt.vpn:5500/relative_liquidity:latest"
|
||||||
|
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
FROM codercom/code-server:latest
|
||||||
|
|
||||||
|
ARG FROM_DIR=./
|
||||||
|
USER root
|
||||||
|
# Update the package list and install required packages
|
||||||
|
RUN sudo apt-get update && apt-get install -y \
|
||||||
|
apt-utils \
|
||||||
|
rsync \
|
||||||
|
openssh-client \
|
||||||
|
software-properties-common \
|
||||||
|
build-essential \
|
||||||
|
zlib1g-dev \
|
||||||
|
libncurses5-dev \
|
||||||
|
libgdbm-dev \
|
||||||
|
libnss3-dev \
|
||||||
|
libssl-dev \
|
||||||
|
libreadline-dev \
|
||||||
|
libffi-dev \
|
||||||
|
libsqlite3-dev \
|
||||||
|
libbz2-dev \
|
||||||
|
curl \
|
||||||
|
wget \
|
||||||
|
jq
|
||||||
|
|
||||||
|
# ----------------------------------------
|
||||||
|
RUN mkdir -p /tmp
|
||||||
|
WORKDIR /tmp
|
||||||
|
RUN wget https://www.python.org/ftp/python/3.12.5/Python-3.12.5.tgz
|
||||||
|
|
||||||
|
RUN tar -xf Python-3.12.5.tgz
|
||||||
|
WORKDIR /tmp/Python-3.12.5
|
||||||
|
|
||||||
|
RUN ./configure --enable-optimizations --prefix=/usr
|
||||||
|
RUN make -j$(nproc)
|
||||||
|
RUN make altinstall
|
||||||
|
|
||||||
|
# ----------------------------------------
|
||||||
|
RUN rm /usr/bin/python3
|
||||||
|
RUN ln -s /usr/bin/python3.12 /usr/bin/python3
|
||||||
|
|
||||||
|
RUN ln -snf /usr/bin/pip3.12 /usr/bin/pip
|
||||||
|
|
||||||
|
COPY ./requirements.txt /
|
||||||
|
RUN pip install --upgrade pip --root-user-action=ignore
|
||||||
|
RUN pip install -r /requirements.txt --root-user-action=ignore
|
||||||
|
|
||||||
|
SHELL ["/bin/bash", "-c"]
|
||||||
|
|
||||||
|
WORKDIR /home/coder
|
||||||
Executable
+2
@@ -0,0 +1,2 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
docker build -t code-server-plus -t code-server-plus:latest -f Dockerfile .
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
version: "3.8"
|
||||||
|
services:
|
||||||
|
code-server:
|
||||||
|
container_name: code-server-plus
|
||||||
|
image: code-server-plus:latest
|
||||||
|
restart: unless-stopped
|
||||||
|
user: 1001:1001
|
||||||
|
volumes:
|
||||||
|
- /opt/code-server/.local:/home/coder/.local
|
||||||
|
- /opt/code-server/.config:/home/coder/.config
|
||||||
|
- /opt/code-server/projects:/home/coder/project
|
||||||
|
environment:
|
||||||
|
- DOCKER_USER=cvtt
|
||||||
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
aiohttp>=3.7.4.post0
|
||||||
|
appdirs>=1.4.4
|
||||||
|
asn1crypto>=0.24.0
|
||||||
|
astroid>=2.8.0
|
||||||
|
async-timeout>=3.0.1
|
||||||
|
attrs>=22.1.0
|
||||||
|
Automat>=0.8.0
|
||||||
|
backports.entry-points-selectable>=1.1.0
|
||||||
|
bcrypt>=4.0.0
|
||||||
|
black>=22.6.0
|
||||||
|
blinker>=1.4
|
||||||
|
boto3>=1.24.43
|
||||||
|
botocore>=1.27.43
|
||||||
|
cachetools>=4.1.1
|
||||||
|
certifi>=2019.11.28
|
||||||
|
cffi>=1.15.0
|
||||||
|
chardet>=4.0.0
|
||||||
|
charset-normalizer>=2.0.7
|
||||||
|
ciso8601>=2.2.0
|
||||||
|
click>=8.1.3
|
||||||
|
coloredlogs>=15.0.1
|
||||||
|
configobj>=5.0.6
|
||||||
|
constantly>=15.1.0
|
||||||
|
cryptography>=2.8
|
||||||
|
dataclasses-json>=0.5.1
|
||||||
|
distlib>=0.3.3
|
||||||
|
distro>=1.4.0
|
||||||
|
docker>=5.0.3
|
||||||
|
entrypoints>=0.3
|
||||||
|
eventkit>=0.8.9
|
||||||
|
ExportCsvToInflux>=0.2.2
|
||||||
|
filelock>=3.3.2
|
||||||
|
google-api-core>=1.22.2
|
||||||
|
google-api-python-client>=2.22.0
|
||||||
|
google-auth>=1.21.1
|
||||||
|
google-auth-httplib2>=0.1.0
|
||||||
|
google-auth-oauthlib>=0.4.6
|
||||||
|
googleapis-common-protos>=1.52.0
|
||||||
|
greenlet>=1.1.2
|
||||||
|
hiredis>=1.1.0
|
||||||
|
hjson>=3.0.2
|
||||||
|
httplib2>=0.18.1
|
||||||
|
humanfriendly>=10.0
|
||||||
|
hyperlink>=19.0.0
|
||||||
|
ib-insync>=0.9.70
|
||||||
|
ibapi
|
||||||
|
idna>=2.8
|
||||||
|
importlab>=0.7
|
||||||
|
importlib-metadata>=1.5.0
|
||||||
|
incremental>=16.10.1
|
||||||
|
influxdb>=5.3.1
|
||||||
|
influxdb-client>=1.29.1
|
||||||
|
isort>=5.5.1
|
||||||
|
jedi>=0.18.0
|
||||||
|
Jinja2>=2.10.1
|
||||||
|
jmespath>=1.0.1
|
||||||
|
json-cfg>=0.4.2
|
||||||
|
jsonpatch>=1.22
|
||||||
|
jsonpointer>=2.0
|
||||||
|
jsonschema>=3.2.0
|
||||||
|
keyring>=18.0.1
|
||||||
|
kwonly-args>=1.0.10
|
||||||
|
launchpadlib>=1.10.13
|
||||||
|
lazr.restfulclient>=0.14.2
|
||||||
|
lazr.uri>=1.0.3
|
||||||
|
lazy-object-proxy>=1.4.3
|
||||||
|
libcst>=0.4.7
|
||||||
|
lxml>=4.6.3
|
||||||
|
lxml-stubs>=0.2.0
|
||||||
|
MarkupSafe>=1.1.0
|
||||||
|
marshmallow>=3.7.0
|
||||||
|
marshmallow-enum>=1.5.1
|
||||||
|
mccabe>=0.6.1
|
||||||
|
more-itertools>=4.2.0
|
||||||
|
msgpack>=0.6.1
|
||||||
|
multidict>=4.7.6
|
||||||
|
multitasking>=0.0.10
|
||||||
|
mypy>=0.961
|
||||||
|
mypy-extensions>=0.4.3
|
||||||
|
nest-asyncio>=1.5.5
|
||||||
|
networkx>=2.8.3
|
||||||
|
ninja>=1.10.2.3
|
||||||
|
numpy #>=1.19.1
|
||||||
|
oauthlib>=3.1.0
|
||||||
|
pandas>=1.3.3
|
||||||
|
pandas-stubs>=1.2.0.17
|
||||||
|
paramiko>=2.11.0
|
||||||
|
parso>=0.8.1
|
||||||
|
pathspec>=0.9.0
|
||||||
|
pbr>=5.6.0
|
||||||
|
pexpect>=4.6.0
|
||||||
|
platformdirs>=2.3.0
|
||||||
|
protobuf>=3.13.0
|
||||||
|
ptyprocess>=0.7.0
|
||||||
|
pyasn1>=0.4.2
|
||||||
|
pyasn1-modules>=0.2.1
|
||||||
|
pycparser>=2.20
|
||||||
|
pydot>=1.4.2
|
||||||
|
PyHamcrest>=1.9.0
|
||||||
|
PyJWT>=1.7.1
|
||||||
|
pylint>=2.11.1
|
||||||
|
pymongo>=4.0.1
|
||||||
|
PyNaCl>=1.5.0
|
||||||
|
pynvim>=0.4.1
|
||||||
|
pyOpenSSL>=19.0.0
|
||||||
|
pyparsing>=3.0.9
|
||||||
|
pyrsistent>=0.15.5
|
||||||
|
pyserial>=3.4
|
||||||
|
python-dateutil>=2.8.2
|
||||||
|
python-rapidjson>=0.9.1
|
||||||
|
pytz>=2020.1
|
||||||
|
PyYAML>=5.3.1
|
||||||
|
redis>=5.0.8
|
||||||
|
regex>=2021.11.10
|
||||||
|
requests>=2.27.1
|
||||||
|
requests-oauthlib>=1.3.0
|
||||||
|
requests-unixsocket>=0.2.0
|
||||||
|
rsa>=4.6
|
||||||
|
Rx>=3.2.0
|
||||||
|
s3transfer>=0.6.0
|
||||||
|
SecretStorage>=2.3.1
|
||||||
|
service-identity>=18.1.0
|
||||||
|
simplefix>=1.0.14
|
||||||
|
simplejson>=3.16.0
|
||||||
|
six>=1.14.0
|
||||||
|
slackclient>=2.9.3
|
||||||
|
sortedcontainers>=2.4.0
|
||||||
|
SQLAlchemy>=1.4.39
|
||||||
|
ssh-import-id>=5.10
|
||||||
|
stringcase>=1.2.0
|
||||||
|
tabulate>=0.8.10
|
||||||
|
termcolor>=1.1.0
|
||||||
|
testresources>=2.0.1
|
||||||
|
toml>=0.10.2
|
||||||
|
tomli>=2.0.1
|
||||||
|
typed-ast>=1.5.1
|
||||||
|
types-chardet>=4.0.1
|
||||||
|
types-cryptography>=3.3.23
|
||||||
|
types-paramiko>=2.11.6
|
||||||
|
types-PyMySQL>=1.0.18
|
||||||
|
types-python-dateutil>=2.8.0
|
||||||
|
types-pytz>=2022.1.1
|
||||||
|
types-redis>=4.2.6
|
||||||
|
types-requests>=2.25.7
|
||||||
|
types-simplejson>=3.17.0
|
||||||
|
types-termcolor>=1.1.1
|
||||||
|
typing-inspect>=0.6.0
|
||||||
|
typing_extensions>=4.3.0
|
||||||
|
uritemplate>=3.0.1
|
||||||
|
urllib3>=1.26.9
|
||||||
|
uvloop>=0.14.0
|
||||||
|
virtualenv>=20.9.0
|
||||||
|
wadllib>=1.3.3
|
||||||
|
websocket-client>=0.53.0
|
||||||
|
websockets>=8.1
|
||||||
|
wrapt>=1.12.1
|
||||||
|
yapf>=0.30.0
|
||||||
|
yarl>=1.5.1
|
||||||
|
yfinance>=0.1.70
|
||||||
|
zipp>=1.0.0
|
||||||
|
zope.interface>=5.4.0
|
||||||
|
|
||||||
@@ -14,7 +14,6 @@ usage() {
|
|||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# ConfigServer=cloud16.cvtt.vpn
|
|
||||||
ActiveExchanges=ALPACA_SNDBX-MDPORTAL
|
ActiveExchanges=ALPACA_SNDBX-MDPORTAL
|
||||||
AdminPort=7225
|
AdminPort=7225
|
||||||
Book=""
|
Book=""
|
||||||
|
|||||||
Reference in New Issue
Block a user