From 421d5310ec128b65392d11c59b7d1654aa23c98e Mon Sep 17 00:00:00 2001 From: Oleg Sheynin Date: Mon, 20 Jan 2025 19:09:54 -0500 Subject: [PATCH] removed dependency on known_hosts file --- market_data/alpaca_hbar/Dockerfile | 6 +++--- market_data/alpaca_hbar/alpaca_hbar.sh | 4 +++- market_data/alpaca_qat/Dockerfile | 5 ++--- market_data/alpaca_qat/alpaca_qat.sh | 4 +++- market_data/crypto_md_day/Dockerfile | 3 +-- market_data/crypto_md_day/crypto_md_day.sh | 5 +++-- release_version.txt | 2 +- shared/known_hosts | 6 ------ test/rsync_test/Dockerfile | 21 +++++++++++++++++++++ test/rsync_test/test.sh | 19 +++++++++++++++++++ 10 files changed, 56 insertions(+), 19 deletions(-) delete mode 100644 shared/known_hosts create mode 100644 test/rsync_test/Dockerfile create mode 100755 test/rsync_test/test.sh diff --git a/market_data/alpaca_hbar/Dockerfile b/market_data/alpaca_hbar/Dockerfile index 345510e..90b1cc7 100644 --- a/market_data/alpaca_hbar/Dockerfile +++ b/market_data/alpaca_hbar/Dockerfile @@ -13,16 +13,16 @@ RUN apt-get update && apt-get install -y \ jq # must be from disted version -COPY cvtt-rust/bin/alpaca_md_hbar_loader / +COPY cvtt-rust/dist/bin/alpaca_md_hbar_loader / # COPY ${FROM_DIR}/.creds /.creds COPY ${FROM_DIR}/alpaca_hbar.sh /alpaca_hbar.sh COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub -COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts -RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts +RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub + # Shared Volumes diff --git a/market_data/alpaca_hbar/alpaca_hbar.sh b/market_data/alpaca_hbar/alpaca_hbar.sh index 35dae41..c52daa0 100755 --- a/market_data/alpaca_hbar/alpaca_hbar.sh +++ b/market_data/alpaca_hbar/alpaca_hbar.sh @@ -99,7 +99,9 @@ Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" for tgt in ${Targets} do - Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}" + Cmd="/usr/bin/rsync -ahv" + Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" + Cmd+=" ${Source} ${tgt}" echo $Cmd eval $Cmd done diff --git a/market_data/alpaca_qat/Dockerfile b/market_data/alpaca_qat/Dockerfile index 2d7bc2e..3a6011d 100644 --- a/market_data/alpaca_qat/Dockerfile +++ b/market_data/alpaca_qat/Dockerfile @@ -14,16 +14,15 @@ RUN apt-get update && apt-get install -y \ # must be from disted version -COPY cvtt-rust/bin/alpaca_md_qat_loader / +COPY cvtt-rust/dist/bin/alpaca_md_qat_loader / # COPY ${FROM_DIR}/.creds /.creds COPY ${FROM_DIR}/alpaca_qat.sh /alpaca_qat.sh COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub -COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts -RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts +RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub # Shared Volumes diff --git a/market_data/alpaca_qat/alpaca_qat.sh b/market_data/alpaca_qat/alpaca_qat.sh index 42411b9..5fa213e 100755 --- a/market_data/alpaca_qat/alpaca_qat.sh +++ b/market_data/alpaca_qat/alpaca_qat.sh @@ -101,7 +101,9 @@ Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/equity" for tgt in ${Targets} do - Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}" + Cmd="/usr/bin/rsync -ahv" + Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" + Cmd+=" ${Source} ${tgt}" echo $Cmd eval $Cmd done diff --git a/market_data/crypto_md_day/Dockerfile b/market_data/crypto_md_day/Dockerfile index a7db1fd..4cba70b 100644 --- a/market_data/crypto_md_day/Dockerfile +++ b/market_data/crypto_md_day/Dockerfile @@ -19,9 +19,8 @@ RUN apt-get update && apt-get install -y rsync openssh-client COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub -COPY docker_dev/shared/known_hosts /root/.ssh/known_hosts -RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub /root/.ssh/known_hosts +RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub # Shared Volumes diff --git a/market_data/crypto_md_day/crypto_md_day.sh b/market_data/crypto_md_day/crypto_md_day.sh index 185038c..f8f0764 100755 --- a/market_data/crypto_md_day/crypto_md_day.sh +++ b/market_data/crypto_md_day/crypto_md_day.sh @@ -72,10 +72,11 @@ Targets= Targets="${Targets} cvtt@cloud21.cvtt.vpn:/opt/store/cvtt/md_archive/crypto" Targets="${Targets} cvtt@hs01.cvtt.vpn:/works/cvtt/md_archive/crypto" - for tgt in ${Targets} do - Cmd="/usr/bin/rsync -ahv ${Source} ${tgt}" + Cmd="/usr/bin/rsync -ahv" + Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" + Cmd+=" ${Source} ${tgt}" echo $Cmd eval $Cmd done diff --git a/release_version.txt b/release_version.txt index d7088b8..9c2b38d 100644 --- a/release_version.txt +++ b/release_version.txt @@ -1 +1 @@ -0.1.1,homestore ==> hs01 \ No newline at end of file +0.1.2,removed dependency on known_hosts file \ No newline at end of file diff --git a/shared/known_hosts b/shared/known_hosts deleted file mode 100644 index 7415d85..0000000 --- a/shared/known_hosts +++ /dev/null @@ -1,6 +0,0 @@ -|1|yT6WmkWOPxXSQeNECVJIX7M3upw=|wQPqbw6mXxwfO83sWVH9IRapSe0= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNIOY8dNaWBcUu31F2OOuXUFGam2sx3Admn51SZR37MOMDEeKzpX+w0OnDGME9uOfuBToDW5F8ZoC+/p492uywZb4ZvUqVE8ic3uTaMro7zB9sykt4AO2rifM/0k7Fo1WoNWSaQptS3qyf7Umgl+qD5H8PxEM1OqKQl38BI5hfKUGx/WSYawgGIKkyO1eFwdjmSJ4RgQpN6/p5+1wJqbqbzkk0cEvZ06BbUbY+nUSUYY5WKa8ICBI0GKDdgRg9FXTV18r0YPT62HhShIyScbmg2h9hDKb7q9lLNLs2qg11jpSZCsIYTR2yms60uqXXq+isReLZ+W4z0t/8YVoID4Q82UlJ8R/LEqiJd8VcjZHaRVqtr7oVQjSurg1ChgCnOHHjvjZLVUg7QRMbCVT2/6PMAjjTyHUx8TKscYjz21ctPMn63pH/6lYzncQrt7ZT6PromVtnBs7p8dHaCyv18z9EBvGuE2rYOsfsCuR7FNlno7ej/gbe6CoaT2KwcNU8vpc= -|1|QgYdyhwM1uy97fUH/l9jXfRnYlE=|P575Q9H/V0Hoszk+G9As1GozQmE= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHlFA8BawdNbgYsFPPUJiG7xfuWI6dwNIFkuUwqco+TPXanMG4CvZ0TjRgb2fwdMXAGyoNAmwcr8eR5HU8wTg8g= -|1|vSzhghnKpUIGuOOgELHWGrbMxd8=|GY2Acsr6iqo4Y/dDS8V5glZ7tmA= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKz0QyrgFpp9NsPvZVwkjoua57SztFOMUnEzOLfW7kPr -|1|LJg/h/d7KSfFYzdtrrEFpcpnF/4=|xRkJEoqiptYGTTt5EuQo4f3zo6c= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKnC41A86Q2mVQWyRXHKzht6t8J1JJsRbYrQRpcMzh6VkVmOPDdAtl4P5iOH/GUaHr+HNhS7Y/QZS0yyjv8u1yOjb6ovfKvIuysj7f87wI1UOtQaUgX/PG5wupk/SgiUcfkpG+hg5wUACbsbMjp+HbE/OB7Nk5ZzUHNZcwQ55mRis8AzLW3IuFEboE+tJCxqTz8mr95ecqEWXdG4cJdDp7BMuCzsPCsOWgpcJiH+kgimok9n4qJPgoFSTVvHKBj0uJlm33rc8ns/Uu8knq6QfeSlzlN7uW0Xue6irW4Uaz69NbSUcFCeG83WNGO7tceYc7tmVM+9jjAtyhh9x1kGyk6thBMimRE5taa+5+bZP5tMctFsze6LVUGOothYW2MVgllJ+ovNFV/lin1y1591eAl340uSs4E4AJH6F/2Tta9OVGvNaXGMTYzKOLBYwo9sXJRHFuPjFxEs0Ya4pFUoCa8B53reNiOafrtwRU8/8G6zMgrF5z4FhCYBcIzoJdWCE= -|1|9oemW7STLSSF7uf+SBwD7JWe0K8=|YCblOLOrxHLzbHkMuZfiUP8nkTU= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPEpfGoofmKWz3/t3Svaahh5ieq4LJ6/n+y3M0rOK/RgndUCESaTZXbw701NOF8BPcJczTQTuQPO2AJwrwQYcK0= -|1|EC/isVVKdC1w/IthdEuhT29iU+w=|VClak/HfuZVXh4zlhhtx9euaTT0= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA2qvtcS1snKAdWKJefMCuKKgVlz0haOT0sElceM06qV diff --git a/test/rsync_test/Dockerfile b/test/rsync_test/Dockerfile new file mode 100644 index 0000000..6fd40a9 --- /dev/null +++ b/test/rsync_test/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +ARG FROM_DIR=docker_dev/test/rsync_test + +COPY ${FROM_DIR}/test.sh /test.sh + +RUN apt-get update && apt-get install -y rsync openssh-client + +COPY docker_dev/shared/id_rsa /root/.ssh/id_rsa +COPY docker_dev/shared/id_rsa.pub /root/.ssh/id_rsa.pub + +RUN chmod 600 /root/.ssh/id_rsa /root/.ssh/id_rsa.pub + +# Shared Volumes +RUN chmod +x /test.sh + +WORKDIR / +SHELL ["/bin/bash", "-c"] +ENV PYTHONPATH=/ +ENTRYPOINT [ "/test.sh" ] + diff --git a/test/rsync_test/test.sh b/test/rsync_test/test.sh new file mode 100755 index 0000000..f388ee1 --- /dev/null +++ b/test/rsync_test/test.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +date > /testfile.txt + +Source=/testfile.txt +Targets= +Targets="${Targets} cvtt@cloud21.cvtt.vpn:/tmp/" +Targets="${Targets} cvtt@hs01.cvtt.vpn:/tmp/" + + +for tgt in ${Targets} +do + Cmd="/usr/bin/rsync -ahv" + Cmd+=" -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" + Cmd+=" ${Source} ${tgt}" + echo $Cmd + eval $Cmd +done +