Skip to content
GitLab
Commit 4a1cab14 authored by Dillenn Terumalai's avatar Dillenn Terumalai
Browse files

Updated main script following WAP recommendations and improved CI/CD

parent b9137815
Pipeline #1357 passed with stage
in 13 minutes and 40 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 4a1cab14
......@@ -10,6 +10,11 @@ centos test:
before_script:
- dnf install -y nc
- dnf install -y openssh-clients
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" >> ~/.ssh/id_ed25519
- chmod 600 ~/.ssh/id_ed25519
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
stage: test
script:
- ./spsp init --without-env
......@@ -29,6 +34,11 @@ debian test:
- apt-get install -y gpg
- apt-get install -y openssh-client
- apt-get install -y gpg
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" >> ~/.ssh/id_ed25519
- chmod 600 ~/.ssh/id_ed25519
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
stage: test
script:
- ./spsp init --without-env
......
spsp
View file @ 4a1cab14
......@@ -2,7 +2,7 @@
# SPSP transfer-tool is developed and maintained by SIB. For more information, please contact spsp-support@sib.swiss
VERSION="1.4.0" # bumped to support spsp --help, spsp --version
VERSION="1.5.0" # bumped to support spsp --help, spsp --version
# Define colors that are used in the help screen
......@@ -57,7 +57,7 @@ showHelp() {
usage="$usage ./spsp <cmd> <options>"
commands="${COL_LYELLOW}Commands:\n${COL_RESET}"
commands="$commands auto Automatically run the commands to Compress$, Hash, Encrypt & Transfer, add --no-archive option to delete the files after the transfer\n"
commands="$commands auto Automatically run the commands to Compress, Hash, Encrypt & Transfer, add --no-archive option to delete the files after the transfer\n"
commands="$commands help Shows help screen\n"
commands="$commands test Dry run with fake generated files with report\n"
commands="$commands cleanup Clean the following directories: logs, sent, viruses, bacteria and .outbox\n"
......@@ -85,7 +85,10 @@ compressFolder() {
}
transferFile() {
printf "put $1" | sftp -q "${SFTP_URL}$2" 1> /dev/null
REMOTE_PATH=$1
shift 1
# printf "put $@" | sftp -q "${SFTP_URL}$2" 1> /dev/null
for f in $@; do echo put "$f" | sftp -q -b- "${SFTP_URL}$REMOTE_PATH" 1> /dev/null; done
}
hashFile() {
......@@ -159,6 +162,7 @@ initMode() {
}
autoMode() {
START=$(date +%s)
CURRENTDATE=$(date +"%Y-%m-%d")
VIRUSES=viruses
BACTERIA=bacteria
......@@ -345,7 +349,7 @@ autoMode() {
STEP_ERRORS=0
printf " - Hashing, encrypting and transferring the archives... "
printf " - Hashing and encrypting the archives... "
cd $OUTBOX/$VIRUSES;
......@@ -365,43 +369,19 @@ autoMode() {
if [ -f "$FILE.gpg" ]; then
echo "[$(date +"%T")] local.SUCCESS: $FILE has been successfully encrypted into $FILE.gpg" >> "../../$LOGFILE"
else
((ERRORS=ERRORS+1))
((STEP_ERRORS=STEP_ERRORS+1))
echo "[$(date +"%T")] local.ERROR: Encryption of $FILE failed" >> "../../$LOGFILE"
echo "[$(date +"%F %T")] local.ERROR: Encryption of $FILE failed" >> "../../$ERRORLOGFILE"
fi
if [[ "$*" == *--local* ]] || [[ "$*" == *-L* ]]; then
((SUCCESS=SUCCESS+1))
else
if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
transferFile "$FILE".gpg /$VIRUSES
transferFile "$FILE".sha256 /$VIRUSES
((SUCCESS=SUCCESS+1))
fi
fi
echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg has been successfully transferred to the remote server" >> "../../$LOGFILE"
echo "[$(date +"%T")] local.SUCCESS: $FILE.sha256 has been successfully transferred to the remote server" >> "../../$LOGFILE"
rm -rf "$FILE"
if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
rm -rf "$FILE".gpg
rm -rf "$FILE".sha256
echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE.gpg and $FILE.sha256 successfully removed" >> "../../$LOGFILE"
fi
else
if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
mv "$FILE".sha256 ../../$SENT
mv "$FILE".gpg ../../$SENT
echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg and $FILE.sha256 successfully moved to sent folder" >> "../../$LOGFILE"
fi
fi
fi
done
rm -rf *.*
cd ../$BACTERIA;
for FILE in *.*; do
......@@ -426,33 +406,64 @@ autoMode() {
echo "[$(date +"%T")] local.ERROR: Encryption of $FILE failed" >> "../../$LOGFILE"
echo "[$(date +"%F %T")] local.ERROR: Encryption of $FILE failed" >> "../../$ERRORLOGFILE"
fi
rm -rf "$FILE"
if [[ "$*" == *--local* ]] || [[ "$*" == *-L* ]]; then
((SUCCESS=SUCCESS+1))
fi
done
if [[ $STEP_ERRORS > 0 ]]; then
printf "${BACK_RED} ERROR ${COL_RESET}\n"
else
printf "${BACK_GREEN} DONE ${COL_RESET}\n"
fi
STEP_ERRORS=0
printf " - Transferring the encrypted archives... "
cd ../$VIRUSES;
for FILE in *.*; do
if [ -f "$FILE" ]; then
echo put "$FILE" | sftp -q -b- "${SFTP_URL}/$VIRUSES" 1> /dev/null
((SUCCESS=SUCCESS+1))
echo "[$(date +"%T")] local.SUCCESS: $FILE has been successfully transferred to the remote server" >> "../../$LOGFILE"
if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
rm -rf "$FILE"
echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE successfully removed" >> "../../$LOGFILE"
else
if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
transferFile "$FILE".gpg /$BACTERIA
transferFile "$FILE".sha256 /$BACTERIA
((SUCCESS=SUCCESS+1))
fi
mv "$FILE" ../../$SENT
echo "[$(date +"%T")] local.SUCCESS: $FILE successfully moved to $SENT folder" >> "../../$LOGFILE"
fi
echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg has been successfully transferred to the remote server" >> "../../$LOGFILE"
echo "[$(date +"%T")] local.SUCCESS: $FILE.sha256 has been successfully transferred to the remote server" >> "../../$LOGFILE"
rm -rf "$FILE"
else
((ERRORS=ERRORS+1))
((STEP_ERRORS=STEP_ERRORS+1))
echo "[$(date +"%T")] local.ERROR: Transfer of $FILE failed" >> "../../$LOGFILE"
echo "[$(date +"%F %T")] local.ERROR: Transfer of $FILE failed" >> "../../$ERRORLOGFILE"
fi
done
cd ../$BACTERIA;
for FILE in *.*; do
if [ -f "$FILE" ]; then
echo put "$FILE" | sftp -q -b- "${SFTP_URL}/$BACTERIA" 1> /dev/null
((SUCCESS=SUCCESS+1))
echo "[$(date +"%T")] local.SUCCESS: $FILE has been successfully transferred to the remote server" >> "../../$LOGFILE"
if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
rm -rf "$FILE".gpg
rm -rf "$FILE".sha256
echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE.gpg and $FILE.sha256 successfully removed" >> "../../$LOGFILE"
fi
rm -rf "$FILE"
echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE successfully removed" >> "../../$LOGFILE"
else
if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
mv "$FILE".sha256 ../../$SENT
mv "$FILE".gpg ../../$SENT
echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg and $FILE.sha256 successfully moved to sent folder" >> "../../$LOGFILE"
fi
mv "$FILE" ../../$SENT
echo "[$(date +"%T")] local.SUCCESS: $FILE successfully moved to $SENT folder" >> "../../$LOGFILE"
fi
fi
else
((ERRORS=ERRORS+1))
((STEP_ERRORS=STEP_ERRORS+1))
echo "[$(date +"%T")] local.ERROR: Transfer of $FILE failed" >> "../../$LOGFILE"
echo "[$(date +"%F %T")] local.ERROR: Transfer of $FILE failed" >> "../../$ERRORLOGFILE"
fi
done
if [[ $STEP_ERRORS > 0 ]]; then
......@@ -471,12 +482,13 @@ autoMode() {
exit 1
fi
exit 0
END=$(date +%s)
DIFF=$(( $END - $START ))
echo "Time: $DIFF s"
}
generateFakeFiles() {
START=$(date +%s)
VIRUSES=viruses
BACTERIA=bacteria
......@@ -503,11 +515,9 @@ generateFakeFiles() {
truncate -s 1G ./$BACTERIA/fake-bacteria/bacteria${i}.fastq.gz
done
autoMode --no-archive --fake --local
autoMode --fake --no-archive
END=$(date +%s)
DIFF=$(( $END - $START ))
echo "Time: $DIFF s"
exit 0
}
# If we pass any arguments...
......@@ -528,7 +538,7 @@ if [ $# -gt 0 ]; then
then
echo "ID=$identifier" > .env
echo 'HOST=spsp-sftp.vital-it.ch' >> .env
echo 'SFTP_URL=${ID}@${HOST}:/data' >> .env
echo 'SFTP_URL=${ID}@${HOST}:data' >> .env
answer=true
else
echo ""
......@@ -567,9 +577,9 @@ if [ $# -gt 0 ]; then
cleanup
exit 0
elif [ "$1" == "compress" ]; then
elif [ "$1" == "transfer" ]; then
shift 1
compressFolder "$@"
transferFile "$@"
exit 0
else
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment