Skip to content

GitLab

Transfer Tool
Transfer Tool
Commit 4a1cab14 authored by Dillenn Terumalai's avatar Dillenn Terumalai 🏠
Browse files
Options

Updated main script following WAP recommendations and improved CI/CD

parent b9137815
Pipeline #1357 passed with stage
    in 13 minutes and 40 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 81 additions and 61 deletions
    .gitlab-ci.yml
    View file @ 4a1cab14
    ......@@ -10,6 +10,11 @@ centos test:
    before_script:
    - dnf install -y nc
    - dnf install -y openssh-clients
    - mkdir -p ~/.ssh
    - echo "$SSH_PRIVATE_KEY" >> ~/.ssh/id_ed25519
    - chmod 600 ~/.ssh/id_ed25519
    - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
    stage: test
    script:
    - ./spsp init --without-env
    ......@@ -29,6 +34,11 @@ debian test:
    - apt-get install -y gpg
    - apt-get install -y openssh-client
    - apt-get install -y gpg
    - mkdir -p ~/.ssh
    - echo "$SSH_PRIVATE_KEY" >> ~/.ssh/id_ed25519
    - chmod 600 ~/.ssh/id_ed25519
    - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
    stage: test
    script:
    - ./spsp init --without-env
    ......
    spsp
    View file @ 4a1cab14
    ......@@ -2,7 +2,7 @@
    # SPSP transfer-tool is developed and maintained by SIB. For more information, please contact spsp-support@sib.swiss
    VERSION="1.4.0" # bumped to support spsp --help, spsp --version
    VERSION="1.5.0" # bumped to support spsp --help, spsp --version
    # Define colors that are used in the help screen
    ......@@ -57,7 +57,7 @@ showHelp() {
    usage="$usage ./spsp <cmd> <options>"
    commands="${COL_LYELLOW}Commands:\n${COL_RESET}"
    commands="$commands auto Automatically run the commands to Compress$, Hash, Encrypt & Transfer, add --no-archive option to delete the files after the transfer\n"
    commands="$commands auto Automatically run the commands to Compress, Hash, Encrypt & Transfer, add --no-archive option to delete the files after the transfer\n"
    commands="$commands help Shows help screen\n"
    commands="$commands test Dry run with fake generated files with report\n"
    commands="$commands cleanup Clean the following directories: logs, sent, viruses, bacteria and .outbox\n"
    ......@@ -85,7 +85,10 @@ compressFolder() {
    }
    transferFile() {
    printf "put $1" | sftp -q "${SFTP_URL}$2" 1> /dev/null
    REMOTE_PATH=$1
    shift 1
    # printf "put $@" | sftp -q "${SFTP_URL}$2" 1> /dev/null
    for f in $@; do echo put "$f" | sftp -q -b- "${SFTP_URL}$REMOTE_PATH" 1> /dev/null; done
    }
    hashFile() {
    ......@@ -159,6 +162,7 @@ initMode() {
    }
    autoMode() {
    START=$(date +%s)
    CURRENTDATE=$(date +"%Y-%m-%d")
    VIRUSES=viruses
    BACTERIA=bacteria
    ......@@ -345,7 +349,7 @@ autoMode() {
    STEP_ERRORS=0
    printf " - Hashing, encrypting and transferring the archives... "
    printf " - Hashing and encrypting the archives... "
    cd $OUTBOX/$VIRUSES;
    ......@@ -365,43 +369,19 @@ autoMode() {
    if [ -f "$FILE.gpg" ]; then
    echo "[$(date +"%T")] local.SUCCESS: $FILE has been successfully encrypted into $FILE.gpg" >> "../../$LOGFILE"
    else
    ((ERRORS=ERRORS+1))
    ((STEP_ERRORS=STEP_ERRORS+1))
    echo "[$(date +"%T")] local.ERROR: Encryption of $FILE failed" >> "../../$LOGFILE"
    echo "[$(date +"%F %T")] local.ERROR: Encryption of $FILE failed" >> "../../$ERRORLOGFILE"
    fi
    if [[ "$*" == *--local* ]] || [[ "$*" == *-L* ]]; then
    ((SUCCESS=SUCCESS+1))
    else
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    transferFile "$FILE".gpg /$VIRUSES
    transferFile "$FILE".sha256 /$VIRUSES
    ((SUCCESS=SUCCESS+1))
    fi
    fi
    echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg has been successfully transferred to the remote server" >> "../../$LOGFILE"
    echo "[$(date +"%T")] local.SUCCESS: $FILE.sha256 has been successfully transferred to the remote server" >> "../../$LOGFILE"
    rm -rf "$FILE"
    if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    rm -rf "$FILE".gpg
    rm -rf "$FILE".sha256
    echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE.gpg and $FILE.sha256 successfully removed" >> "../../$LOGFILE"
    fi
    else
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    mv "$FILE".sha256 ../../$SENT
    mv "$FILE".gpg ../../$SENT
    echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg and $FILE.sha256 successfully moved to sent folder" >> "../../$LOGFILE"
    fi
    fi
    fi
    done
    rm -rf *.*
    cd ../$BACTERIA;
    for FILE in *.*; do
    ......@@ -426,33 +406,64 @@ autoMode() {
    echo "[$(date +"%T")] local.ERROR: Encryption of $FILE failed" >> "../../$LOGFILE"
    echo "[$(date +"%F %T")] local.ERROR: Encryption of $FILE failed" >> "../../$ERRORLOGFILE"
    fi
    rm -rf "$FILE"
    if [[ "$*" == *--local* ]] || [[ "$*" == *-L* ]]; then
    ((SUCCESS=SUCCESS+1))
    fi
    done
    if [[ $STEP_ERRORS > 0 ]]; then
    printf "${BACK_RED} ERROR ${COL_RESET}\n"
    else
    printf "${BACK_GREEN} DONE ${COL_RESET}\n"
    fi
    STEP_ERRORS=0
    printf " - Transferring the encrypted archives... "
    cd ../$VIRUSES;
    for FILE in *.*; do
    if [ -f "$FILE" ]; then
    echo put "$FILE" | sftp -q -b- "${SFTP_URL}/$VIRUSES" 1> /dev/null
    ((SUCCESS=SUCCESS+1))
    echo "[$(date +"%T")] local.SUCCESS: $FILE has been successfully transferred to the remote server" >> "../../$LOGFILE"
    if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
    rm -rf "$FILE"
    echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE successfully removed" >> "../../$LOGFILE"
    else
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    transferFile "$FILE".gpg /$BACTERIA
    transferFile "$FILE".sha256 /$BACTERIA
    ((SUCCESS=SUCCESS+1))
    fi
    mv "$FILE" ../../$SENT
    echo "[$(date +"%T")] local.SUCCESS: $FILE successfully moved to $SENT folder" >> "../../$LOGFILE"
    fi
    echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg has been successfully transferred to the remote server" >> "../../$LOGFILE"
    echo "[$(date +"%T")] local.SUCCESS: $FILE.sha256 has been successfully transferred to the remote server" >> "../../$LOGFILE"
    rm -rf "$FILE"
    else
    ((ERRORS=ERRORS+1))
    ((STEP_ERRORS=STEP_ERRORS+1))
    echo "[$(date +"%T")] local.ERROR: Transfer of $FILE failed" >> "../../$LOGFILE"
    echo "[$(date +"%F %T")] local.ERROR: Transfer of $FILE failed" >> "../../$ERRORLOGFILE"
    fi
    done
    cd ../$BACTERIA;
    for FILE in *.*; do
    if [ -f "$FILE" ]; then
    echo put "$FILE" | sftp -q -b- "${SFTP_URL}/$BACTERIA" 1> /dev/null
    ((SUCCESS=SUCCESS+1))
    echo "[$(date +"%T")] local.SUCCESS: $FILE has been successfully transferred to the remote server" >> "../../$LOGFILE"
    if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    rm -rf "$FILE".gpg
    rm -rf "$FILE".sha256
    echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE.gpg and $FILE.sha256 successfully removed" >> "../../$LOGFILE"
    fi
    rm -rf "$FILE"
    echo "[$(date +"%T")] local.SUCCESS: --NA activated, $FILE successfully removed" >> "../../$LOGFILE"
    else
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    mv "$FILE".sha256 ../../$SENT
    mv "$FILE".gpg ../../$SENT
    echo "[$(date +"%T")] local.SUCCESS: $FILE.gpg and $FILE.sha256 successfully moved to sent folder" >> "../../$LOGFILE"
    fi
    mv "$FILE" ../../$SENT
    echo "[$(date +"%T")] local.SUCCESS: $FILE successfully moved to $SENT folder" >> "../../$LOGFILE"
    fi
    fi
    else
    ((ERRORS=ERRORS+1))
    ((STEP_ERRORS=STEP_ERRORS+1))
    echo "[$(date +"%T")] local.ERROR: Transfer of $FILE failed" >> "../../$LOGFILE"
    echo "[$(date +"%F %T")] local.ERROR: Transfer of $FILE failed" >> "../../$ERRORLOGFILE"
    fi
    done
    if [[ $STEP_ERRORS > 0 ]]; then
    ......@@ -471,12 +482,13 @@ autoMode() {
    exit 1
    fi
    exit 0
    END=$(date +%s)
    DIFF=$(( $END - $START ))
    echo "Time: $DIFF s"
    }
    generateFakeFiles() {
    START=$(date +%s)
    VIRUSES=viruses
    BACTERIA=bacteria
    ......@@ -503,11 +515,9 @@ generateFakeFiles() {
    truncate -s 1G ./$BACTERIA/fake-bacteria/bacteria${i}.fastq.gz
    done
    autoMode --no-archive --fake --local
    autoMode --fake --no-archive
    END=$(date +%s)
    DIFF=$(( $END - $START ))
    echo "Time: $DIFF s"
    exit 0
    }
    # If we pass any arguments...
    ......@@ -528,7 +538,7 @@ if [ $# -gt 0 ]; then
    then
    echo "ID=$identifier" > .env
    echo 'HOST=spsp-sftp.vital-it.ch' >> .env
    echo 'SFTP_URL=${ID}@${HOST}:/data' >> .env
    echo 'SFTP_URL=${ID}@${HOST}:data' >> .env
    answer=true
    else
    echo ""
    ......@@ -567,9 +577,9 @@ if [ $# -gt 0 ]; then
    cleanup
    exit 0
    elif [ "$1" == "compress" ]; then
    elif [ "$1" == "transfer" ]; then
    shift 1
    compressFolder "$@"
    transferFile "$@"
    exit 0
    else
    ......
    Markdown is supported
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment