Skip to content

GitLab

Commit 673d6ee1 authored by Dillenn Terumalai's avatar Dillenn Terumalai 🏠
Browse files
Options

Fixed multiple small issues in the script and replace archive name by timestamp

parent 08e89ee7
Pipeline #1252 passed with stage
    in 2 minutes and 35 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 133 additions and 105 deletions
    spsp
    View file @ 673d6ee1
    #!/usr/bin/env bash
    # SPSP transfer-tool is developped and maintained by SIB. For more informations, please contact spsp-support@sib.swiss
    # SPSP transfer-tool is developed and maintained by SIB. For more informations, please contact spsp-support@sib.swiss
    VERSION="1.2.0" # bumped to support spsp --help, spsp --version
    VERSION="1.3.0" # bumped to support spsp --help, spsp --version
    # Define colors that are used in the help screen
    ......@@ -33,7 +33,7 @@ if [ "$MACHINE" == "UNKNOWN" ]; then
    echo "System must be a Macintosh or Linux"
    echo ""
    echo "System detection determined via uname command"
    echo "If the following is empty, could not find uname command: $(which uname)"
    echo "If the following is empty, could not find uname command: $(uname -v)"
    echo "Your reported uname is: $(uname -s)"
    fi
    ......@@ -95,22 +95,27 @@ showHelp() {
    }
    compressFolder() {
    message="Compressing folder ${COL_LGREEN}$1${COL_RESET}..."
    printf "$message\n"
    tar -czvf $1.tar.gz $1
    message="Folder is compressed as ${COL_LYELLOW}$1.tar.gz${COL_RESET}!"
    printf "$message\n"
    if [ $# -eq 2 ]; then
    message="Compressing folder ${COL_LGREEN}$1${COL_RESET}..."
    printf "$message\n"
    tar -czvf "$2".tar.gz "$1"
    message="Folder is compressed as ${COL_LYELLOW}$2.tar.gz${COL_RESET}!"
    printf "$message\n"
    else
    message="Compressing folder ${COL_LGREEN}$1${COL_RESET}..."
    printf "$message\n"
    tar -czvf "$1".tar.gz "$1"
    message="Folder is compressed as ${COL_LYELLOW}$1.tar.gz${COL_RESET}!"
    printf "$message\n"
    fi
    }
    transferFile() {
    if [ $# -eq 2 ]; then
    for f in "$1"; do
    printf "progress\nput $f" | sftp -b- "${SFTP_URL}$2" || exit 1
    done
    echo "SUBFOLDER REMOTELY IS: $2";
    printf "progress\nput $1" | sftp -b- "${SFTP_URL}$2" || exit 1
    else
    for f in "$1"; do
    printf "progress\nput $f" | sftp -b- "${SFTP_URL}" || exit 1
    done
    printf "progress\nput $1" | sftp -b- "${SFTP_URL}" || exit 1
    fi
    }
    ......@@ -118,7 +123,7 @@ transferFile() {
    hashFile() {
    message="Hashing the file ${COL_LGREEN}$1${COL_RESET} using SHA-256..."
    printf "$message\n"
    $SHACMD $1 > $1.sha256
    $SHACMD "$1" > "$1".sha256
    message="Hash has been properly generated for ${COL_LYELLOW}$1${COL_RESET}!"
    printf "$message\n"
    }
    ......@@ -126,7 +131,7 @@ hashFile() {
    encryptFile() {
    message="Encrypting file ${COL_LGREEN}$1${COL_RESET}..."
    printf "$message\n"
    $GPGCMD --batch --yes --always-trust -o $1.gpg -e -r $RECIPIENT $1
    $GPGCMD --batch --yes --always-trust -o "$1".gpg -e -r $RECIPIENT "$1"
    message="File is encrypted as ${COL_LYELLOW}$1.gpg${COL_RESET}!"
    printf "$message\n"
    }
    ......@@ -134,7 +139,7 @@ encryptFile() {
    deleteSource() {
    message="Deleting the source ${COL_LGREEN}$1${COL_RESET}..."
    printf "$message\n"
    rm -rf $1
    rm -rf "$1"
    message="${COL_LYELLOW}$1${COL_RESET} has been properly deleted!"
    printf "$message\n"
    }
    ......@@ -205,7 +210,8 @@ initMode() {
    }
    autoMode() {
    CURRENTDATE=`date +"%Y-%m-%d"`
    CURRENTDATE=$(date +"%Y-%m-%d")
    ARCHIVENAME=$(date +"%Y%m%d%H%M")
    VIRUSES=viruses
    BACTERIA=bacteria
    ......@@ -214,24 +220,24 @@ autoMode() {
    OUTBOX=".outbox"
    if [ ! -d "$OUTBOX" ]; then
    echo "SPSP: No $OUTBOX folder found within current working directory ${pwd}"
    echo "SPSP: No $OUTBOX folder found within current working directory $(pwd)"
    echo "SPSP: Creating the $OUTBOX folder..."
    mkdir $OUTBOX
    fi
    if [ ! -d "$OUTBOX/$VIRUSES" ]; then
    echo "SPSP: No $VIRUSES folder found in $OUTBOX directory ${pwd}"
    echo "SPSP: No $VIRUSES folder found in $OUTBOX directory $(pwd)"
    echo "SPSP: Creating the $VIRUSES folder..."
    mkdir -p $OUTBOX/$VIRUSES
    fi
    if [ ! -d "$OUTBOX/$BACTERIA" ]; then
    echo "SPSP: No $BACTERIA folder found in $OUTBOX directory ${pwd}"
    echo "SPSP: No $BACTERIA folder found in $OUTBOX directory $(pwd)"
    echo "SPSP: Creating the $BACTERIA folder..."
    mkdir -p $OUTBOX/$BACTERIA
    fi
    SENT=sent
    if [ ! -d "$SENT" ]; then
    echo "SPSP: No $SENT folder found within current working directory ${pwd}"
    echo "SPSP: No $SENT folder found within current working directory $(pwd)"
    echo "SPSP: Creating the $SENT folder..."
    mkdir $SENT
    fi
    ......@@ -245,8 +251,8 @@ autoMode() {
    LOGFILE="./$LOGS/spsp-$CURRENTDATE.log"
    if [ ! -f $LOGFILE ]; then
    touch $LOGFILE
    if [ ! -f "$LOGFILE" ]; then
    touch "$LOGFILE"
    fi
    if [ ! -x "$(command -v gpg)" ]; then
    ......@@ -259,25 +265,35 @@ autoMode() {
    fi
    fi
    if ! nc -w 1 -z ${HOST} 22 &> /dev/null;then
    echo "[`date +"%T"`] local.ERROR: ${HOST}:22 is unreachable, please check your connection" >> $LOGFILE
    if ! nc -w 1 -z "${HOST}" 22 &> /dev/null;then
    echo "[$(date +"%T")] local.ERROR: ${HOST}:22 is unreachable, please check your connection" >> "$LOGFILE"
    echo "SPSP: Error detected please check $LOGFILE"
    echo "########################################"
    exit 55
    fi
    for DIRECTORY in ./$VIRUSES/*; do
    if [ -d "$DIRECTORY" ] && [ $DIRECTORY != $OUTBOX ] && [ $DIRECTORY != $SENT ] && [ $DIRECTORY != $LOGS ]; then
    if ls $DIRECTORY/*.fastq* 1> /dev/null 2>&1 && [ -f $DIRECTORY/*.xlsx ]; then
    cd $VIRUSES
    for DIRECTORY in ./*; do
    if [ -d "$DIRECTORY" ] && [ "$DIRECTORY" != $OUTBOX ] && [ "$DIRECTORY" != $SENT ] && [ "$DIRECTORY" != $LOGS ]; then
    if ls "$DIRECTORY"/*.fastq* 1> /dev/null 2>&1 && ls "$DIRECTORY"/*.xlsx 1> /dev/null 2>&1; then
    echo "SPSP: Compressing $DIRECTORY folder..."
    compressFolder $DIRECTORY &> /dev/null
    echo "SPSP: Sucessfully compressed!"
    if [[ "$*" == *--fake* ]] || [[ "$*" == *-F* ]]; then
    compressFolder "$DIRECTORY" &> /dev/null
    else
    compressFolder "$DIRECTORY" "$ARCHIVENAME" &> /dev/null
    fi
    echo "SPSP: Successfully compressed!"
    echo "SPSP: Removing the source and moving the tar.gz to '$OUTBOX'/'$VIRUSES' folder."
    mv $DIRECTORY.tar.gz ./$OUTBOX/$VIRUSES
    rm -rf $DIRECTORY
    echo "SPSP: Deleted!"
    if [[ "$*" == *--fake* ]] || [[ "$*" == *-F* ]]; then
    mv "$DIRECTORY".tar.gz ../$OUTBOX/$VIRUSES/
    else
    mv "$ARCHIVENAME".tar.gz ../$OUTBOX/$VIRUSES/
    fi
    rm -rf "$DIRECTORY"
    echo "SPSP: Original $DIRECTORY deleted!"
    else
    echo "[`date +"%T"`] local.ERROR: Missing files (*.fastq/.fastq.gz and/or *.xlsx) inside ${DIRECTORY}" >> $LOGFILE
    echo "[$(date +"%T")] local.ERROR: Missing files (*.fastq/.fastq.gz and/or *.xlsx) inside ${DIRECTORY}" >> "$LOGFILE"
    echo "SPSP: Error detected please check $LOGFILE"
    echo "########################################"
    exit 13
    ......@@ -285,18 +301,30 @@ autoMode() {
    fi
    done
    for DIRECTORY in ./$BACTERIA/*; do
    if [ -d "$DIRECTORY" ] && [ $DIRECTORY != $OUTBOX ] && [ $DIRECTORY != $SENT ] && [ $DIRECTORY != $LOGS ]; then
    if ls $DIRECTORY/*.fastq* 1> /dev/null 2>&1 && [ -f $DIRECTORY/*.xlsx ]; then
    cd ..;
    cd $BACTERIA;
    for DIRECTORY in ./*; do
    if [ -d "$DIRECTORY" ] && [ "$DIRECTORY" != $OUTBOX ] && [ "$DIRECTORY" != $SENT ] && [ "$DIRECTORY" != $LOGS ]; then
    if ls "$DIRECTORY"/*.fastq* 1> /dev/null 2>&1 && ls "$DIRECTORY"/*.xlsx 1> /dev/null 2>&1; then
    echo "SPSP: Compressing $DIRECTORY folder..."
    compressFolder $DIRECTORY &> /dev/null
    echo "SPSP: Sucessfully compressed!"
    if [[ "$*" == *--fake* ]] || [[ "$*" == *-F* ]]; then
    compressFolder "$DIRECTORY" &> /dev/null
    else
    compressFolder "$DIRECTORY" "$ARCHIVENAME" &> /dev/null
    fi
    echo "SPSP: Successfully compressed!"
    echo "SPSP: Removing the source and moving the tar.gz to '$OUTBOX'/'$BACTERIA' folder."
    mv $DIRECTORY.tar.gz ./$OUTBOX/$BACTERIA
    rm -rf $DIRECTORY
    echo "SPSP: Deleted!"
    if [[ "$*" == *--fake* ]] || [[ "$*" == *-F* ]]; then
    mv "$DIRECTORY".tar.gz ../$OUTBOX/$BACTERIA/
    else
    mv "$ARCHIVENAME".tar.gz ../$OUTBOX/$BACTERIA/
    fi
    rm -rf "$DIRECTORY"
    echo "SPSP: Original $DIRECTORY deleted!"
    else
    echo "[`date +"%T"`] local.ERROR: Missing files (*.fastq/.fastq.gz and/or *.xlsx) inside ${DIRECTORY}" >> $LOGFILE
    echo "[$(date +"%T")] local.ERROR: Missing files (*.fastq/.fastq.gz and/or *.xlsx) inside ${DIRECTORY}" >> "$LOGFILE"
    echo "SPSP: Error detected please check $LOGFILE"
    echo "########################################"
    exit 13
    ......@@ -304,37 +332,39 @@ autoMode() {
    fi
    done
    cd ..;
    cd $OUTBOX/$VIRUSES;
    for FILE in *.*; do
    if [ -f "$FILE" ]; then
    echo "SPSP: Encrypting $FILE using SPSP public key..."
    encryptFile $FILE
    encryptFile "$FILE"
    echo "SPSP: $FILE is now encrypted"
    echo "SPSP: Digesting $FILE using SHA-256 algorithm..."
    hashFile "$FILE"
    echo "SPSP: $FILE is now hashed!"
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    echo "SPSP: Starting the transfer"
    transferFile $FILE.gpg /$VIRUSES
    transferFile $FILE.sha256 /$VIRUSES
    echo "SPSP: $FILE has been transferred sucessfully!"
    transferFile "$FILE".gpg /$VIRUSES
    transferFile "$FILE".sha256 /$VIRUSES
    echo "SPSP: $FILE has been transferred successfully!"
    echo "SPSP: Removing unencrypted version of $FILE..."
    rm $FILE
    rm -rf "$FILE"
    echo "SPSP: Deleted!"
    if [[ "$@" == *--no-archive* ]] || [[ "$@" == *-NA* ]]; then
    if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
    echo "SPSP: 'No archive' mode is activated, removing archives..."
    rm $FILE.gpg
    rm $FILE.sha256
    else
    rm -rf "$FILE".gpg
    rm -rf "$FILE".sha256
    else
    echo "SPSP: Moving $FILE.gpg and $FILE.sha256 to '$SENT' directory!"
    mv $FILE.sha256 ../../$SENT
    mv $FILE.gpg ../../$SENT
    echo "SPSP: Archives have been moved sucessfully!"
    mv "$FILE".sha256 ../../$SENT
    mv "$FILE".gpg ../../$SENT
    echo "SPSP: Archives have been moved successfully!"
    fi
    else
    else
    cd ../..
    echo "[`date +"%T"`] local.ERROR: Missing the encrypted version or the hash for $FILE" >> $LOGFILE
    echo "[$(date +"%T")] local.ERROR: Missing the encrypted version or the hash for $FILE" >> "$LOGFILE"
    echo "SPSP: Error detected please check $LOGFILE"
    echo "########################################"
    exit 2
    ......@@ -342,39 +372,37 @@ autoMode() {
    fi
    done
    cd ..
    cd $BACTERIA
    cd ../$BACTERIA;
    for FILE in *.*; do
    if [ -f "$FILE" ]; then
    echo "SPSP: Encrypting $FILE using SPSP public key..."
    encryptFile $FILE
    encryptFile "$FILE"
    echo "SPSP: $FILE is now encrypted"
    echo "SPSP: Digesting $FILE using SHA-256 algorithm..."
    hashFile "$FILE"
    echo "SPSP: $FILE is now hashed!"
    if [ -f "$FILE.sha256" ] && [ -f "$FILE.gpg" ]; then
    echo "SPSP: Starting the transfer"
    transferFile $FILE.gpg /$BACTERIA
    transferFile $FILE.sha256 /$BACTERIA
    echo "SPSP: $FILE has been transferred sucessfully!"
    transferFile "$FILE".gpg /$BACTERIA
    transferFile "$FILE".sha256 /$BACTERIA
    echo "SPSP: $FILE has been transferred successfully!"
    echo "SPSP: Removing unencrypted version of $FILE..."
    rm $FILE
    rm -rf "$FILE"
    echo "SPSP: Deleted!"
    if [[ "$@" == *--no-archive* ]] || [[ "$@" == *-NA* ]]; then
    if [[ "$*" == *--no-archive* ]] || [[ "$*" == *-NA* ]]; then
    echo "SPSP: 'No archive' mode is activated, removing archives..."
    rm $FILE.gpg
    rm $FILE.sha256
    else
    rm -rf "$FILE".gpg
    rm -rf "$FILE".sha256
    else
    echo "SPSP: Moving $FILE.gpg and $FILE.sha256 to '$SENT' directory!"
    mv $FILE.sha256 ../../$SENT
    mv $FILE.gpg ../../$SENT
    echo "SPSP: Archives have been moved sucessfully!"
    mv "$FILE".sha256 ../../$SENT
    mv "$FILE".gpg ../../$SENT
    echo "SPSP: Archives have been moved successfully!"
    fi
    else
    else
    cd ../..
    echo "[`date +"%T"`] local.ERROR: Missing the encrypted version or the hash for $FILE" >> $LOGFILE
    echo "[$(date +"%T")] local.ERROR: Missing the encrypted version or the hash for $FILE" >> "$LOGFILE"
    echo "SPSP: Error detected please check $LOGFILE"
    echo "########################################"
    exit 2
    ......@@ -390,47 +418,47 @@ generateFakeFiles() {
    VIRUSES=viruses
    BACTERIA=bacteria
    if [ ! -d "$VIRUSES/fake" ]; then
    mkdir -p $VIRUSES/fake
    if [ ! -d "$VIRUSES/fake-viruses" ]; then
    mkdir -p $VIRUSES/fake-viruses
    fi
    rm -rf ./$VIRUSES/fake/*
    rm -rf ./$VIRUSES/fake-viruses/*
    truncate -s 5M ./$VIRUSES/fake/fake-viruses.xlsx
    truncate -s 5M ./$VIRUSES/fake-viruses/fake-viruses.xlsx
    for i in {1..10}; do
    truncate -s 1G ./$VIRUSES/fake/fake-virus${i}.fastq.gz
    truncate -s 1G ./$VIRUSES/fake-viruses/fake-virus${i}.fastq.gz
    done
    if [ ! -d "$BACTERIA/fake" ]; then
    mkdir -p $BACTERIA/fake
    if [ ! -d "$BACTERIA/fake-bacteria" ]; then
    mkdir -p $BACTERIA/fake-bacteria
    fi
    rm -rf ./$BACTERIA/fake/*
    rm -rf ./$BACTERIA/fake-bacteria/*
    truncate -s 5M ./$BACTERIA/fake/bacteria.xlsx
    truncate -s 5M ./$BACTERIA/fake-bacteria/bacteria.xlsx
    for i in {1..10}; do
    truncate -s 1G ./$BACTERIA/fake/bacteria${i}.fastq.gz
    truncate -s 1G ./$BACTERIA/fake-bacteria/bacteria${i}.fastq.gz
    done
    autoMode --no-archive
    autoMode -NA -F
    message="Removing the ${COL_LGREEN}remote fake files${COL_RESET} from SFTP..."
    printf "$message\n"
    echo "rm fake*" | sftp ${SFTP_URL}/viruses || exit 1
    echo "rm fake*" | sftp ${SFTP_URL}/bacteria || exit 1
    echo "Files has been properly removed!"
    # message="Removing the ${COL_LGREEN}remote fake files${COL_RESET} from SFTP..."
    # printf "$message\n"
    # echo "rm fake*" | sftp ${SFTP_URL}/viruses || exit 1
    # echo "rm fake*" | sftp ${SFTP_URL}/bacteria || exit 1
    # echo "Files has been properly removed!"
    message="Removing the ${COL_LGREEN}local fake files${COL_RESET} from SFTP..."
    printf "$message\n"
    cleanup
    echo "Files has been properly removed!"
    exit 0
    # message="Removing the ${COL_LGREEN}local fake files${COL_RESET} from SFTP..."
    # printf "$message\n"
    # cleanup
    # echo "Files has been properly removed!"
    # exit 0
    }
    # If we pass any arguments...
    if [ $# -gt 0 ]; then
    if [[ "$@" == *--without-env* ]] || [[ "$@" == *-W* ]]; then
    if [[ "$*" == *--without-env* ]] || [[ "$*" == *-W* ]]; then
    echo "Ignoring the .env file"
    echo "Using environment variables"
    else
    ......@@ -441,13 +469,13 @@ if [ $# -gt 0 ]; then
    echo "Creating a new .env file..."
    answer=false
    while [ "$answer" = false ]; do
    read -p "Please type laboratory/institution identifier: " identifier
    read -p "Is $identifier correct?[y/n] " -n 1 -r
    read -pr "Please type laboratory/institution identifier: " identifier
    read -pr "Is $identifier correct?[y/n] " -n 1 -r
    if [[ $REPLY =~ ^[Yy]$ ]]
    then
    echo "ID=$identifier" > .env
    echo 'HOST=spsp-sftp.vital-it.ch' >> .env
    echo 'SFTP_URL=${ID}@${HOST}:/data' >> .env
    echo "SFTP_URL=${ID}@${HOST}:/data" >> .env
    answer=true
    else
    echo ""
    ......@@ -458,7 +486,7 @@ if [ $# -gt 0 ]; then
    fi
    fi
    if [ "$1" == "--version" ] || [ "$1" == "-v" ] || [ "$1" == "version" ]; then
    if [ "$1" == "--version" ] || [ "$1" == "-V" ] || [ "$1" == "version" ]; then
    showVersion
    exit 0
    fi
    ......@@ -488,25 +516,25 @@ if [ $# -gt 0 ]; then
    # Compress a folder given as an input
    elif [ "$1" == "compress" ]; then
    shift 1
    compressFolder $@
    compressFolder "$@"
    exit 0
    # Transfer a file given as an input
    elif [ "$1" == "transfer" ]; then
    shift 1
    transferFile $@
    transferFile "$@"
    exit 0
    # Digest a file given as an input
    elif [ "$1" == "hash" ]; then
    shift 1
    hashFile $@
    hashFile "$@"
    exit 0
    # Encrypt a folder/file given as an input
    elif [ "$1" == "encrypt" ]; then
    shift 1
    encryptFile $@
    encryptFile "$@"
    exit 0
    else
    ......
    Markdown is supported
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment