Commit 105a2588 authored by Dillenn Terumalai's avatar Dillenn Terumalai
Browse files

Improved the test function and changed key management

parent 33096904
ID=TST_0001
HOST=spsp-sftp.vital-it.ch
SFTP_URL=${ID}@${HOST}:/data
\ No newline at end of file
......@@ -2,7 +2,7 @@
!logs
!.outbox
!sent
!.env
!.gitlab-ci.yml
!.gitignore
!.pub
!README.md
......
# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
# you can delete this line if you're not using Docker
image: busybox:latest
before_script:
- echo "Before script section"
- echo "For example you might run an update here or install a build dependency"
- echo "Or perhaps you might print out some debugging details"
after_script:
- echo "After script section"
- echo "For example you might do some cleanup here"
build1:
stage: build
script:
- echo "Do your build here"
test1:
stage: test
script:
- echo "Do a test here"
- echo "For example run a test suite"
test2:
stage: test
script:
- echo "Do another parallel test here"
- echo "For example run a lint test"
deploy1:
stage: deploy
script:
- echo "Do your deploy here"
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: SKS 1.1.6
Comment: Hostname: c783b2988888
mQENBF33qhoBCACyr8Ga4LPHSleEOvuZYUfU3/xLH3tiV7IKM5Rsp6vPvSAmXjFz
h8MR0M1tT7daTgT4pgcbuiogq33Jf9bTQCao3zwd5YRc7ZsdxWOeZJQj0w8A4HWO
nCi9bO6peB1sNzq6dq/AXv38r5iVfV/vpNDlCwYmZxfNyj53qsApsthlMH3rgOrA
nG1wk3a3bPNT/ubMbnsKg2Mi3Z22E5LdYKCkzHLlqbWdt96vs9pM63FjP371TfvC
WxTlQ7y3bN5h+MAeiFEYPsDoWbMXSIent7cyP1z0EpnaaIwK3y1EbGq/CYj9N2sq
HexFP0f/2lRoZqxNG7oxm1Jas1ZGrmavk7j5ABEBAAG0IlNQU1AgU0ZUUCA8c3Bz
cC1zdXBwb3J0QHNpYi5zd2lzcz6JAU4EEwEIADgWIQSryfwUqslS53Z/0UpItw5y
S6/gowUCXfeqGgIbAwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBItw5yS6/g
o//rB/9g0fTMePnq+moVYZBNnOJf1MiEgX+nvJSlsdwZXMNv9873ZHlKnD1axY4y
FYgRg8sqExp+Q4Ea265pUF6Z2UI4ir7WP8+TBz1UMv9O5HMEXmSuWCk96SYQdEqD
V6Nx+cY08uOm9NDIaAkm83mtwXAmPty5xJVyzImjunVQQ7VFHg6Y9Sf/sDQQX0c6
wx9QKXubK06tN6JLWmY0QwxTDHCc2t5Ujtx8rLAzaBJP+9zsKYWX9b/fD//3fFK2
aQyJh1febvtx5GFgkW76cuf/wPObKoo1k+4Z7JDqSnct/pj7qytMC1g6tVGLJATv
6qt3s9jPdGmYwVR0WcNt9ETQNjN+uQENBF33qhoBCACW0C/VUxGZPEypilwX1T09
nDjVP6bAVaDhqQzY2L5QPgg70CSM863gBnGV5wr1yLsdf7t+T1Z7diio+edg0ICS
9VTysJyfjuu/cA6mcBxhM3SRgKtPxfgObjvt9toy+ulXxSfWWad1xv1jnkVNTE1y
/Im8+BkQ3zRhdpTOLHT/OCtqvXrNqlN91lL7JWZJ3dRheVkI0Bd1Y7EuFWGXqVLT
5teU4bCeKbBBlBCWyhYomOpvErovufXLFZyXWdfPEbEpiz0v3f1LlDO22cSaYd3y
FFQP8sPfwtS40NabeOoIVMTRiL9OG/JGkCA1l8Q1+HLG/sGaPqFSjB6W4Tt6qcDn
ABEBAAGJATYEGAEIACAWIQSryfwUqslS53Z/0UpItw5yS6/gowUCXfeqGgIbDAAK
CRBItw5yS6/go1adB/4+EpKqokTNw8bOPAdOP1SnumtFMBXihGqpfpcjWG48vgHK
00CB3ocUxLheo7j/35SzUnfK9RV1+yKi8QWs/4REBWAyxyc47AaoHiEofn/muwXG
m48O5ivHMO3I7xjwptDxBr7awfTAAHIE7Zf84dkadHm6mw0Ik+2fX+/0bXpZYxgH
aREL2CblAWr23DLrcr0qgtxB07mi/HCvy7AFUFFhJl+Xgeb4xOcEPnI49RcwRoMh
5nhvWFChKezomuu8JXCGBjh1r4+RhFjkDj67O9hNmbi1KsQ7wAJJdgpkcqyC+OiL
JIj37JtjJsDKkzDyPZjqDZAiKCfy6gbt+8CPtCZd
=8nX5
mQENBF33qhoBCACyr8Ga4LPHSleEOvuZYUfU3/xLH3tiV7IKM5Rsp6vPvSAmXjFzh8MR0M1t
T7daTgT4pgcbuiogq33Jf9bTQCao3zwd5YRc7ZsdxWOeZJQj0w8A4HWOnCi9bO6peB1sNzq6
dq/AXv38r5iVfV/vpNDlCwYmZxfNyj53qsApsthlMH3rgOrAnG1wk3a3bPNT/ubMbnsKg2Mi
3Z22E5LdYKCkzHLlqbWdt96vs9pM63FjP371TfvCWxTlQ7y3bN5h+MAeiFEYPsDoWbMXSIen
t7cyP1z0EpnaaIwK3y1EbGq/CYj9N2sqHexFP0f/2lRoZqxNG7oxm1Jas1ZGrmavk7j5ABEB
AAG0IlNQU1AgU0ZUUCA8c3BzcC1zdXBwb3J0QHNpYi5zd2lzcz6JAU4EEwEIADgWIQSryfwU
qslS53Z/0UpItw5yS6/gowUCXfeqGgIbAwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBI
tw5yS6/go//rB/9g0fTMePnq+moVYZBNnOJf1MiEgX+nvJSlsdwZXMNv9873ZHlKnD1axY4y
FYgRg8sqExp+Q4Ea265pUF6Z2UI4ir7WP8+TBz1UMv9O5HMEXmSuWCk96SYQdEqDV6Nx+cY0
8uOm9NDIaAkm83mtwXAmPty5xJVyzImjunVQQ7VFHg6Y9Sf/sDQQX0c6wx9QKXubK06tN6JL
WmY0QwxTDHCc2t5Ujtx8rLAzaBJP+9zsKYWX9b/fD//3fFK2aQyJh1febvtx5GFgkW76cuf/
wPObKoo1k+4Z7JDqSnct/pj7qytMC1g6tVGLJATv6qt3s9jPdGmYwVR0WcNt9ETQNjN+iQIz
BBMBCAAdFiEEs3zioQHr+nCUHfiFiBaFte4Py9MFAl7fQ2sACgkQiBaFte4Py9PpSQ//StHr
FYTHAwq3GChXkbDyF0CGAEBE6kq4Klhr9OyTFlmoZhxpy2yu1gg1LmW5RH7CtEJSI22Y9VWW
06D/8fjSMEW4gRuWfMQuF8ZRG/yqA2ZkElmhW+/7agABxBrGu5V4PRDqPO0QN75Z/6x/fc05
+kjQatE7b6k6CrD34cOJscv/9isQSrP1QrCivSkjVk8jnLZN48W9qzh3/KNxfNCFLYyslkd5
lLyiNNMLzRVYa2zyawVcx9hV1u5dS8Kuox1XtWUI1azl1SRkmLO5wrB3g3WJLikStiadMWMQ
6Z7VAZJYAPbCsZUe26rWYjd77W/LPVqubfHk8PAFARaD7/wEcOzItBclX2tvSaQj/nq6YZu2
HBKFbQUI9waZ/gMMV84fjuHCqJNKs1jgDeaRnnkrvnHo/LkWX30oSJRXRTqvjlT2XabYS7Oy
vJRemunPg6XluoueklLDuoRTMYdj9TPFUK2cXRn29bNUpnmG0/3/HIdclXLsHpA3MxMxDGET
IxnlVHWZg9WNzeEjCcUc4iZioU2bDXMgMbLi+8PrN05hrbc9V/16mkYI526nbV9jwyp/gi4M
PoeHZ4A1r33uNFCo5HWb7x6HVrE+8oMItCyGIVX3QNH68ZGvveulgGhk0/JMA9NI3wHy4B5D
DqVdJW8omzk6TKT7ps3lj5hfvshgv1q5AQ0EXfeqGgEIAJbQL9VTEZk8TKmKXBfVPT2cONU/
psBVoOGpDNjYvlA+CDvQJIzzreAGcZXnCvXIux1/u35PVnt2KKj552DQgJL1VPKwnJ+O679w
DqZwHGEzdJGAq0/F+A5uO+322jL66VfFJ9ZZp3XG/WOeRU1MTXL8ibz4GRDfNGF2lM4sdP84
K2q9es2qU33WUvslZknd1GF5WQjQF3VjsS4VYZepUtPm15ThsJ4psEGUEJbKFiiY6m8Sui+5
9csVnJdZ188RsSmLPS/d/UuUM7bZxJph3fIUVA/yw9/C1LjQ1pt46ghUxNGIv04b8kaQIDWX
xDX4csb+wZo+oVKMHpbhO3qpwOcAEQEAAYkBNgQYAQgAIBYhBKvJ/BSqyVLndn/RSki3DnJL
r+CjBQJd96oaAhsMAAoJEEi3DnJLr+CjVp0H/j4SkqqiRM3Dxs48B04/VKe6a0UwFeKEaql+
lyNYbjy+AcrTQIHehxTEuF6juP/flLNSd8r1FXX7IqLxBaz/hEQFYDLHJzjsBqgeISh+f+a7
Bcabjw7mK8cw7cjvGPCm0PEGvtrB9MAAcgTtl/zh2Rp0ebqbDQiT7Z9f7/RtelljGAdpEQvY
JuUBavbcMutyvSqC3EHTuaL8cK/LsAVQUWEmX5eB5vjE5wQ+cjj1FzBGgyHmeG9YUKEp7Oia
67wlcIYGOHWvj5GEWOQOPrs72E2ZuLUqxDvAAkl2CmRyrIL46IskiPfsm2MmwMqTMPI9mOoN
kCIoJ/LqBu37wI+0Jl0=
=dkq+
-----END PGP PUBLIC KEY BLOCK-----
......@@ -44,25 +44,27 @@ Before using the script, you need to make sure that you create an SSH key pair f
Start by generating a key pair, make sure to replace `user` by your specific ID provided by the board of SPSP. Open a terminal and type:
```bash
ssh-keygen -t rsa -b 4096 -C user@spsp.sib.swiss #PLEASE REPLACE user WITH YOUR OWN ID
ssh-keygen -o -a 64 -t ed25519 -f ~/.ssh/id_ed25519 -C "user@spsp.sib.swiss" #PLEASE REPLACE user WITH YOUR OWN ID
```
You will be asked to `Enter file in which to save the key (/Users/user/.ssh/id_rsa):`, leave it by default by typing the return key.
You will be asked to `Enter file in which to save the key (/Users/user/.ssh/id_ed25519):`, leave it by default by typing the return key.
Then you will be asked to `Enter passphrase (empty for no passphrase):`, leave it empty and type two times the return key to confirm your choice.
You will then be prompted that your SSH public key has been saved to `/Users/user/.ssh/id_rsa.pub`. This is your public key that needs to be authorized on the SPSP SFTP Server.
You will then be prompted that your SSH public key has been saved to `/Users/user/.ssh/id_ed25519.pub`. This is your public key that needs to be authorized on the SPSP SFTP Server.
For the next step, you will need to upload your key. Start by copying your key. Type the following to display the public key:
```bash
cat /Users/user/.ssh/id_rsa.pub #PLEASE REPLACE user WITH YOUR LOCAL ACCOUNT
cat /Users/user/.ssh/id_ed25519.pub #PLEASE REPLACE user WITH YOUR LOCAL ACCOUNT
```
Then click [here](mailto:spsp-support@sib.swiss?subject=[SPSP-SFTP]Request%20Authorization) to send your key. Once the key has been validated, you will notified by mail.
## Installation
### Getting started
A step by step series of commands that tell you how to setup properly the transfer tool.
Start by cloning the transfer-tool on your local machine:
......@@ -80,6 +82,8 @@ ls -la
Your terminal should output 3 folders (logs,outbox,sent) and 2 files (README.md, spsp) and 2 hidden files (.env, .pub). Here is a short description of each folder and file:
- **viruses** - main repository where you should move your folder which contains your **viruses** fastq files and metadata file that you want to send
- **bacteria** - main repository where you should move your folder which contains your **bacteria** fastq files and metadata file that you want to send
- **sent** - contains encrypted files with their SHA256 hash that have been properly sent
- **.logs** - contains all the log files when you use the auto mode (log files record only errors)
- **.outbox** - contains files to be sent to the SPSP server through sftp
......@@ -87,6 +91,7 @@ Your terminal should output 3 folders (logs,outbox,sent) and 2 files (README.md,
- *spsp* - script containing all the commands to run, type `./spsp help` to display the commands
- *.env* - setting file to be configured by the user
- *.pub* - public key of SPSP for encryption
- *.gitlab* - some markdown files for GitLab templating
(.git and .gitignore are juste GIT related files)
......@@ -98,49 +103,18 @@ Let's start by setting up the transfer-tool. To do so, type:
This will make sure that some commands are available, that the script is executable and will also import the public key to your own list of keys.
### Sign the public key
At one point, you will be asked to sign the SPSP provided public key. This public key is used for [asymmetric encryption](https://en.wikipedia.org/wiki/Public-key_cryptography) of your data.
Before the transfer can be allowed to use it, you need check the fingerprint of the public key and manually sign the key (see below). If the public used for encryption is not exactly the one provided with the tranfer-tool, that means that we won't be able to decrypt the data correctly.
/!\ **TO SIGN THE PUBLIC KEY YOU NEED YOUR OWN PRIVATE KEY, TO DO SO PLEASE CHECK THIS [LINK](https://www.gnupg.org/gph/en/manual.html#AEN26) OR THIS [LINK](https://help.github.com/en/github/authenticating-to-github/generating-a-new-gpg-key)** /!\
```bash
pub rsa2048/48B70E724BAFE0A3
created: 2019-12-16 expires: never usage: SC
trust: full validity: unknown
Primary key fingerprint: ABC9 FC14 AAC9 52E7 767F D14A 48B7 0E72 4BAF E0A3
SPSP SFTP <spsp-support@sib.swiss>
How carefully have you verified the key you are about to sign actually belongs
to the person named above? If you don't know what to answer, enter "0".
### Verify the public key
(0) I will not answer. (default)
(1) I have not checked at all.
(2) I have done casual checking.
(3) I have done very careful checking.
At one point, the terminal should output the fingerprint of the imported key. Please make sure that the fingerprint corresponds to:
Your selection? (enter '?' for more information):
```
Make sure that the "Primary key fingerprint" corresponds to:
**ABC9 FC14 AAC9 52E7 767F D14A 48B7 0E72 4BAF E0A3**
If it doesn't, please [contact us](mailto:spsp-support@sib.swiss?subject=[SPSP-SFTP]Wrong%20Public%20Key) and send us the public key (.pub file in the directory).
Then, type **3** to validate your choice and then you should confirm the signature of the public key with your own private key:
```bash
Are you sure that you want to sign this key with your
key "John Doe <john.doe@example.com>" (################)
I have checked this key very carefully.
### Configure the .env file
Really sign? (y/N)
```
/!\ **THIS STEP IS EXTREMELY IMPORTANT, WITHOUT THE CORRECT SETUP, THE TRANSFER WILL FAIL** /!\
Type **y** to sign the key. Keep in mind that you need your own private key (previsouly [generated by yourself](https://www.gnupg.org/gph/en/manual.html#AEN26) to sign the SPSP public key.
Next, you need to configure the .env file to use the correct ID. Open the file with a text editor and change the line below:
......
......@@ -100,7 +100,6 @@ compressFolder() {
}
transferFile() {
if [ $# -eq 2 ]; then
for f in "$1"; do
printf "progress\nput $f" | sftp -b- "${SFTP_URL}$2" || exit 1
......@@ -124,7 +123,7 @@ signFile() {
encryptFile() {
message="Encrypting file ${COL_LGREEN}$1${COL_RESET}..."
printf "$message\n"
$GPGCMD --batch --yes -o $1.gpg -e -r $RECIPIENT $1
$GPGCMD --batch --yes --always-trust --sign --armor -o $1.gpg -e -r $RECIPIENT $1
message="File is encrypted as ${COL_LYELLOW}$1.gpg${COL_RESET}!"
printf "$message\n"
}
......@@ -188,14 +187,15 @@ initMode() {
chmod +x spsp
echo "SPSP: Importing SPSP public key for encryption protocol..."
$GPGCMD --import .pub
echo "SPSP: Verifying key... Please consult https://gitlab.sib.swiss/SPSP/transfer-tool#sign-the-public-key,"
echo "to check the fingerprint of the key (below) before validating anything. Once checked, please type '3'"
echo "and then 'y' to sign the key with your own key."
$GPGCMD --sign-key --ask-cert-level spsp-support@sib.swiss
echo "SPSP: Please consult: https://gitlab.sib.swiss/SPSP/transfer-tool, to make sure that the fingerprint below matches"
printf "Fingerprint: ${COL_LGREEN}"
$GPGCMD --with-colons --import-options show-only --import --fingerprint < .pub | awk -F: '$1 == "fpr" {print $10;}' | head -n 1
printf "${COL_RESET}"
# echo "SPSP: Verifying key... Please consult https://gitlab.sib.swiss/SPSP/transfer-tool#sign-the-public-key,"
# echo "to check the fingerprint of the key (below) before validating anything. Once checked, please type '3'"
# echo "and then 'y' to sign the key with your own key."
# $GPGCMD --sign-key --ask-cert-level spsp-support@sib.swiss
echo ""
echo "SPSP: Complete!"
echo "SPSP: You can now use SPSP transfer-tool"
echo "SPSP: List all the commands with:"
......@@ -411,18 +411,44 @@ generateFakeFiles() {
done
autoMode --no-archive
message="Removing the ${COL_LGREEN}remote fake files${COL_RESET} from SFTP..."
printf "$message\n"
echo "rm fake*" | sftp ${SFTP_URL}/viruses || exit 1
echo "rm fake*" | sftp ${SFTP_URL}/bacteria || exit 1
echo "Files has been properly removed!"
exit 0
}
# If we pass any arguments...
if [ $# -gt 0 ]; then
if [ -f .env ]; then
source .env
if [[ "$@" == *--without-env* ]] || [[ "$@" == *-without-env* ]]; then
echo "Ignoring the .env file"
echo "Using environment variables"
else
echo "No .env file found within current working directory ${pwd}"
echo "You need to create this file to use this script"
echo "Please refer to the documentation for further informations"
exit 2
if [ -f .env ]; then
source .env
else
echo "No .env file found within current working directory $(pwd)"
echo "Creating a new .env file..."
answer=false
while [ "$answer" = false ]; do
read -p "Please type laboratory/institution identifier: " identifier
read -p "Is $identifier correct?[y/n] " -n 1 -r
if [[ $REPLY =~ ^[Yy]$ ]]
then
echo "ID=$identifier" > .env
echo 'HOST=spsp-sftp.vital-it.ch' >> .env
echo 'SFTP_URL=${ID}@${HOST}:/data' >> .env
answer=true
else
echo ""
fi
done
source .env
fi
fi
if [ "$1" == "--version" ] || [ "$1" == "-v" ] || [ "$1" == "version" ]; then
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment