Verified Commit d1e88233 authored by Huste, Tobias's avatar Huste, Tobias
Browse files

api: improve error messages and detect errors before job submission

parent 0534509a
Pipeline #2332 passed with stage
in 36 minutes and 56 seconds
......@@ -22,8 +22,12 @@
import json
import os
import socket
import stat
from io import StringIO
from urllib.parse import urlparse
import paramiko
from celery.result import AsyncResult
from flask import Blueprint, abort, current_app, jsonify
from flask_login import current_user
......@@ -34,8 +38,10 @@ from webargs import fields
from webargs.flaskparser import use_kwargs
from werkzeug.local import LocalProxy
from .errors import MissingPathError, MissingURLError, NoAbsolutePathError, \
RemoteServerNotFoundError, SSHKeyNotFoundError, UnsupportedProtocolError
from .errors import AuthenticationError, FileDoesNotExist, FileTooLargeError, \
MissingPathError, MissingURLError, NoAbsolutePathError, NoFileError, \
RemoteServerNotFoundError, SSHException, SSHKeyNotFoundError, \
UnsupportedProtocolError
from .models import RemoteServer, SSHKey
from .tasks import download_files, download_via_sftp
from .utils import create_key_from_bucket
......@@ -163,6 +169,7 @@ class UploadViaSFTP(ContentNegotiatedMethodView):
key = SSHKey.get(current_user.id, remote.id)
if not key:
raise SSHKeyNotFoundError()
self.verify_request(remote, path, bucket)
download_via_sftp.delay(
str(bucket.id), remote.id, current_user.id, path)
response = jsonify(
......@@ -173,6 +180,37 @@ class UploadViaSFTP(ContentNegotiatedMethodView):
return response
raise MissingPathError()
@staticmethod
def verify_request(remote_server, filepath, bucket):
"""Verify request and return descriptive error message."""
key = SSHKey.get(current_user.id, remote_server.id)
pkey = current_app.config[
'UPLOADBYURL_KEY_DISPATCH_TABLE'][key.keytype].from_private_key(
StringIO(key.private_key))
# get filesize and check if is file via paramiko
with paramiko.SSHClient() as client:
try:
client.load_system_host_keys()
client.connect(key.remote_server.server_address,
username=key.username,
pkey=pkey)
sftp = client.open_sftp()
file_stat = sftp.stat(filepath)
if stat.S_ISREG(file_stat.st_mode):
size = sftp.stat(filepath).st_size
else:
raise NoFileError()
# check file size limits
if (bucket.quota_left < size or
bucket.size_limit < size):
raise FileTooLargeError()
except IOError:
raise FileDoesNotExist()
except paramiko.AuthenticationException:
raise AuthenticationError()
except (paramiko.SSHException, socket.error):
raise SSHException()
upload_view = UploadByUrl.as_view(
'uploadbyurl_api',
......
......@@ -20,6 +20,7 @@
"""REST Errors for invenio-uploadbyurl."""
from flask import current_app
from invenio_rest.errors import RESTException
......@@ -67,3 +68,48 @@ class UnsupportedProtocolError(RESTException):
description = 'Unsupported protocol. Use one of {0}'.format(
str(allowed_protocols)
)
class FileTooLargeError(RESTException):
"""Give file exceeds file size limits."""
code = 400
description = 'The given file is too large. The default limit for a ' \
'dataset per record is {0} GiB. The default maximum file size is ' \
'{1} GiB.'.format(
100,
50,
)
class NoFileError(RESTException):
"""A valid regular file path must be given."""
code = 400
description = 'Please provide an absolute file path. Upload of ' \
'directories is not supported.'
class FileDoesNotExist(RESTException):
"""Given file does not exist."""
code = 400
description = 'The given file does not exist or you do not have ' \
'read permissions.'
class AuthenticationError(RESTException):
"""SSH Authentication failes."""
code = 400
description = 'SSH Authentication failed. Probably the SSH Key is ' \
'not added to the remote server. Try to add it manually or ' \
'contact the support.'
class SSHException(RESTException):
"""SSH connection cannot be established."""
code = 400
description = 'SSH connection cannot be established currently. Please ' \
'try again later.'
......@@ -25,6 +25,7 @@ import os
import uuid
import mock
import pytest
from flask import Flask, url_for
from invenio_accounts.testutils import create_test_user
from invenio_db import db
......@@ -32,8 +33,9 @@ from invenio_files_rest.models import Bucket
from testutils import MockAsyncResult, login_user
from invenio_uploadbyurl.api import _redisstore
from invenio_uploadbyurl.errors import FileDoesNotExist
from invenio_uploadbyurl.models import SSHKey
from invenio_uploadbyurl.utils import create_key_from_bucket
from invenio_uploadbyurl.utils import create_key_from_bucket, generate_rsa_key
def test_proxy(app):
......@@ -113,7 +115,7 @@ def test_post_api(client, bucket, user):
assert resp.status_code == 400
def test_sftp_post(client, bucket, remote, user, user2):
def test_sftp_post(client, bucket, remote, user, user2, db):
"""Test API endpoint for sftp download."""
login_user(client, user)
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp',
......@@ -124,6 +126,51 @@ def test_sftp_post(client, bucket, remote, user, user2):
assert resp.status_code == 202
assert bucket.size == os.path.getsize('README.rst')
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp',
bucket_id=bucket.id,
remote_server=remote.name,
path="/home/foo/upload/README123.rst")
resp = client.post(url)
assert resp.status_code == 400
assert b'The given file does not exist or you do not have ' \
b'read permissions.' in resp.data
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp',
bucket_id=bucket.id,
remote_server=remote.name,
path="/home/foo/upload")
resp = client.post(url)
assert resp.status_code == 400
assert b'Please provide an absolute file path. Upload of ' \
b'directories is not supported.' in resp.data
# set max file size of bucket
bucket.quota_size = 1500
db.session.add(bucket)
db.session.commit()
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp',
bucket_id=bucket.id,
remote_server=remote.name,
path="/home/foo/upload/README.rst")
resp = client.post(url)
assert resp.status_code == 400
assert b'The given file is too large. The default limit for a ' \
b'dataset per record is 100 GiB. The default maximum file size is ' \
b'50 GiB.' in resp.data
# generate new SSH key without transferring it
key = SSHKey.delete(user_id=user.id, remote_server_id=remote.id)
prv, pub = generate_rsa_key()
key = SSHKey.create(private_key=prv, username='foo',
user=user, remote_server=remote)
db.session.commit()
resp = client.post(url)
assert resp.status_code == 400
assert b'SSH Authentication failed. Probably the SSH Key is ' \
b'not added to the remote server. Try to add it manually or ' \
b'contact the support.' in resp.data
# no bucket permissions
login_user(client, user2[1])
resp = client.post(url)
......
......@@ -20,8 +20,13 @@
"""Test utilities."""
from io import StringIO
import paramiko
from flask import current_app
from invenio_uploadbyurl.models import SSHKey
def login_user(client, user):
"""Log in a specified user."""
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment