From d1e8823385c7b7d84da85f70c782c5b3878c7be7 Mon Sep 17 00:00:00 2001 From: Tobias Frust Date: Mon, 22 Jan 2018 14:45:12 +0100 Subject: [PATCH] api: improve error messages and detect errors before job submission --- invenio_uploadbyurl/api.py | 42 +++++++++++++++++++++++++++-- invenio_uploadbyurl/errors.py | 46 +++++++++++++++++++++++++++++++ tests/test_views.py | 51 +++++++++++++++++++++++++++++++++-- tests/testutils.py | 5 ++++ 4 files changed, 140 insertions(+), 4 deletions(-) diff --git a/invenio_uploadbyurl/api.py b/invenio_uploadbyurl/api.py index 7a6f1e5..52cbd4a 100644 --- a/invenio_uploadbyurl/api.py +++ b/invenio_uploadbyurl/api.py @@ -22,8 +22,12 @@ import json import os +import socket +import stat +from io import StringIO from urllib.parse import urlparse +import paramiko from celery.result import AsyncResult from flask import Blueprint, abort, current_app, jsonify from flask_login import current_user @@ -34,8 +38,10 @@ from webargs import fields from webargs.flaskparser import use_kwargs from werkzeug.local import LocalProxy -from .errors import MissingPathError, MissingURLError, NoAbsolutePathError, \ - RemoteServerNotFoundError, SSHKeyNotFoundError, UnsupportedProtocolError +from .errors import AuthenticationError, FileDoesNotExist, FileTooLargeError, \ + MissingPathError, MissingURLError, NoAbsolutePathError, NoFileError, \ + RemoteServerNotFoundError, SSHException, SSHKeyNotFoundError, \ + UnsupportedProtocolError from .models import RemoteServer, SSHKey from .tasks import download_files, download_via_sftp from .utils import create_key_from_bucket @@ -163,6 +169,7 @@ class UploadViaSFTP(ContentNegotiatedMethodView): key = SSHKey.get(current_user.id, remote.id) if not key: raise SSHKeyNotFoundError() + self.verify_request(remote, path, bucket) download_via_sftp.delay( str(bucket.id), remote.id, current_user.id, path) response = jsonify( @@ -173,6 +180,37 @@ class UploadViaSFTP(ContentNegotiatedMethodView): return response raise MissingPathError() + @staticmethod + def verify_request(remote_server, filepath, bucket): + """Verify request and return descriptive error message.""" + key = SSHKey.get(current_user.id, remote_server.id) + pkey = current_app.config[ + 'UPLOADBYURL_KEY_DISPATCH_TABLE'][key.keytype].from_private_key( + StringIO(key.private_key)) + # get filesize and check if is file via paramiko + with paramiko.SSHClient() as client: + try: + client.load_system_host_keys() + client.connect(key.remote_server.server_address, + username=key.username, + pkey=pkey) + sftp = client.open_sftp() + file_stat = sftp.stat(filepath) + if stat.S_ISREG(file_stat.st_mode): + size = sftp.stat(filepath).st_size + else: + raise NoFileError() + # check file size limits + if (bucket.quota_left < size or + bucket.size_limit < size): + raise FileTooLargeError() + except IOError: + raise FileDoesNotExist() + except paramiko.AuthenticationException: + raise AuthenticationError() + except (paramiko.SSHException, socket.error): + raise SSHException() + upload_view = UploadByUrl.as_view( 'uploadbyurl_api', diff --git a/invenio_uploadbyurl/errors.py b/invenio_uploadbyurl/errors.py index fdec3ac..120db8a 100644 --- a/invenio_uploadbyurl/errors.py +++ b/invenio_uploadbyurl/errors.py @@ -20,6 +20,7 @@ """REST Errors for invenio-uploadbyurl.""" +from flask import current_app from invenio_rest.errors import RESTException @@ -67,3 +68,48 @@ class UnsupportedProtocolError(RESTException): description = 'Unsupported protocol. Use one of {0}'.format( str(allowed_protocols) ) + + +class FileTooLargeError(RESTException): + """Give file exceeds file size limits.""" + + code = 400 + description = 'The given file is too large. The default limit for a ' \ + 'dataset per record is {0} GiB. The default maximum file size is ' \ + '{1} GiB.'.format( + 100, + 50, + ) + + +class NoFileError(RESTException): + """A valid regular file path must be given.""" + + code = 400 + description = 'Please provide an absolute file path. Upload of ' \ + 'directories is not supported.' + + +class FileDoesNotExist(RESTException): + """Given file does not exist.""" + + code = 400 + description = 'The given file does not exist or you do not have ' \ + 'read permissions.' + + +class AuthenticationError(RESTException): + """SSH Authentication failes.""" + + code = 400 + description = 'SSH Authentication failed. Probably the SSH Key is ' \ + 'not added to the remote server. Try to add it manually or ' \ + 'contact the support.' + + +class SSHException(RESTException): + """SSH connection cannot be established.""" + + code = 400 + description = 'SSH connection cannot be established currently. Please ' \ + 'try again later.' diff --git a/tests/test_views.py b/tests/test_views.py index 9792835..a254e44 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -25,6 +25,7 @@ import os import uuid import mock +import pytest from flask import Flask, url_for from invenio_accounts.testutils import create_test_user from invenio_db import db @@ -32,8 +33,9 @@ from invenio_files_rest.models import Bucket from testutils import MockAsyncResult, login_user from invenio_uploadbyurl.api import _redisstore +from invenio_uploadbyurl.errors import FileDoesNotExist from invenio_uploadbyurl.models import SSHKey -from invenio_uploadbyurl.utils import create_key_from_bucket +from invenio_uploadbyurl.utils import create_key_from_bucket, generate_rsa_key def test_proxy(app): @@ -113,7 +115,7 @@ def test_post_api(client, bucket, user): assert resp.status_code == 400 -def test_sftp_post(client, bucket, remote, user, user2): +def test_sftp_post(client, bucket, remote, user, user2, db): """Test API endpoint for sftp download.""" login_user(client, user) url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp', @@ -124,6 +126,51 @@ def test_sftp_post(client, bucket, remote, user, user2): assert resp.status_code == 202 assert bucket.size == os.path.getsize('README.rst') + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp', + bucket_id=bucket.id, + remote_server=remote.name, + path="/home/foo/upload/README123.rst") + resp = client.post(url) + assert resp.status_code == 400 + assert b'The given file does not exist or you do not have ' \ + b'read permissions.' in resp.data + + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp', + bucket_id=bucket.id, + remote_server=remote.name, + path="/home/foo/upload") + resp = client.post(url) + assert resp.status_code == 400 + assert b'Please provide an absolute file path. Upload of ' \ + b'directories is not supported.' in resp.data + + # set max file size of bucket + bucket.quota_size = 1500 + db.session.add(bucket) + db.session.commit() + + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp', + bucket_id=bucket.id, + remote_server=remote.name, + path="/home/foo/upload/README.rst") + resp = client.post(url) + assert resp.status_code == 400 + assert b'The given file is too large. The default limit for a ' \ + b'dataset per record is 100 GiB. The default maximum file size is ' \ + b'50 GiB.' in resp.data + + # generate new SSH key without transferring it + key = SSHKey.delete(user_id=user.id, remote_server_id=remote.id) + prv, pub = generate_rsa_key() + key = SSHKey.create(private_key=prv, username='foo', + user=user, remote_server=remote) + db.session.commit() + resp = client.post(url) + assert resp.status_code == 400 + assert b'SSH Authentication failed. Probably the SSH Key is ' \ + b'not added to the remote server. Try to add it manually or ' \ + b'contact the support.' in resp.data + # no bucket permissions login_user(client, user2[1]) resp = client.post(url) diff --git a/tests/testutils.py b/tests/testutils.py index 3e311e5..e51b1e9 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -20,8 +20,13 @@ """Test utilities.""" +from io import StringIO + +import paramiko from flask import current_app +from invenio_uploadbyurl.models import SSHKey + def login_user(client, user): """Log in a specified user.""" -- GitLab