From b156067725275ab837c842320679fcd9fa2be9be Mon Sep 17 00:00:00 2001 From: Tobias Frust Date: Wed, 24 Jan 2018 08:42:43 +0100 Subject: [PATCH] api: add file browsing api endpoint --- invenio_uploadbyurl/api.py | 102 ++++++++++++++++++++++++++++-- invenio_uploadbyurl/config.py | 2 +- invenio_uploadbyurl/errors.py | 9 ++- invenio_uploadbyurl/links.py | 13 ---- tests/test_invenio_uploadbyurl.py | 6 -- tests/test_views.py | 96 ++++++++++++++++++++++++++++ 6 files changed, 200 insertions(+), 28 deletions(-) diff --git a/invenio_uploadbyurl/api.py b/invenio_uploadbyurl/api.py index 52cbd4a..9ee2259 100644 --- a/invenio_uploadbyurl/api.py +++ b/invenio_uploadbyurl/api.py @@ -29,8 +29,8 @@ from urllib.parse import urlparse import paramiko from celery.result import AsyncResult -from flask import Blueprint, abort, current_app, jsonify -from flask_login import current_user +from flask import Blueprint, abort, current_app, jsonify, url_for +from flask_login import current_user, login_required from invenio_files_rest.serializer import json_serializer from invenio_files_rest.views import need_bucket_permission, pass_bucket from invenio_rest import ContentNegotiatedMethodView @@ -40,8 +40,8 @@ from werkzeug.local import LocalProxy from .errors import AuthenticationError, FileDoesNotExist, FileTooLargeError, \ MissingPathError, MissingURLError, NoAbsolutePathError, NoFileError, \ - RemoteServerNotFoundError, SSHException, SSHKeyNotFoundError, \ - UnsupportedProtocolError + NoPathError, RemoteServerNotFoundError, SSHException, \ + SSHKeyNotFoundError, UnsupportedProtocolError from .models import RemoteServer, SSHKey from .tasks import download_files, download_via_sftp from .utils import create_key_from_bucket @@ -142,6 +142,79 @@ class UploadByUrl(ContentNegotiatedMethodView): raise MissingURLError() +class SFTPBrowserAPI(ContentNegotiatedMethodView): + """Browse files API via sftp.""" + + post_args = { + 'path': fields.Str( + location='query', + validate=path_validator, + missing='/', + ), + } + + @use_kwargs(post_args) + @login_required + def post(self, remote_server, path='/'): + """Return list of directories and files.""" + remote = RemoteServer.get_by_name(remote_server) + if not remote: + raise RemoteServerNotFoundError() + key = SSHKey.get(current_user.id, remote.id) + if not key: + raise SSHKeyNotFoundError() + pkey = current_app.config[ + 'UPLOADBYURL_KEY_DISPATCH_TABLE'][key.keytype].from_private_key( + StringIO(key.private_key)) + with paramiko.SSHClient() as client: + try: + client.load_system_host_keys() + client.connect( + key.remote_server.server_address, + username=key.username, + pkey=pkey, + timeout=current_app.config['UPLOADBYURL_SHORT_TIMEOUT'], + ) + sftp = client.open_sftp() + dirlist = [] + if path != '/': + endpoint = url_for( + 'invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path=os.path.split(path)[0], + _external=True, + ) + dirlist = [dict( + short_path='..', + path=os.path.split(path)[0], + isdir=True, + endpoint=endpoint, + )] + for element in sftp.listdir_iter(path): + endpoint = url_for( + 'invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path=os.path.join(path, element.filename), + _external=True, + ) + node = dict( + short_path=element.filename, + path=os.path.join(path, element.filename), + isdir=stat.S_ISDIR(element.st_mode), + endpoint=endpoint, + ) + dirlist.append(node) + except IOError: + raise NoPathError() + except paramiko.AuthenticationException: + raise AuthenticationError() + except (paramiko.SSHException, socket.error): + raise SSHException() + response = jsonify(dirlist) + response.status_code = 200 + return response + + class UploadViaSFTP(ContentNegotiatedMethodView): """Upload via sftp REST class.""" @@ -191,9 +264,12 @@ class UploadViaSFTP(ContentNegotiatedMethodView): with paramiko.SSHClient() as client: try: client.load_system_host_keys() - client.connect(key.remote_server.server_address, - username=key.username, - pkey=pkey) + client.connect( + key.remote_server.server_address, + username=key.username, + pkey=pkey, + timeout=current_app.config['UPLOADBYURL_SHORT_TIMEOUT'], + ) sftp = client.open_sftp() file_stat = sftp.stat(filepath) if stat.S_ISREG(file_stat.st_mode): @@ -226,6 +302,13 @@ upload_sftp_view = UploadViaSFTP.as_view( } ) +upload_sftp_browse_view = SFTPBrowserAPI.as_view( + 'uploadbyurl_api_sftp_browse', + serializers={ + 'application/json': json_serializer, + } +) + blueprint.add_url_rule( '/', view_func=upload_view, @@ -235,3 +318,8 @@ blueprint.add_url_rule( '//', view_func=upload_sftp_view, ) + +blueprint.add_url_rule( + '/browse/', + view_func=upload_sftp_browse_view, +) diff --git a/invenio_uploadbyurl/config.py b/invenio_uploadbyurl/config.py index 849b519..8b5a418 100644 --- a/invenio_uploadbyurl/config.py +++ b/invenio_uploadbyurl/config.py @@ -59,7 +59,7 @@ UPLOADBYURL_COMMENT = 'key@uploadbyurl.de' UPLOADBYURL_TIMEOUT = 60 * 60 # 1 hour """Timeout in s after which upload task should be quit.""" -UPLOADBYURL_SHORT_TIMEOUT = 30 +UPLOADBYURL_SHORT_TIMEOUT = 15 """Timeout in s after which short tasks, e.g. echo, should be quit.""" UPLOADBYURL_SENDER_EMAIL = 'rodare@hzdr.de' diff --git a/invenio_uploadbyurl/errors.py b/invenio_uploadbyurl/errors.py index 120db8a..c52d16a 100644 --- a/invenio_uploadbyurl/errors.py +++ b/invenio_uploadbyurl/errors.py @@ -111,5 +111,12 @@ class SSHException(RESTException): """SSH connection cannot be established.""" code = 400 - description = 'SSH connection cannot be established currently. Please ' \ + description = 'SSH connection cannot be established, currently. Please ' \ 'try again later.' + + +class NoPathError(RESTException): + """No path is given.""" + + code = 400 + description = 'Please provide a valid directory path.' diff --git a/invenio_uploadbyurl/links.py b/invenio_uploadbyurl/links.py index 13ebf4a..77e7afa 100644 --- a/invenio_uploadbyurl/links.py +++ b/invenio_uploadbyurl/links.py @@ -32,8 +32,6 @@ def default_uploadbyurl_link_factory(pid): record = Record.get_record(pid.get_assigned_object()) bucket = record.files.bucket - remote_servers = RemoteServer.all() - links = dict( uploadviaurl=url_for( 'invenio_uploadbyurl.uploadbyurl_api', @@ -42,17 +40,6 @@ def default_uploadbyurl_link_factory(pid): ), ) - for remote_server in remote_servers: - link = 'uploadviasftp_{0}'.format( - remote_server.name, - ) - links[link] = url_for( - 'invenio_uploadbyurl.uploadbyurl_api_sftp', - bucket_id=bucket.id, - remote_server=remote_server.name, - _external=True, - ) - return links except AttributeError: return None diff --git a/tests/test_invenio_uploadbyurl.py b/tests/test_invenio_uploadbyurl.py index fa362eb..946b332 100644 --- a/tests/test_invenio_uploadbyurl.py +++ b/tests/test_invenio_uploadbyurl.py @@ -71,12 +71,6 @@ def test_link_factory_with_bucket(app, db, bucket, remote): _external=True ) ) - links['uploadviasftp_foo'] = url_for( - 'invenio_uploadbyurl.uploadbyurl_api_sftp', - bucket_id=bucket.id, - remote_server='foo', - _external=True, - ) assert default_uploadbyurl_link_factory(pid) == links diff --git a/tests/test_views.py b/tests/test_views.py index a254e44..efc1bad 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -220,3 +220,99 @@ def test_sftp_post_fail(client, bucket, remote, user, user2): resp = client.post(url) assert resp.status_code == 400 assert b'You need to specify an absolute file path.' in resp.data + + +def test_sftp_browse(client, remote, user): + """Test get API call.""" + login_user(client, user) + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name) + resp = client.post(url) + assert resp.status_code == 200 + + node = dict( + endpoint=url_for( + 'invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + _external=True, + ), + path='/', + short_path='..', + isdir=True, + ) + assert node not in json.loads(resp.data.decode()) + + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path="/home/foo/upload") + resp = client.post(url) + assert resp.status_code == 200 + node = dict( + endpoint=url_for( + 'invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path='/home/foo', + _external=True, + ), + path='/home/foo', + short_path='..', + isdir=True, + ) + assert node in json.loads(resp.data.decode()) + + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path="/home/foo/upload") + # generate new SSH key without transferring it + key = SSHKey.delete(user_id=user.id, remote_server_id=remote.id) + prv, pub = generate_rsa_key() + key = SSHKey.create(private_key=prv, username='foo', + user=user, remote_server=remote) + db.session.commit() + resp = client.post(url) + assert resp.status_code == 400 + assert b'SSH Authentication failed. Probably the SSH Key is ' \ + b'not added to the remote server. Try to add it manually or ' \ + b'contact the support.' in resp.data + + +def test_sftp_browse_fail(client, remote, user): + """Test failing get API call.""" + login_user(client, user) + # path not absolute + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path="upload/README.rst") + resp = client.post(url) + assert resp.status_code == 400 + assert b'Please provide an absolute path ' \ + b'(e.g. /bigdata/rz/test.jpg).' in resp.data + + # unavailable remote server + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server='test1234', + path="/home/foo/upload") + resp = client.post(url) + assert resp.status_code == 404 + assert b'Remote Server not found.' in resp.data + + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path="/home/foo/upload/12345") + resp = client.post(url) + assert resp.status_code == 400 + assert b'Please provide a valid directory path.' in resp.data + + # not connected yet + SSHKey.delete(user.id, remote.id) + db.session.commit() + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name, + path="/home/foo/upload") + resp = client.post(url) + assert resp.status_code == 400 + assert b'Please connect your account with ' \ + b'the remote server first.' in resp.data + + url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse', + remote_server=remote.name) -- GitLab