Verified Commit b1560677 authored by Huste, Tobias's avatar Huste, Tobias
Browse files

api: add file browsing api endpoint

parent d7d0421c
Pipeline #2384 passed with stage
in 23 minutes and 59 seconds
......@@ -29,8 +29,8 @@ from urllib.parse import urlparse
import paramiko
from celery.result import AsyncResult
from flask import Blueprint, abort, current_app, jsonify
from flask_login import current_user
from flask import Blueprint, abort, current_app, jsonify, url_for
from flask_login import current_user, login_required
from invenio_files_rest.serializer import json_serializer
from invenio_files_rest.views import need_bucket_permission, pass_bucket
from invenio_rest import ContentNegotiatedMethodView
......@@ -40,8 +40,8 @@ from werkzeug.local import LocalProxy
from .errors import AuthenticationError, FileDoesNotExist, FileTooLargeError, \
MissingPathError, MissingURLError, NoAbsolutePathError, NoFileError, \
RemoteServerNotFoundError, SSHException, SSHKeyNotFoundError, \
UnsupportedProtocolError
NoPathError, RemoteServerNotFoundError, SSHException, \
SSHKeyNotFoundError, UnsupportedProtocolError
from .models import RemoteServer, SSHKey
from .tasks import download_files, download_via_sftp
from .utils import create_key_from_bucket
......@@ -142,6 +142,79 @@ class UploadByUrl(ContentNegotiatedMethodView):
raise MissingURLError()
class SFTPBrowserAPI(ContentNegotiatedMethodView):
"""Browse files API via sftp."""
post_args = {
'path': fields.Str(
location='query',
validate=path_validator,
missing='/',
),
}
@use_kwargs(post_args)
@login_required
def post(self, remote_server, path='/'):
"""Return list of directories and files."""
remote = RemoteServer.get_by_name(remote_server)
if not remote:
raise RemoteServerNotFoundError()
key = SSHKey.get(current_user.id, remote.id)
if not key:
raise SSHKeyNotFoundError()
pkey = current_app.config[
'UPLOADBYURL_KEY_DISPATCH_TABLE'][key.keytype].from_private_key(
StringIO(key.private_key))
with paramiko.SSHClient() as client:
try:
client.load_system_host_keys()
client.connect(
key.remote_server.server_address,
username=key.username,
pkey=pkey,
timeout=current_app.config['UPLOADBYURL_SHORT_TIMEOUT'],
)
sftp = client.open_sftp()
dirlist = []
if path != '/':
endpoint = url_for(
'invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path=os.path.split(path)[0],
_external=True,
)
dirlist = [dict(
short_path='..',
path=os.path.split(path)[0],
isdir=True,
endpoint=endpoint,
)]
for element in sftp.listdir_iter(path):
endpoint = url_for(
'invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path=os.path.join(path, element.filename),
_external=True,
)
node = dict(
short_path=element.filename,
path=os.path.join(path, element.filename),
isdir=stat.S_ISDIR(element.st_mode),
endpoint=endpoint,
)
dirlist.append(node)
except IOError:
raise NoPathError()
except paramiko.AuthenticationException:
raise AuthenticationError()
except (paramiko.SSHException, socket.error):
raise SSHException()
response = jsonify(dirlist)
response.status_code = 200
return response
class UploadViaSFTP(ContentNegotiatedMethodView):
"""Upload via sftp REST class."""
......@@ -191,9 +264,12 @@ class UploadViaSFTP(ContentNegotiatedMethodView):
with paramiko.SSHClient() as client:
try:
client.load_system_host_keys()
client.connect(key.remote_server.server_address,
username=key.username,
pkey=pkey)
client.connect(
key.remote_server.server_address,
username=key.username,
pkey=pkey,
timeout=current_app.config['UPLOADBYURL_SHORT_TIMEOUT'],
)
sftp = client.open_sftp()
file_stat = sftp.stat(filepath)
if stat.S_ISREG(file_stat.st_mode):
......@@ -226,6 +302,13 @@ upload_sftp_view = UploadViaSFTP.as_view(
}
)
upload_sftp_browse_view = SFTPBrowserAPI.as_view(
'uploadbyurl_api_sftp_browse',
serializers={
'application/json': json_serializer,
}
)
blueprint.add_url_rule(
'/<string:bucket_id>',
view_func=upload_view,
......@@ -235,3 +318,8 @@ blueprint.add_url_rule(
'/<string:bucket_id>/<string:remote_server>',
view_func=upload_sftp_view,
)
blueprint.add_url_rule(
'/browse/<string:remote_server>',
view_func=upload_sftp_browse_view,
)
......@@ -59,7 +59,7 @@ UPLOADBYURL_COMMENT = 'key@uploadbyurl.de'
UPLOADBYURL_TIMEOUT = 60 * 60 # 1 hour
"""Timeout in s after which upload task should be quit."""
UPLOADBYURL_SHORT_TIMEOUT = 30
UPLOADBYURL_SHORT_TIMEOUT = 15
"""Timeout in s after which short tasks, e.g. echo, should be quit."""
UPLOADBYURL_SENDER_EMAIL = 'rodare@hzdr.de'
......
......@@ -111,5 +111,12 @@ class SSHException(RESTException):
"""SSH connection cannot be established."""
code = 400
description = 'SSH connection cannot be established currently. Please ' \
description = 'SSH connection cannot be established, currently. Please ' \
'try again later.'
class NoPathError(RESTException):
"""No path is given."""
code = 400
description = 'Please provide a valid directory path.'
......@@ -32,8 +32,6 @@ def default_uploadbyurl_link_factory(pid):
record = Record.get_record(pid.get_assigned_object())
bucket = record.files.bucket
remote_servers = RemoteServer.all()
links = dict(
uploadviaurl=url_for(
'invenio_uploadbyurl.uploadbyurl_api',
......@@ -42,17 +40,6 @@ def default_uploadbyurl_link_factory(pid):
),
)
for remote_server in remote_servers:
link = 'uploadviasftp_{0}'.format(
remote_server.name,
)
links[link] = url_for(
'invenio_uploadbyurl.uploadbyurl_api_sftp',
bucket_id=bucket.id,
remote_server=remote_server.name,
_external=True,
)
return links
except AttributeError:
return None
......@@ -71,12 +71,6 @@ def test_link_factory_with_bucket(app, db, bucket, remote):
_external=True
)
)
links['uploadviasftp_foo'] = url_for(
'invenio_uploadbyurl.uploadbyurl_api_sftp',
bucket_id=bucket.id,
remote_server='foo',
_external=True,
)
assert default_uploadbyurl_link_factory(pid) == links
......
......@@ -220,3 +220,99 @@ def test_sftp_post_fail(client, bucket, remote, user, user2):
resp = client.post(url)
assert resp.status_code == 400
assert b'You need to specify an absolute file path.' in resp.data
def test_sftp_browse(client, remote, user):
"""Test get API call."""
login_user(client, user)
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name)
resp = client.post(url)
assert resp.status_code == 200
node = dict(
endpoint=url_for(
'invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
_external=True,
),
path='/',
short_path='..',
isdir=True,
)
assert node not in json.loads(resp.data.decode())
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path="/home/foo/upload")
resp = client.post(url)
assert resp.status_code == 200
node = dict(
endpoint=url_for(
'invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path='/home/foo',
_external=True,
),
path='/home/foo',
short_path='..',
isdir=True,
)
assert node in json.loads(resp.data.decode())
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path="/home/foo/upload")
# generate new SSH key without transferring it
key = SSHKey.delete(user_id=user.id, remote_server_id=remote.id)
prv, pub = generate_rsa_key()
key = SSHKey.create(private_key=prv, username='foo',
user=user, remote_server=remote)
db.session.commit()
resp = client.post(url)
assert resp.status_code == 400
assert b'SSH Authentication failed. Probably the SSH Key is ' \
b'not added to the remote server. Try to add it manually or ' \
b'contact the support.' in resp.data
def test_sftp_browse_fail(client, remote, user):
"""Test failing get API call."""
login_user(client, user)
# path not absolute
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path="upload/README.rst")
resp = client.post(url)
assert resp.status_code == 400
assert b'Please provide an absolute path ' \
b'(e.g. /bigdata/rz/test.jpg).' in resp.data
# unavailable remote server
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server='test1234',
path="/home/foo/upload")
resp = client.post(url)
assert resp.status_code == 404
assert b'Remote Server not found.' in resp.data
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path="/home/foo/upload/12345")
resp = client.post(url)
assert resp.status_code == 400
assert b'Please provide a valid directory path.' in resp.data
# not connected yet
SSHKey.delete(user.id, remote.id)
db.session.commit()
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name,
path="/home/foo/upload")
resp = client.post(url)
assert resp.status_code == 400
assert b'Please connect your account with ' \
b'the remote server first.' in resp.data
url = url_for('invenio_uploadbyurl.uploadbyurl_api_sftp_browse',
remote_server=remote.name)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment