Commit 717010f2 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

It is now possible to auth with token or credentials.

parent 94636db1
Pipeline #1784 passed with stage
in 2 minutes and 29 seconds
...@@ -25,11 +25,11 @@ def prepare_session(): ...@@ -25,11 +25,11 @@ def prepare_session():
@validate_schema(create_data_schema) @validate_schema(create_data_schema)
def create_data(): def create_data():
wait = request.args.get('wait') == '1' wait = request.args.get('wait') == '1'
json = request.get_json(force=True) # pattern enforced by decorators payload = request.get_json(force=True) # pattern enforced by decorators
etlhandler = ETLHandler.factory(handler=json['handler'], etl_handler = ETLHandler.factory(handler=payload['handler'],
server=json['server'], server=payload['server'],
token=json['token']) auth=payload['auth'])
data_ids = etlhandler.handle(descriptors=json['descriptors'], wait=wait) data_ids = etl_handler.handle(descriptors=payload['descriptors'], wait=wait)
session['data_ids'] += data_ids session['data_ids'] += data_ids
return jsonify({'data_ids': data_ids}), 201 return jsonify({'data_ids': data_ids}), 201
......
...@@ -28,9 +28,25 @@ class ETLHandler(metaclass=abc.ABCMeta): ...@@ -28,9 +28,25 @@ class ETLHandler(metaclass=abc.ABCMeta):
""" """
pass pass
def __init__(self, server, token): @abc.abstractmethod
def _get_token_for_credentials(self, server: str,
user: str, passwd: str) -> str:
""" Authenticate with the server and return a token.
:param server: The server to authenticate with.
:param user: The user id.
:param passwd: The password.
"""
pass
def __init__(self, server, auth):
self._server = server self._server = server
self._token = token # if no token is given we have to get one
try:
self._token = auth['token']
except KeyError:
self._token = self._get_token_for_credentials(
server, auth['user'], auth['passwd'])
@staticmethod @staticmethod
def compute_data_id(server: str, descriptor: dict) -> str: def compute_data_id(server: str, descriptor: dict) -> str:
...@@ -47,9 +63,9 @@ class ETLHandler(metaclass=abc.ABCMeta): ...@@ -47,9 +63,9 @@ class ETLHandler(metaclass=abc.ABCMeta):
return hash_key return hash_key
def handle(self, descriptors: List[dict], wait: bool = False) -> List[str]: def handle(self, descriptors: List[dict], wait: bool = False) -> List[str]:
"""Create instances of ETL for the given descriptors and submit them (ETL """Create instances of ETL for the given descriptors and submit them
implements celery.Task) to the broker. The task ids are returned to keep (ETL implements celery.Task) to the broker. The task ids are returned to
track of them. keep track of them.
:param descriptors: A list of items describing the data to download. :param descriptors: A list of items describing the data to download.
:param wait: Makes this method synchronous by waiting for the tasks to :param wait: Makes this method synchronous by waiting for the tasks to
...@@ -86,20 +102,20 @@ class ETLHandler(metaclass=abc.ABCMeta): ...@@ -86,20 +102,20 @@ class ETLHandler(metaclass=abc.ABCMeta):
return data_ids return data_ids
@classmethod @classmethod
def factory(cls, handler: str, server: str, token: str) -> 'ETLHandler': def factory(cls, handler: str, server: str, auth: dict) -> 'ETLHandler':
"""Return an instance of the implementation of ETLHandler that can """Return an instance of the implementation of ETLHandler that can
handle the given parameters. handle the given parameters.
:param handler: Describes the handler. E.g.: transmart, ada :param handler: Describes the handler. E.g.: transmart, ada
:param server: The server to download data from. :param server: The server to download data from.
:param token: The token used for authentication. :param auth: Contains credentials to authenticate with the API.
:return: An instance of an implementation of ETLHandler that returns :return: An instance of an implementation of ETLHandler that returns
True for self.can_handle True for self.can_handle
""" """
from . import HANDLER_REGISTRY from . import HANDLER_REGISTRY
for Handler in HANDLER_REGISTRY: for Handler in HANDLER_REGISTRY:
if Handler.can_handle(handler): if Handler.can_handle(handler):
return Handler(server, token) return Handler(server, auth)
raise NotImplementedError( raise NotImplementedError(
"No ETLHandler implementation found for: '{}'".format(handler)) "No ETLHandler implementation found for: '{}'".format(handler))
......
"""Default ETL to get data from ADA"""
from pandas import DataFrame
from fractalis.data.etl import ETL
class DefaultETL(ETL):
name = 'ada_default_etl'
_handler = 'ada'
_data_type = 'default'
def extract(self, server: str, token: str, descriptor: dict) -> object:
pass
def transform(self, raw_data: object) -> DataFrame:
pass
\ No newline at end of file
...@@ -6,4 +6,8 @@ class AdaHandler(ETLHandler): ...@@ -6,4 +6,8 @@ class AdaHandler(ETLHandler):
_handler = 'ada' _handler = 'ada'
def _heartbeat(self): def _heartbeat(self):
pass
def _get_token_for_credentials(self, server: str,
user: str, passwd: str) -> str:
pass pass
\ No newline at end of file
...@@ -5,5 +5,9 @@ class TestHandler(ETLHandler): ...@@ -5,5 +5,9 @@ class TestHandler(ETLHandler):
_handler = 'test' _handler = 'test'
def _heartbeat(): def _heartbeat(self):
pass
def _get_token_for_credentials(self, server: str,
user: str, passwd: str) -> str:
pass pass
from fractalis.data.etlhandler import ETLHandler
class TransmartHandler(ETLHandler):
_handler = 'transmart'
def heartbeat():
pass
...@@ -3,7 +3,15 @@ create_data_schema = { ...@@ -3,7 +3,15 @@ create_data_schema = {
"properties": { "properties": {
"handler": {"type": "string"}, "handler": {"type": "string"},
"server": {"type": "string"}, "server": {"type": "string"},
"token": {"type": "string"}, "auth": {
"type": "object",
"properties": {
"token": {"type": "string"},
"user": {"type": "string"},
"passwd": {"type": "string"}
},
"minProperties": 1
},
"descriptors": { "descriptors": {
"type": "array", "type": "array",
"items": { "items": {
...@@ -16,5 +24,5 @@ create_data_schema = { ...@@ -16,5 +24,5 @@ create_data_schema = {
} }
} }
}, },
"required": ["handler", "server", "token", "descriptors"] "required": ["handler", "server", "auth", "descriptors"]
} }
...@@ -12,7 +12,8 @@ setup( ...@@ -12,7 +12,8 @@ setup(
'celery[redis]', 'celery[redis]',
'redis', 'redis',
'pandas', 'pandas',
'numpy' 'numpy',
'requests'
], ],
setup_requires=[ setup_requires=[
'pytest-runner', 'pytest-runner',
......
...@@ -31,7 +31,7 @@ class TestAnalytics: ...@@ -31,7 +31,7 @@ class TestAnalytics:
'/data', data=flask.json.dumps(dict( '/data', data=flask.json.dumps(dict(
handler='test', handler='test',
server='localhost:1234', server='localhost:1234',
token='7746391376142672192764', auth={'token': '7746391376142672192764'},
descriptors=[ descriptors=[
{ {
'data_type': 'randomdf', 'data_type': 'randomdf',
......
...@@ -29,7 +29,7 @@ class TestData: ...@@ -29,7 +29,7 @@ class TestData:
'/data', data=flask.json.dumps(dict( '/data', data=flask.json.dumps(dict(
handler='test', handler='test',
server='localhost:1234', server='localhost:1234',
token='7746391376142672192764', auth={'token': '7746391376142672192764'},
descriptors=[ descriptors=[
{ {
'data_type': 'foo', 'data_type': 'foo',
...@@ -44,7 +44,7 @@ class TestData: ...@@ -44,7 +44,7 @@ class TestData:
'/data', data=flask.json.dumps(dict( '/data', data=flask.json.dumps(dict(
handler='test', handler='test',
server='localhost:1234', server='localhost:1234',
token='7746391376142672192764', auth={'token': '7746391376142672192764'},
descriptors=[ descriptors=[
{ {
'data_type': 'foo', 'data_type': 'foo',
...@@ -65,63 +65,69 @@ class TestData: ...@@ -65,63 +65,69 @@ class TestData:
{ {
'handler': '', 'handler': '',
'server': 'localhost', 'server': 'localhost',
'token': '1234567890', 'auth': '{"''tok"n'"" '12345678"90',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]' 'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': '', 'server': '',
'token': '1234567890', 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]' 'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': 'localhost', 'server': 'localhost',
'token': '', 'auth': '',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]' 'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': 'localhost', 'server': 'localhost',
'token': '1234567890', 'auth': '{"token": "1234567890"}',
'descriptors': '' 'descriptors': ''
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': 'localhost', 'server': 'localhost',
'token': '1234567890', 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]' 'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': 'localhost', 'server': 'localhost',
'token': '1234567890', 'auth': '{"token": "1234567890"}',
'descriptors': '[{"concept": "GSE1234"}]' 'descriptors': '[{"concept": "GSE1234"}]'
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': 'localhost', 'server': 'localhost',
'token': '1234567890', 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "foo"}]' 'descriptors': '[{"data_type": "foo"}]'
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': 'localhost', 'server': 'localhost',
'token': '1234567890', 'auth': '{"token": "1234567890"}',
'descriptors': '[{"data_type": "", "concept": "GSE1234"}]' 'descriptors': '[{"data_type": "", "concept": "GSE1234"}]'
}, },
{ {
'handler': 'test', 'handler': 'test',
'server': 'localhost', 'server': 'localhost',
'token': '1234567890', 'auth': '{"token": "234567890"}',
'descriptors': '[]' 'descriptors': '[]'
},
{
'handler': 'test',
'server': 'localhost',
'auth': '{}',
'descriptors': '[{"data_type": "foo", "concept": "GSE1234"}]'
} }
]) ])
def bad_post(self, test_client, request): def bad_post(self, test_client, request):
return lambda: test_client.post('/data', data=flask.json.dumps(dict( return lambda: test_client.post('/data', data=flask.json.dumps(dict(
handler=request.param['handler'], handler=request.param['handler'],
server=request.param['server'], server=request.param['server'],
token=request.param['token'], auth=request.param['auth'],
descriptors=request.param['descriptors'] descriptors=request.param['descriptors']
))) )))
...@@ -273,7 +279,7 @@ class TestData: ...@@ -273,7 +279,7 @@ class TestData:
data = dict( data = dict(
handler='test', handler='test',
server='localhost:1234', server='localhost:1234',
token='7746391376142672192764', auth={'token': '7746391376142672192764'},
descriptors=[ descriptors=[
{ {
'data_type': 'foo', 'data_type': 'foo',
...@@ -320,7 +326,7 @@ class TestData: ...@@ -320,7 +326,7 @@ class TestData:
data = dict( data = dict(
handler='test', handler='test',
server='localhost:1234', server='localhost:1234',
token='7746391376142672192764', auth={'token': '7746391376142672192764'},
descriptors=[ descriptors=[
{ {
'data_type': 'foo', 'data_type': 'foo',
...@@ -361,7 +367,7 @@ class TestData: ...@@ -361,7 +367,7 @@ class TestData:
data = dict( data = dict(
handler='abc', handler='abc',
server='localhost:1234', server='localhost:1234',
token='7746391376142672192764', auth={'token': '7746391376142672192764'},
descriptors=[ descriptors=[
{ {
'data_type': 'foo', 'data_type': 'foo',
...@@ -376,7 +382,7 @@ class TestData: ...@@ -376,7 +382,7 @@ class TestData:
data = dict( data = dict(
handler='test', handler='test',
server='localhost:1234', server='localhost:1234',
token='7746391376142672192764', auth={'token': '7746391376142672192764'},
descriptors=[ descriptors=[
{ {
'data_type': 'abc', 'data_type': 'abc',
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment