Commit c5b76848 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

Fixed some bugs and made all unit tests pass

parent 52b9ecae
......@@ -4,7 +4,8 @@ import abc
import json
import re
import logging
from typing import List, Tuple
from uuid import UUID
from typing import List, Tuple, Union
from pandas import read_csv, DataFrame
from celery import Task
......@@ -126,7 +127,7 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
value.endswith('$')
@staticmethod
def parse_value(value: str) -> Tuple[str, dict]:
def parse_value(value: str) -> Tuple[Union[str, None], dict]:
"""Extract data task id and filters from the string.
:param value: A string that contains a data task id.
:return: A tuple of id and filters to apply later.
......@@ -135,7 +136,7 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
# noinspection PyBroadException
try:
value = json.loads(value)
data_task_id = value['id']
data_task_id = str(value['id'])
filters = value.get('filters')
except Exception:
logger.warning("Failed to parse value. "
......@@ -143,7 +144,13 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
"but nothing else.")
data_task_id = value
filters = None
return str(data_task_id), filters
# noinspection PyBroadException
try:
data_task_id = str(UUID(data_task_id))
except Exception:
logger.warning("'{}' is no valid task id.".format(data_task_id))
data_task_id = None
return data_task_id, filters
def prepare_args(self, session_data_tasks: List[str],
args: dict, decrypt: bool) -> dict:
......
......@@ -3,6 +3,7 @@
import re
import json
import logging
import ast
from uuid import UUID, uuid4
from typing import Tuple
......@@ -29,11 +30,11 @@ def save_state() -> Tuple[Response, int]:
"""
logger.debug("Received POST request on /state.")
payload = request.get_json(force=True)
state = json.dumps(payload['state'])
state = str(payload['state'])
matches = re.findall('\$.+?\$', state)
task_ids = [AnalyticTask.parse_value(match)[0] for match in matches]
task_ids = list(set(task_ids))
if not matches:
task_ids = [task_id for task_id in set(task_ids) if task_id is not None]
if not task_ids:
error = "This state cannot be saved because it contains no data " \
"task ids. These are used to verify access to the state and " \
"its potentially sensitive data."
......@@ -51,7 +52,7 @@ def save_state() -> Tuple[Response, int]:
descriptors.append(data_state['meta']['descriptor'])
assert len(task_ids) == len(descriptors)
meta_state = {
'state': state,
'state': ast.literal_eval(state),
'server': payload['server'],
'handler': payload['handler'],
'task_ids': task_ids,
......
......@@ -36,7 +36,7 @@ class TestState:
def test_400_if_task_id_not_in_redis(self, test_client):
payload = {
'state': {'abc': '$123$'},
'state': {'abc': '${}$'.format(uuid4())},
'handler': 'test',
'server': 'localfoo'
}
......@@ -47,30 +47,35 @@ class TestState:
assert 'could not be found in redis' in body['error']
def test_save_state_saves_and_returns(self, test_client):
uuid = str(uuid4())
payload = {
'state': {'test': ['$123$']},
'state': {'test': ['${}$'.format(uuid), '${${}']},
'handler': 'test',
'server': 'localfoo'
}
redis.set(name='data:123',
redis.set(name='data:{}'.format(uuid),
value=json.dumps({'meta': {'descriptor': 'foo'}}))
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
assert 201 == rv.status_code, body
assert UUID(body['state_id'])
meta_state = json.loads(redis.get('state:{}'.format(body['state_id'])))
state = json.loads(meta_state['state'])
assert ['$123$'] == state.get('test')
assert meta_state['task_ids'] == [uuid]
assert meta_state['state']['test'][0] == '${}$'.format(uuid)
def test_save_state_discards_duplicates(self, test_client):
uuid1 = str(uuid4())
uuid2 = str(uuid4())
payload = {
'state': {'test': ['$123$', '$123$', '$456$']},
'state': {'test': ['${}$'.format(uuid1),
'${}$'.format(uuid1),
'${}$'.format(uuid2)]},
'handler': 'test',
'server': 'localfoo'
}
redis.set(name='data:123',
redis.set(name='data:{}'.format(uuid1),
value=json.dumps({'meta': {'descriptor': 'foo'}}))
redis.set(name='data:456',
redis.set(name='data:{}'.format(uuid2),
value=json.dumps({'meta': {'descriptor': 'bar'}}))
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
......@@ -79,14 +84,14 @@ class TestState:
meta_state = json.loads(redis.get('state:{}'.format(body['state_id'])))
assert len(meta_state['task_ids']) == 2
assert len(meta_state['descriptors']) == 2
assert '123' in meta_state['task_ids']
assert '456' in meta_state['task_ids']
assert uuid1 in meta_state['task_ids']
assert uuid2 in meta_state['task_ids']
assert 'foo' in meta_state['descriptors']
assert 'bar' in meta_state['descriptors']
def test_400_if_payload_schema_incorrect_1(self, test_client):
payload = {
'state': {'test': ['$123$']},
'state': {'test': ['${}$'.format(uuid4())]},
'server': 'localfoo'
}
rv = test_client.post('/state', data=flask.json.dumps(payload))
......@@ -142,11 +147,13 @@ class TestState:
def test_request_state_access_reuses_duplicate(
self, test_client):
uuid1 = str(uuid4())
uuid2 = str(uuid4())
meta_state = {
'state': {'foo': ['$123$', '$456$']},
'state': {'foo': ['${}$'.format(uuid1), '${}$'.format(uuid2)]},
'handler': 'test',
'server': 'localfoo',
'task_ids': ['123', '456'],
'task_ids': [uuid1, uuid2],
'descriptors': [
{'data_type': 'default'},
{'data_type': 'default'}
......@@ -173,15 +180,17 @@ class TestState:
def test_request_state_reuses_previous_etls_but_only_in_own_scope(
self, test_client, monkeypatch):
uuid1 = str(uuid4())
uuid2 = str(uuid4())
descriptor_1 = {'data_type': 'default', 'id': 1}
descriptor_2 = {'data_type': 'default', 'id': 2}
handler = 'test'
server = 'localfoo'
meta_state = {
'state': {'foo': ['$123$', '$456$']},
'state': {'foo': ['${}$'.format(uuid1), '${}$'.format(uuid2)]},
'handler': handler,
'server': server,
'task_ids': ['123', '456'],
'task_ids': [uuid1, uuid2],
'descriptors': [
descriptor_1,
descriptor_2
......@@ -193,16 +202,16 @@ class TestState:
etlhandler = ETLHandler.factory(handler=handler,
server=server,
auth={})
etlhandler.create_redis_entry(task_id='123',
etlhandler.create_redis_entry(task_id=uuid1,
file_path='',
descriptor=descriptor_1,
data_type='')
etlhandler.create_redis_entry(task_id='456',
etlhandler.create_redis_entry(task_id=uuid2,
file_path='',
descriptor=descriptor_2,
data_type='')
with test_client.session_transaction() as sess:
sess['data_tasks'] = ['456']
sess['data_tasks'] = [uuid2]
class FakeAsyncResult:
def __init__(self, *args, **kwargs):
......
......@@ -2,6 +2,7 @@
import pandas as pd
from uuid import uuid4
from fractalis.analytics.task import AnalyticTask
......@@ -32,12 +33,18 @@ class TestAnalyticsTask:
assert df2.shape == (3, 3)
def test_parse_value(self):
uuid = str(uuid4())
arg1 = '${"id": 123, "filters": {"foo": [1,2]}}$'
arg2 = '$123$'
arg2 = '${}$'.format(uuid)
arg3 = '${{"id": "{}", "filters": {{"foo": [1,2]}}}}$'.format(uuid)
data_task_id, filters = self.task.parse_value(arg1)
assert data_task_id == '123'
assert data_task_id is None
assert 'foo' in filters
assert filters['foo'] == [1, 2]
data_task_id, filters = self.task.parse_value(arg2)
assert data_task_id == '123'
assert data_task_id == uuid
assert not filters
data_task_id, filters = self.task.parse_value(arg3)
assert data_task_id == uuid
assert 'foo' in filters
assert filters['foo'] == [1, 2]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment