Skip to content

GitLab

Commit c5b76848 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

Fixed some bugs and made all unit tests pass

parent 52b9ecae
Hide whitespace changes
Inline Side-by-side
fractalis/analytics/task.py
View file @ c5b76848
......@@ -4,7 +4,8 @@ import abc
import json
import re
import logging
from typing import List, Tuple
from uuid import UUID
from typing import List, Tuple, Union
from pandas import read_csv, DataFrame
from celery import Task
......@@ -126,7 +127,7 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
value.endswith('$')
@staticmethod
def parse_value(value: str) -> Tuple[str, dict]:
def parse_value(value: str) -> Tuple[Union[str, None], dict]:
"""Extract data task id and filters from the string.
:param value: A string that contains a data task id.
:return: A tuple of id and filters to apply later.
......@@ -135,7 +136,7 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
# noinspection PyBroadException
try:
value = json.loads(value)
data_task_id = value['id']
data_task_id = str(value['id'])
filters = value.get('filters')
except Exception:
logger.warning("Failed to parse value. "
......@@ -143,7 +144,13 @@ class AnalyticTask(Task, metaclass=abc.ABCMeta):
"but nothing else.")
data_task_id = value
filters = None
return str(data_task_id), filters
# noinspection PyBroadException
try:
data_task_id = str(UUID(data_task_id))
except Exception:
logger.warning("'{}' is no valid task id.".format(data_task_id))
data_task_id = None
return data_task_id, filters
def prepare_args(self, session_data_tasks: List[str],
args: dict, decrypt: bool) -> dict:
......
fractalis/state/controller.py
View file @ c5b76848
......@@ -3,6 +3,7 @@
import re
import json
import logging
import ast
from uuid import UUID, uuid4
from typing import Tuple
......@@ -29,11 +30,11 @@ def save_state() -> Tuple[Response, int]:
"""
logger.debug("Received POST request on /state.")
payload = request.get_json(force=True)
state = json.dumps(payload['state'])
state = str(payload['state'])
matches = re.findall('\$.+?\$', state)
task_ids = [AnalyticTask.parse_value(match)[0] for match in matches]
task_ids = list(set(task_ids))
if not matches:
task_ids = [task_id for task_id in set(task_ids) if task_id is not None]
if not task_ids:
error = "This state cannot be saved because it contains no data " \
"task ids. These are used to verify access to the state and " \
"its potentially sensitive data."
......@@ -51,7 +52,7 @@ def save_state() -> Tuple[Response, int]:
descriptors.append(data_state['meta']['descriptor'])
assert len(task_ids) == len(descriptors)
meta_state = {
'state': state,
'state': ast.literal_eval(state),
'server': payload['server'],
'handler': payload['handler'],
'task_ids': task_ids,
......
tests/functional/test_state.py
View file @ c5b76848
......@@ -36,7 +36,7 @@ class TestState:
def test_400_if_task_id_not_in_redis(self, test_client):
payload = {
'state': {'abc': '$123$'},
'state': {'abc': '${}$'.format(uuid4())},
'handler': 'test',
'server': 'localfoo'
}
......@@ -47,30 +47,35 @@ class TestState:
assert 'could not be found in redis' in body['error']
def test_save_state_saves_and_returns(self, test_client):
uuid = str(uuid4())
payload = {
'state': {'test': ['$123$']},
'state': {'test': ['${}$'.format(uuid), '${${}']},
'handler': 'test',
'server': 'localfoo'
}
redis.set(name='data:123',
redis.set(name='data:{}'.format(uuid),
value=json.dumps({'meta': {'descriptor': 'foo'}}))
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
assert 201 == rv.status_code, body
assert UUID(body['state_id'])
meta_state = json.loads(redis.get('state:{}'.format(body['state_id'])))
state = json.loads(meta_state['state'])
assert ['$123$'] == state.get('test')
assert meta_state['task_ids'] == [uuid]
assert meta_state['state']['test'][0] == '${}$'.format(uuid)
def test_save_state_discards_duplicates(self, test_client):
uuid1 = str(uuid4())
uuid2 = str(uuid4())
payload = {
'state': {'test': ['$123$', '$123$', '$456$']},
'state': {'test': ['${}$'.format(uuid1),
'${}$'.format(uuid1),
'${}$'.format(uuid2)]},
'handler': 'test',
'server': 'localfoo'
}
redis.set(name='data:123',
redis.set(name='data:{}'.format(uuid1),
value=json.dumps({'meta': {'descriptor': 'foo'}}))
redis.set(name='data:456',
redis.set(name='data:{}'.format(uuid2),
value=json.dumps({'meta': {'descriptor': 'bar'}}))
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
......@@ -79,14 +84,14 @@ class TestState:
meta_state = json.loads(redis.get('state:{}'.format(body['state_id'])))
assert len(meta_state['task_ids']) == 2
assert len(meta_state['descriptors']) == 2
assert '123' in meta_state['task_ids']
assert '456' in meta_state['task_ids']
assert uuid1 in meta_state['task_ids']
assert uuid2 in meta_state['task_ids']
assert 'foo' in meta_state['descriptors']
assert 'bar' in meta_state['descriptors']
def test_400_if_payload_schema_incorrect_1(self, test_client):
payload = {
'state': {'test': ['$123$']},
'state': {'test': ['${}$'.format(uuid4())]},
'server': 'localfoo'
}
rv = test_client.post('/state', data=flask.json.dumps(payload))
......@@ -142,11 +147,13 @@ class TestState:
def test_request_state_access_reuses_duplicate(
self, test_client):
uuid1 = str(uuid4())
uuid2 = str(uuid4())
meta_state = {
'state': {'foo': ['$123$', '$456$']},
'state': {'foo': ['${}$'.format(uuid1), '${}$'.format(uuid2)]},
'handler': 'test',
'server': 'localfoo',
'task_ids': ['123', '456'],
'task_ids': [uuid1, uuid2],
'descriptors': [
{'data_type': 'default'},
{'data_type': 'default'}
......@@ -173,15 +180,17 @@ class TestState:
def test_request_state_reuses_previous_etls_but_only_in_own_scope(
self, test_client, monkeypatch):
uuid1 = str(uuid4())
uuid2 = str(uuid4())
descriptor_1 = {'data_type': 'default', 'id': 1}
descriptor_2 = {'data_type': 'default', 'id': 2}
handler = 'test'
server = 'localfoo'
meta_state = {
'state': {'foo': ['$123$', '$456$']},
'state': {'foo': ['${}$'.format(uuid1), '${}$'.format(uuid2)]},
'handler': handler,
'server': server,
'task_ids': ['123', '456'],
'task_ids': [uuid1, uuid2],
'descriptors': [
descriptor_1,
descriptor_2
......@@ -193,16 +202,16 @@ class TestState:
etlhandler = ETLHandler.factory(handler=handler,
server=server,
auth={})
etlhandler.create_redis_entry(task_id='123',
etlhandler.create_redis_entry(task_id=uuid1,
file_path='',
descriptor=descriptor_1,
data_type='')
etlhandler.create_redis_entry(task_id='456',
etlhandler.create_redis_entry(task_id=uuid2,
file_path='',
descriptor=descriptor_2,
data_type='')
with test_client.session_transaction() as sess:
sess['data_tasks'] = ['456']
sess['data_tasks'] = [uuid2]
class FakeAsyncResult:
def __init__(self, *args, **kwargs):
......
tests/unit/analytics/test_analytics_task.py
View file @ c5b76848
......@@ -2,6 +2,7 @@
import pandas as pd
from uuid import uuid4
from fractalis.analytics.task import AnalyticTask
......@@ -32,12 +33,18 @@ class TestAnalyticsTask:
assert df2.shape == (3, 3)
def test_parse_value(self):
uuid = str(uuid4())
arg1 = '${"id": 123, "filters": {"foo": [1,2]}}$'
arg2 = '$123$'
arg2 = '${}$'.format(uuid)
arg3 = '${{"id": "{}", "filters": {{"foo": [1,2]}}}}$'.format(uuid)
data_task_id, filters = self.task.parse_value(arg1)
assert data_task_id == '123'
assert data_task_id is None
assert 'foo' in filters
assert filters['foo'] == [1, 2]
data_task_id, filters = self.task.parse_value(arg2)
assert data_task_id == '123'
assert data_task_id == uuid
assert not filters
data_task_id, filters = self.task.parse_value(arg3)
assert data_task_id == uuid
assert 'foo' in filters
assert filters['foo'] == [1, 2]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment