Skip to content

GitLab

Commit 4c48f417 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

Fixed some tests

parent c3b5d656
Hide whitespace changes
Inline Side-by-side
fractalis/state/controller.py
View file @ 4c48f417
......@@ -13,7 +13,8 @@ from fractalis.validator import validate_json, validate_schema
from fractalis.analytics.task import AnalyticTask
from fractalis.data.etlhandler import ETLHandler
from fractalis.data.controller import get_data_state_for_task_id
from fractalis.state.schema import request_state_access_schema
from fractalis.state.schema import request_state_access_schema, \
save_state_schema
state_blueprint = Blueprint('state_blueprint', __name__)
......@@ -22,6 +23,7 @@ logger = logging.getLogger(__name__)
@state_blueprint.route('', methods=['POST'])
@validate_json
@validate_schema(save_state_schema)
def save_state() -> Tuple[Response, int]:
"""Save given payload to redis, so it can be accessed later on.
:return: UUID linked to the saved state.
......
fractalis/state/schema.py
View file @ 4c48f417
request_state_access_schema = {
save_state_schema = {
"type": "object",
"properties": {
"state": {"type": "object"},
"handler": {"type": "string"},
"server": {"type": "string"},
"server": {"type": "string"}
},
"required": ["handler", "server", "state"]
}
request_state_access_schema = {
"type": "object",
"properties": {
"auth": {
"type": "object",
"properties": {
......@@ -13,5 +22,5 @@ request_state_access_schema = {
"minProperties": 1
}
},
"required": ["handler", "server", "auth"]
"required": ["auth"]
}
tests/functional/test_data.py
View file @ 4c48f417
......@@ -58,10 +58,12 @@ class TestData:
@pytest.fixture(scope='function', params=['small', 'big'])
def payload(self, request):
load = self.small_load() if request.param == 'small' \
else self.big_load()
return {'size': len(load['descriptors']),
'serialized': flask.json.dumps(load)}
def _payload():
load = self.small_load() if request.param == 'small' \
else self.big_load()
return {'size': len(load['descriptors']),
'serialized': flask.json.dumps(load)}
return _payload
@pytest.fixture(scope='function', params=['small', 'big'])
def faiload(self, request):
......@@ -144,15 +146,17 @@ class TestData:
assert bad_post().status_code == 400
def test_valid_response_on_post(self, test_client, payload):
rv = test_client.post('/data', data=payload['serialized'])
data = payload()
rv = test_client.post('/data', data=data['serialized'])
assert rv.status_code == 201
body = flask.json.loads(rv.get_data())
assert not body
def test_valid_redis_before_loaded_on_post(self, test_client, payload):
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
keys = redis.keys('data:*')
assert len(keys) == payload['size']
assert len(keys) == data['size']
for key in keys:
value = redis.get(key)
data_state = json.loads(value)
......@@ -162,9 +166,10 @@ class TestData:
assert 'meta' in data_state
def test_valid_redis_after_loaded_on_post(self, test_client, payload):
test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
keys = redis.keys('data:*')
assert len(keys) == payload['size']
assert len(keys) == data['size']
for key in keys:
value = redis.get(key)
data_state = json.loads(value)
......@@ -176,7 +181,8 @@ class TestData:
def test_valid_filesystem_before_loaded_on_post(
self, test_client, payload):
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
if os.path.exists(data_dir):
assert len(os.listdir(data_dir)) == 0
keys = redis.keys('data:*')
......@@ -188,8 +194,9 @@ class TestData:
def test_valid_filesystem_after_loaded_on_post(
self, test_client, payload):
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
test_client.post('/data?wait=1', data=payload['serialized'])
assert len(os.listdir(data_dir)) == payload['size']
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
assert len(os.listdir(data_dir)) == data['size']
for f in os.listdir(data_dir):
assert UUID(f)
keys = redis.keys('data:*')
......@@ -199,13 +206,15 @@ class TestData:
assert os.path.exists(data_state['file_path'])
def test_valid_session_on_post(self, test_client, payload):
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
with test_client.session_transaction() as sess:
assert len(sess['data_tasks']) == payload['size']
assert len(sess['data_tasks']) == data['size']
def test_session_matched_redis_in_post_big_payload(
self, test_client, payload):
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
with test_client.session_transaction() as sess:
for task_id in sess['data_tasks']:
assert redis.exists('data:{}'.format(task_id))
......@@ -213,14 +222,18 @@ class TestData:
def test_many_post_and_valid_state(self, test_client, payload):
requests = 5
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
size = 0
for i in range(requests):
rv = test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
size += data['size']
rv = test_client.post('/data?wait=1', data=data['serialized'])
assert rv.status_code == 201
assert len(os.listdir(data_dir)) == requests * payload['size']
assert len(redis.keys('data:*')) == requests * payload['size']
assert len(os.listdir(data_dir)) == size
assert len(redis.keys('data:*')) == size
def test_valid_response_before_loaded_on_get(self, test_client, payload):
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
rv = test_client.get('/data')
assert rv.status_code == 200
body = flask.json.loads(rv.get_data())
......@@ -233,7 +246,8 @@ class TestData:
assert 'task_id' in data_state
def test_valid_response_after_loaded_on_get(self, test_client, payload):
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
rv = test_client.get('/data?wait=1')
assert rv.status_code == 200
body = flask.json.loads(rv.get_data())
......@@ -258,7 +272,8 @@ class TestData:
def test_valid_state_for_finished_etl_on_delete(
self, test_client, payload):
test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
......@@ -270,7 +285,8 @@ class TestData:
assert data_state['task_id'] not in sess['data_tasks']
def test_valid_state_for_running_etl_on_delete(self, test_client, payload):
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
......@@ -294,7 +310,8 @@ class TestData:
assert data_state['task_id'] not in sess['data_tasks']
def test_403_if_no_auth_on_delete(self, test_client, payload):
test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
with test_client.session_transaction() as sess:
sess['data_tasks'] = []
for key in redis.keys('data:*'):
......@@ -312,7 +329,8 @@ class TestData:
def test_valid_state_for_finished_etl_on_delete_all(
self, test_client, payload):
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
test_client.delete('/data?wait=1')
assert not redis.keys('data:*')
assert len(os.listdir(data_dir)) == 0
......@@ -321,7 +339,8 @@ class TestData:
def test_encryption_works(self, test_client, payload):
app.config['FRACTALIS_ENCRYPT_CACHE'] = True
test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
keys = redis.keys('data:*')
for key in keys:
value = redis.get(key)
......@@ -332,7 +351,8 @@ class TestData:
app.config['FRACTALIS_ENCRYPT_CACHE'] = False
def test_valid_response_before_loaded_on_meta(self, test_client, payload):
test_client.post('/data', data=payload['serialized'])
data = payload()
test_client.post('/data', data=data['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
......@@ -342,7 +362,8 @@ class TestData:
assert 'features' not in body['meta']
def test_valid_response_after_loaded_on_meta(self, test_client, payload):
test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
......@@ -353,7 +374,8 @@ class TestData:
assert 'features' in body['meta']
def test_403_if_no_auth_on_get_meta(self, test_client, payload):
test_client.post('/data?wait=1', data=payload['serialized'])
data = payload()
test_client.post('/data?wait=1', data=data['serialized'])
with test_client.session_transaction() as sess:
sess['data_tasks'] = []
for key in redis.keys('data:*'):
......
tests/functional/test_state.py
View file @ 4c48f417
......@@ -22,42 +22,62 @@ class TestState:
sync.cleanup_all()
def test_400_if_no_task_id_in_payload(self, test_client):
rv = test_client.post('/state', data=flask.json.dumps('$...foo'))
payload = {
'state': {'abc': '$...foo'},
'handler': 'test',
'server': 'localfoo'
}
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
assert 400 == rv.status_code, body
assert 'error' in body
assert 'contains no data task ids' in body['error']
def test_400_if_task_id_not_in_redis(self, test_client):
rv = test_client.post('/state', data=flask.json.dumps('$123$'))
payload = {
'state': {'abc': '$123$'},
'handler': 'test',
'server': 'localfoo'
}
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
assert 400 == rv.status_code, body
assert 'error' in body
assert 'could not be found in redis' in body['error']
def test_400_if_task_id_in_redis_but_no_data_state(self, test_client):
payload = {
'state': {'abc': '$123$'},
'handler': 'test',
'server': 'localfoo'
}
redis.set('data:123', '')
rv = test_client.post('/state', data=flask.json.dumps('$123$'))
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
assert 400 == rv.status_code, body
assert 'error' in body
assert 'no valid data state' in body['error']
def test_save_state_saves_and_returns(self, test_client):
payload = {
'state': {'test': ['$123$']},
'handler': 'test',
'server': 'localfoo'
}
redis.set('data:123', json.dumps({'meta': {'descriptor': 'foo'}}))
rv = test_client.post('/state',
data=flask.json.dumps({'test': ['$123$']}))
data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
assert 201 == rv.status_code, body
assert UUID(body['state_id'])
state = json.loads(redis.get('state:{}'.format(body['state_id'])))
assert 'test' in state
assert ['$123$'] == state['test']
meta_state = json.loads(redis.get('state:{}'.format(body['state_id'])))
assert 'test' in meta_state['state']
assert ['$123$'] == json.loads(meta_state['state'])['test']
def test_404_if_request_invalid_state_id(self, test_client):
rv = test_client.post(
'/state/{}'.format(str(uuid4())), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
{'auth': {'token': ''}}))
assert 404 == rv.status_code
body = flask.json.loads(rv.get_data())
assert 'error' in body
......@@ -73,11 +93,17 @@ class TestState:
assert 400 == rv.status_code
def test_error_if_task_id_is_no_etl_id(self, test_client):
payload = {
'state': {'foo': '$123$'},
'handler': 'test',
'server': 'localfoo',
'descriptors': ''
}
uuid = str(uuid4())
redis.set(name='state:{}'.format(uuid),
value=json.dumps({'foo': '$123$'}))
rv = test_client.post('/state/{}'.format(uuid), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
value=json.dumps(payload))
rv = test_client.post('/state/{}'.format(uuid),
data=flask.json.dumps({'auth': {'token': ''}}))
body = flask.json.loads(rv.get_data())
assert 403 == rv.status_code, body
assert 'error' in body
......@@ -85,15 +111,21 @@ class TestState:
assert not redis.exists('state:{}'.format(uuid))
def test_202_create_valid_state_if_valid_conditions(self, test_client):
payload = {
'state': {'foo': '$123$'},
'handler': 'test',
'server': 'localfoo',
'descriptors': ''
}
uuid = str(uuid4())
redis.set(name='data:123',
value=json.dumps(
{'meta': {'descriptor': {'data_type': 'default'}}}))
redis.set(name='state:{}'.format(uuid),
value=json.dumps({'foo': '$123$'}))
value=json.dumps(payload))
rv = test_client.post(
'/state/{}'.format(uuid), data=flask.json.dumps(
{'handler': 'test', 'server': 'foo', 'auth': {'token': ''}}))
{'auth': {'token': ''}}))
body = flask.json.loads(rv.get_data())
assert 202 == rv.status_code, body
assert not body
......@@ -117,17 +149,21 @@ class TestState:
def test_404_if_get_not_previously_self_requested_state(self, test_client):
uuid = str(uuid4())
payload = {
'state': '${}$'.format(uuid),
'handler': 'test',
'server': 'localfoo'
}
redis.set(name='data:{}'.format(uuid),
value=json.dumps(
{'meta': {'descriptor': {'data_type': 'default'}}}))
rv = test_client.post('/state',
data=flask.json.dumps('${}$'.format(uuid)))
data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
assert 201 == rv.status_code, body
state_id = body['state_id']
rv = test_client.post(
'/state/{}'.format(state_id), data=flask.json.dumps(
{'handler': 'test', 'server': 'foo', 'auth': {'token': ''}}))
rv = test_client.post('/state/{}'.format(state_id),
data=flask.json.dumps({'auth': {'token': ''}}))
body = flask.json.loads(rv.get_data())
assert 202 == rv.status_code, body
with test_client.session_transaction() as sess:
......@@ -139,18 +175,19 @@ class TestState:
assert 'Cannot get state.' in body['error']
def test_403_if_etl_fails(self, test_client):
payload = {
'state': '$123$',
'handler': 'test',
'server': 'localfoo'
}
redis.set(name='data:123',
value=json.dumps(
{'meta': {'descriptor': {'data_type': 'default'}}}))
rv = test_client.post('/state', data=flask.json.dumps('$123$'))
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
state_id = body['state_id']
test_client.post(
'/state/{}'.format(state_id),
data=flask.json.dumps(
{'handler': 'test',
'server': 'foo',
'auth': {'token': 'fail'}}))
test_client.post('/state/{}'.format(state_id),
data=flask.json.dumps({'auth': {'token': 'fail'}}))
rv = test_client.get('/state/{}?wait=1'.format(state_id))
body = flask.json.loads(rv.get_data())
assert 403 == rv.status_code, body
......@@ -158,18 +195,18 @@ class TestState:
assert 'ETLs failed' in body['error']
def test_202_then_200_for_running_etl(self, test_client):
redis.set(name='data:123',
value=json.dumps(
{'meta': {'descriptor': {'data_type': 'default'}}}))
rv = test_client.post('/state', data=flask.json.dumps('$123$'))
payload = {
'state': '$123$',
'handler': 'test',
'server': 'localfoo'
}
redis.set(name='data:123', value=json.dumps(
{'meta': {'descriptor': {'data_type': 'default'}}}))
rv = test_client.post('/state', data=flask.json.dumps(payload))
body = flask.json.loads(rv.get_data())
state_id = body['state_id']
test_client.post(
'/state/{}'.format(state_id),
data=flask.json.dumps(
{'handler': 'test',
'server': 'foo',
'auth': {'token': ''}}))
test_client.post('/state/{}'.format(state_id),
data=flask.json.dumps({'auth': {'token': ''}}))
rv = test_client.get('/state/{}'.format(state_id))
body = flask.json.loads(rv.get_data())
assert 202 == rv.status_code, body
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment