Commit ec67cfc2 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

Fixing several issues (and tests) in the new /state code

parent ca59b6c2
......@@ -63,10 +63,12 @@ celery = make_celery(app)
from fractalis.analytics.controller import analytics_blueprint # noqa: E402
from fractalis.data.controller import data_blueprint # noqa: E402
from fractalis.misc.controller import misc_blueprint # noqa: E402
from fractalis.state.controller import state_blueprint # noqa: E402
log.info("Registering Flask blueprints.")
app.register_blueprint(analytics_blueprint, url_prefix='/analytics')
app.register_blueprint(data_blueprint, url_prefix='/data')
app.register_blueprint(misc_blueprint, url_prefix='/misc')
app.register_blueprint(state_blueprint, url_prefix='/state')
# registering all application celery tasks
log.info("Registering celery tasks.")
......
......@@ -9,10 +9,11 @@ from typing import Tuple
from flask import Blueprint, jsonify, Response, request, session
from fractalis import redis
from fractalis.validator import validate_json
from fractalis.validator import validate_json, validate_schema
from fractalis.analytics.task import AnalyticTask
from fractalis.data.etlhandler import ETLHandler
from fractalis.data.controller import get_data_state_for_task_id
from fractalis.state.schema import request_state_access_schema
state_blueprint = Blueprint('state_blueprint', __name__)
......@@ -26,7 +27,22 @@ def save_state() -> Tuple[Response, int]:
:return: UUID linked to the saved state.
"""
logger.debug("Received POST request on /state.")
payload = request.get_json(force=True)
# check if task ids in payload are valid
for match in re.findall('\$.+?\$', request.data):
task_id = AnalyticTask.parse_value(match)
value = redis.get('data:{}'.format(task_id))
if value is None:
error = "Data task id is {} could not be found in redis. " \
"State cannot be saved".format(task_id)
logger.error(error)
return jsonify({'error': error}), 400
try:
json.loads(value)['meta']['descriptor']
except (ValueError, KeyError):
error = "Task with id {} was found in redis but it is no valid " \
"data task id. State cannot be saved.".format(task_id)
return jsonify({'error': error}), 400
payload = json.loads(request.data)
uuid = uuid4()
redis.set(name='state:{}'.format(uuid), value=payload)
logger.debug("Successfully saved data to redis. Sending response.")
......@@ -34,6 +50,8 @@ def save_state() -> Tuple[Response, int]:
@state_blueprint.route('/<uuid:state_id>', methods=['POST'])
@validate_json
@validate_schema(request_state_access_schema)
def request_state_access(state_id: UUID) -> Tuple[Response, int]:
"""Traverse through the state object linked to the given UUID and look for
data ids. Then attempt to load the data into the current session to verify
......@@ -52,22 +70,18 @@ def request_state_access(state_id: UUID) -> Tuple[Response, int]:
descriptors = []
for match in re.findall('\$.+?\$', value):
task_id = AnalyticTask.parse_value(match)
data = redis.get('data:{}'.format(task_id))
try:
descriptors.append(data['meta']['descriptor'])
except KeyError:
error = "The given payload cannot be saved. One of more task " \
"objects identified by the surrounding $ character is " \
"either a) not a valid ETL task or " \
"b) the corresponding ETL taskhas been deleted."
if redis.get('data:{}'.format(task_id)) is None:
error = "The state with id {} exists, but one or more of the " \
"associated data task ids are missing. Hence this state " \
"is lost forever because access can no longer be " \
"verified. Deleting state..."
logger.error(error)
return jsonify({'error': error}), 400
redis.delete('data:{}'.format(task_id))
return jsonify({'error': error}), 403
etl_handler = ETLHandler.factory(handler=payload['handler'],
server=payload['server'],
auth=payload['auth'])
task_ids = etl_handler.handle(descriptors=payload['descriptors'],
wait=wait)
task_ids = etl_handler.handle(descriptors=descriptors, wait=wait)
session['data_tasks'] += task_ids
session['data_tasks'] = list(set(session['data_tasks']))
# if all task finish successfully we now that session has access to state
......
request_state_access_schema = {
"type": "object",
"properties": {
"handler": {"type": "string"},
"server": {"type": "string"},
"auth": {
"type": "object",
"properties": {
"token": {"type": "string"},
"user": {"type": "string"},
"passwd": {"type": "string"}
},
"minProperties": 1
}
},
"required": ["handler", "server", "auth"]
}
......@@ -50,7 +50,10 @@ def cleanup_all() -> None:
celery.control.purge()
for key in redis.keys('data:*'):
value = redis.get(key)
data_state = json.loads(value)
try:
data_state = json.loads(value)
except ValueError:
continue
task_id = data_state.get('task_id')
if task_id is not None:
async_result = celery.AsyncResult(task_id)
......
......@@ -27,17 +27,28 @@ class TestState:
assert redis.get('state:{}'.format(body['state_id'])) == 'test'
def test_404_if_request_invalid_state_id(self, test_client):
rv = test_client.post('/state/{}'.format(str(uuid4())))
rv = test_client.post(
'/state/{}'.format(str(uuid4())), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
assert rv.status_code == 404
body = flask.json.loads(rv.get_data())
assert 'error' in body
assert 'not find state associated with id' in body['error']
def test_400_if_state_id_is_no_uuid(self, test_client):
def test_404_if_state_id_is_no_uuid(self, test_client):
rv = test_client.post('/state/123')
assert rv.status_code == 404
def test_400_if_payload_schema_incorrect(self, test_client):
rv = test_client.post('/state/{}'.format(str(uuid4())),
data=flask.json.dumps({'foo': 123}))
assert rv.status_code == 400
def test_error_if_task_id_is_no_etl_id(self, test_client):
uuid = str(uuid4())
redis.set('state:{}'.format(uuid), 'foo')
rv = test_client.post('/state/{}'.format(uuid))
rv = test_client.post('/state/{}'.format(uuid), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 400, body
assert 'error' in body
......@@ -46,7 +57,9 @@ class TestState:
def test_202_create_valid_state_if_valid_conditions(self, test_client):
uuid = str(uuid4())
redis.set('data:{}'.format(uuid), {'meta': {'descriptor': 'foo'}})
rv = test_client.post('/state/{}'.format(uuid))
rv = test_client.post(
'/state/{}'.format(uuid), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 202, body
assert not body
......@@ -58,7 +71,9 @@ class TestState:
def test_404_if_get_non_existing_state(self, test_client):
uuid = str(uuid4())
rv = test_client.post('/state/{}'.format(uuid))
rv = test_client.post(
'/state/{}'.format(uuid), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
assert rv.status_code == 404
def test_400_if_get_non_uuid_state(self, test_client):
......@@ -72,7 +87,9 @@ class TestState:
body = flask.json.loads(rv.get_data())
assert rv.status_code == 201, body
state_id = body['state_id']
rv = test_client.post('/state/{}'.format(state_id))
rv = test_client.post(
'/state/{}'.format(state_id), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 202, body
with test_client.session_transaction() as sess:
......@@ -89,7 +106,9 @@ class TestState:
body = flask.json.loads(rv.get_data())
assert rv.status_code == 201, body
state_id = body['state_id']
rv = test_client.post('/state/{}'.format(state_id))
rv = test_client.post(
'/state/{}'.format(state_id), data=flask.json.dumps(
{'handler': '', 'server': '', 'auth': {'token': ''}}))
body = flask.json.loads(rv.get_data())
assert rv.status_code == 202, body
rv = test_client.get('/state/{}'.format(state_id))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment