Commit ea7be7fe authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

#3

parent d63d737d
...@@ -136,7 +136,8 @@ class ETLHandler(metaclass=abc.ABCMeta): ...@@ -136,7 +136,8 @@ class ETLHandler(metaclass=abc.ABCMeta):
task_ids = self.find_duplicates(data_tasks, descriptor) task_ids = self.find_duplicates(data_tasks, descriptor)
for task_id in task_ids: for task_id in task_ids:
redis.delete('data:{}'.format(task_id)) redis.delete('data:{}'.format(task_id))
janitor.delay() if task_ids:
janitor.delay()
def find_duplicate_task_id(self, data_tasks: List[str], def find_duplicate_task_id(self, data_tasks: List[str],
descriptor: dict) -> Union[str, None]: descriptor: dict) -> Union[str, None]:
......
"""This module provides tests for the etlhandler module.""" """This module provides tests for the etlhandler module."""
import os
from pathlib import Path
import pytest import pytest
from fractalis import celery from fractalis import celery, app
from fractalis.data.etlhandler import ETLHandler from fractalis.data.etlhandler import ETLHandler
...@@ -85,6 +88,9 @@ class TestETLHandler: ...@@ -85,6 +88,9 @@ class TestETLHandler:
file_path='', file_path='',
descriptor=descriptor, descriptor=descriptor,
data_type='') data_type='')
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
os.makedirs(data_dir, exist_ok=True)
Path(os.path.join(data_dir, '456')).touch()
assert redis.exists('data:456') assert redis.exists('data:456')
self.etlhandler.remove_duplicates(data_tasks=['456'], self.etlhandler.remove_duplicates(data_tasks=['456'],
descriptor=descriptor) descriptor=descriptor)
...@@ -104,6 +110,11 @@ class TestETLHandler: ...@@ -104,6 +110,11 @@ class TestETLHandler:
file_path='', file_path='',
descriptor={'a': 1}, descriptor={'a': 1},
data_type='') data_type='')
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
os.makedirs(data_dir, exist_ok=True)
Path(os.path.join(data_dir, '123')).touch()
Path(os.path.join(data_dir, '456')).touch()
Path(os.path.join(data_dir, '789')).touch()
assert redis.exists('data:123') assert redis.exists('data:123')
assert redis.exists('data:456') assert redis.exists('data:456')
assert redis.exists('data:789') assert redis.exists('data:789')
...@@ -127,6 +138,11 @@ class TestETLHandler: ...@@ -127,6 +138,11 @@ class TestETLHandler:
file_path='', file_path='',
descriptor={'a': 1}, descriptor={'a': 1},
data_type='') data_type='')
data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
os.makedirs(data_dir, exist_ok=True)
Path(os.path.join(data_dir, '123')).touch()
Path(os.path.join(data_dir, '456')).touch()
Path(os.path.join(data_dir, '789')).touch()
assert redis.exists('data:123') assert redis.exists('data:123')
assert redis.exists('data:456') assert redis.exists('data:456')
assert redis.exists('data:789') assert redis.exists('data:789')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment