Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Fractalis
fractalis
Commits
ea7be7fe
Commit
ea7be7fe
authored
Jul 19, 2018
by
Sascha Herzinger
Browse files
#3
parent
d63d737d
Changes
2
Hide whitespace changes
Inline
Side-by-side
fractalis/data/etlhandler.py
View file @
ea7be7fe
...
...
@@ -136,7 +136,8 @@ class ETLHandler(metaclass=abc.ABCMeta):
task_ids
=
self
.
find_duplicates
(
data_tasks
,
descriptor
)
for
task_id
in
task_ids
:
redis
.
delete
(
'data:{}'
.
format
(
task_id
))
janitor
.
delay
()
if
task_ids
:
janitor
.
delay
()
def
find_duplicate_task_id
(
self
,
data_tasks
:
List
[
str
],
descriptor
:
dict
)
->
Union
[
str
,
None
]:
...
...
tests/unit/etls/test_etlhandler.py
View file @
ea7be7fe
"""This module provides tests for the etlhandler module."""
import
os
from
pathlib
import
Path
import
pytest
from
fractalis
import
celery
from
fractalis
import
celery
,
app
from
fractalis.data.etlhandler
import
ETLHandler
...
...
@@ -85,6 +88,9 @@ class TestETLHandler:
file_path
=
''
,
descriptor
=
descriptor
,
data_type
=
''
)
data_dir
=
os
.
path
.
join
(
app
.
config
[
'FRACTALIS_TMP_DIR'
],
'data'
)
os
.
makedirs
(
data_dir
,
exist_ok
=
True
)
Path
(
os
.
path
.
join
(
data_dir
,
'456'
)).
touch
()
assert
redis
.
exists
(
'data:456'
)
self
.
etlhandler
.
remove_duplicates
(
data_tasks
=
[
'456'
],
descriptor
=
descriptor
)
...
...
@@ -104,6 +110,11 @@ class TestETLHandler:
file_path
=
''
,
descriptor
=
{
'a'
:
1
},
data_type
=
''
)
data_dir
=
os
.
path
.
join
(
app
.
config
[
'FRACTALIS_TMP_DIR'
],
'data'
)
os
.
makedirs
(
data_dir
,
exist_ok
=
True
)
Path
(
os
.
path
.
join
(
data_dir
,
'123'
)).
touch
()
Path
(
os
.
path
.
join
(
data_dir
,
'456'
)).
touch
()
Path
(
os
.
path
.
join
(
data_dir
,
'789'
)).
touch
()
assert
redis
.
exists
(
'data:123'
)
assert
redis
.
exists
(
'data:456'
)
assert
redis
.
exists
(
'data:789'
)
...
...
@@ -127,6 +138,11 @@ class TestETLHandler:
file_path
=
''
,
descriptor
=
{
'a'
:
1
},
data_type
=
''
)
data_dir
=
os
.
path
.
join
(
app
.
config
[
'FRACTALIS_TMP_DIR'
],
'data'
)
os
.
makedirs
(
data_dir
,
exist_ok
=
True
)
Path
(
os
.
path
.
join
(
data_dir
,
'123'
)).
touch
()
Path
(
os
.
path
.
join
(
data_dir
,
'456'
)).
touch
()
Path
(
os
.
path
.
join
(
data_dir
,
'789'
)).
touch
()
assert
redis
.
exists
(
'data:123'
)
assert
redis
.
exists
(
'data:456'
)
assert
redis
.
exists
(
'data:789'
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment