Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Fractalis
fractalis
Commits
4502426c
Commit
4502426c
authored
May 17, 2018
by
Sascha Herzinger
Browse files
Improved janitor
parent
b02cab4a
Pipeline
#4908
failed with stages
in 10 seconds
Changes
5
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
fractalis/cleanup.py
View file @
4502426c
...
...
@@ -8,10 +8,19 @@ def janitor():
"""Ideally this is maintained by a systemd service to cleanup redis and the
file system while Fractalis is running.
"""
tmp_dir
=
app
.
config
[
'FRACTALIS_TMP_DIR'
]
data_dir
=
os
.
path
.
join
(
app
.
config
[
'FRACTALIS_TMP_DIR'
],
'data'
)
cached_files
=
[
f
for
f
in
os
.
listdir
(
data_dir
)
if
os
.
path
.
isfile
(
os
.
path
.
join
(
data_dir
,
f
))]
tracked_files
=
[
key
.
split
(
':'
)[
1
]
for
key
in
redis
.
scan_iter
(
'data:*'
)]
cached_files
=
[
f
for
f
in
os
.
listdir
(
tmp_dir
)
if
os
.
path
.
isfile
(
os
.
path
.
join
(
tmp_dir
,
f
))]
# clean cached files
for
cached_file
in
cached_files
:
if
cached_file
not
in
tracked_files
:
sync
.
remove_file
(
os
.
path
.
join
(
tmp_dir
,
cached_file
))
sync
.
remove_file
(
os
.
path
.
join
(
data_dir
,
cached_file
))
# clean tracked files
for
task_id
in
tracked_files
:
path
=
os
.
path
.
join
(
data_dir
,
task_id
)
async_result
=
celery
.
AsyncResult
(
task_id
)
if
async_result
.
state
==
'SUCCESS'
and
not
os
.
path
.
exists
(
path
):
redis
.
delete
(
'data:{}'
.
format
(
task_id
))
fractalis/misc/controller.py
View file @
4502426c
...
...
@@ -6,6 +6,8 @@ from typing import Tuple
from
flask
import
Blueprint
,
jsonify
,
Response
from
fractalis.cleanup
import
janitor
misc_blueprint
=
Blueprint
(
'misc_blueprint'
,
__name__
)
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -14,4 +16,7 @@ logger = logging.getLogger(__name__)
@
misc_blueprint
.
route
(
'/version'
,
methods
=
[
'GET'
])
def
get_version
()
->
Tuple
[
Response
,
int
]:
version
=
'0.6.0'
# this is a good place to launch the janitor because /version is one of the
# first requests sent by the front-end on initialization
janitor
.
delay
()
return
jsonify
({
'version'
:
version
}),
201
manage.py
deleted
100644 → 0
View file @
b02cab4a
from
flask_script
import
Manager
import
fractalis.cleanup
from
fractalis
import
app
manager
=
Manager
(
app
)
@
manager
.
command
def
janitor
():
fractalis
.
cleanup
.
janitor
.
delay
()
if
__name__
==
"__main__"
:
manager
.
run
()
setup.py
View file @
4502426c
flask
# -*- coding: utf-8 -*-
from
distutils.core
import
setup
...
...
tests/unit/test_cleanup.py
View file @
4502426c
"""This module provides tests for the janitor"""
import
os
import
json
from
pathlib
import
Path
from
shutil
import
rmtree
from
fractalis.cleanup
import
janitor
from
fractalis
import
app
,
redis
from
fractalis
import
app
,
redis
,
celery
# noinspection PyMissingOrEmptyDocstring,PyMissingTypeHints
class
TestManage
:
class
TestCleanup
:
data_dir
=
os
.
path
.
join
(
app
.
config
[
'FRACTALIS_TMP_DIR'
],
'data'
)
def
teardown_method
(
self
,
method
):
redis
.
flushall
()
rmtree
(
self
.
data_dir
)
def
test_janitor_removes_untracked_files
(
self
):
tmp_dir
=
app
.
config
[
'FRACTALIS_TMP_DIR'
]
os
.
makedirs
(
tmp_dir
,
exist_ok
=
True
)
Path
(
os
.
path
.
join
(
tmp_dir
,
'abc'
)).
touch
()
os
.
makedirs
(
self
.
data_dir
,
exist_ok
=
True
)
Path
(
os
.
path
.
join
(
self
.
data_dir
,
'abc'
)).
touch
()
janitor
()
assert
not
os
.
path
.
exists
(
os
.
path
.
join
(
tmp
_dir
,
'abc'
))
assert
not
os
.
path
.
exists
(
os
.
path
.
join
(
self
.
data
_dir
,
'abc'
))
def
test_janitor_does_not_remove_tracked_files
(
self
):
tmp_dir
=
app
.
config
[
'FRACTALIS_TMP_DIR'
]
os
.
makedirs
(
tmp_dir
,
exist_ok
=
True
)
Path
(
os
.
path
.
join
(
tmp_dir
,
'abc'
)).
touch
()
os
.
makedirs
(
self
.
data_dir
,
exist_ok
=
True
)
Path
(
os
.
path
.
join
(
self
.
data_dir
,
'abc'
)).
touch
()
redis
.
set
(
'data:abc'
,
''
)
janitor
()
assert
os
.
path
.
exists
(
os
.
path
.
join
(
tmp_dir
,
'abc'
))
assert
os
.
path
.
exists
(
os
.
path
.
join
(
self
.
data_dir
,
'abc'
))
def
test_janitor_removes_redis_keys_without_file
(
self
,
monkeypatch
):
os
.
makedirs
(
self
.
data_dir
,
exist_ok
=
True
)
path
=
os
.
path
.
join
(
self
.
data_dir
,
'123'
)
data_state
=
{
'file_path'
:
path
}
redis
.
set
(
'data:123'
,
json
.
dumps
(
data_state
))
class
FakeAsyncResult
:
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
state
=
'SUCCESS'
def
get
(
self
,
*
args
,
**
kwargs
):
pass
monkeypatch
.
setattr
(
celery
,
'AsyncResult'
,
FakeAsyncResult
)
janitor
()
assert
not
redis
.
exists
(
'data:123'
)
def
test_janitor_doesnt_remove_redis_keys_with_file
(
self
,
monkeypatch
):
os
.
makedirs
(
self
.
data_dir
,
exist_ok
=
True
)
path
=
os
.
path
.
join
(
self
.
data_dir
,
'123'
)
Path
(
path
).
touch
()
data_state
=
{
'file_path'
:
path
}
redis
.
set
(
'data:123'
,
json
.
dumps
(
data_state
))
class
FakeAsyncResult
:
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
state
=
'SUCCESS'
def
get
(
self
,
*
args
,
**
kwargs
):
pass
monkeypatch
.
setattr
(
celery
,
'AsyncResult'
,
FakeAsyncResult
)
janitor
()
assert
redis
.
exists
(
'data:123'
)
def
test_janitor_doesnt_remove_redis_keys_if_not_finished
(
self
,
monkeypatch
):
os
.
makedirs
(
self
.
data_dir
,
exist_ok
=
True
)
path
=
os
.
path
.
join
(
self
.
data_dir
,
'123'
)
data_state
=
{
'file_path'
:
path
}
redis
.
set
(
'data:123'
,
json
.
dumps
(
data_state
))
class
FakeAsyncResult
:
def
__init__
(
self
,
*
args
,
**
kwargs
):
self
.
state
=
'SUBMITTED'
def
get
(
self
,
*
args
,
**
kwargs
):
pass
monkeypatch
.
setattr
(
celery
,
'AsyncResult'
,
FakeAsyncResult
)
janitor
()
assert
redis
.
exists
(
'data:123'
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment