Commit a9ea7ad6 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

Implemented Janitor

parent 1960798b
Pipeline #2494 failed with stage
in 16 minutes and 52 seconds
......@@ -14,7 +14,7 @@ from fractalis import redis, app, celery
logger = logging.getLogger(__name__)
def remove_data(task_id: str, wait: bool=False) -> None:
def remove_data(task_id: str) -> None:
"""Remove all traces of any data associated with the given id. That includes
redis and the file system.
:param task_id: The id associated with a data state
......@@ -26,18 +26,14 @@ def remove_data(task_id: str, wait: bool=False) -> None:
redis.delete(key)
if value:
data_state = json.loads(value)
async_result = remove_file.delay(data_state['file_path'])
if wait:
async_result.get(propagate=False)
remove_file(data_state['file_path'])
else:
logger.warning("Can't delete file for task id '{}',because there is "
"no associated entry in Redis.".format(task_id))
@celery.task
def remove_file(file_path: str) -> None:
"""Remove the file for the given file path. This is a task because celery
workers might not have the same file system than the web service.
"""Remove the file for the given file path.
:param file_path: Path of file to remove.
"""
try:
......
import os
import json
from flask_script import Manager
from fractalis import app, redis
from fractalis.sync import remove_file
from fractalis import app, redis, sync
manager = Manager(app)
@manager.command
def janitor() -> None:
def janitor():
"""Ideally this is maintained by a systemd service to cleanup redis and the
file system while Fractalis is running.
"""
raise NotImplementedError()
tmp_dir = app.config['FRACTALIS_TMP_DIR']
tracked_files = [key.split(':')[1] for key in redis.scan_iter('data:*')]
cached_files = [f for f in os.listdir(tmp_dir) if os.path.isfile(os.path.join(tmp_dir, f))]
for cached_file in cached_files:
if cached_file not in tracked_files:
sync.remove_file(os.path.join(tmp_dir, cached_file))
if __name__ == "__main__":
......
"""This module provides tests for the janitor"""
import os
from pathlib import Path
import manage
from fractalis import app, redis
# noinspection PyMissingOrEmptyDocstring,PyMissingTypeHints
class TestManage:
def test_janitor_removes_untracked_files(self):
tmp_dir = app.config['FRACTALIS_TMP_DIR']
os.makedirs(tmp_dir, exist_ok=True)
Path(os.path.join(tmp_dir, 'abc')).touch()
manage.janitor()
assert not os.path.exists(os.path.join(tmp_dir, 'abc'))
def test_janitor_does_not_remove_tracked_files(self):
tmp_dir = app.config['FRACTALIS_TMP_DIR']
os.makedirs(tmp_dir, exist_ok=True)
Path(os.path.join(tmp_dir, 'abc')).touch()
redis.set('data:abc', '')
manage.janitor()
assert os.path.exists(os.path.join(tmp_dir, 'abc'))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment