Commit 90b612aa authored by Sascha Herzinger's avatar Sascha Herzinger

Fixing import manage issue in sdist

parent 744563a6
include fractalis/logging.yaml
include manage.py
import os
from fractalis import app, redis, sync, celery
@celery.task
def janitor():
"""Ideally this is maintained by a systemd service to cleanup redis and the
file system while Fractalis is running.
"""
tmp_dir = app.config['FRACTALIS_TMP_DIR']
tracked_files = [key.split(':')[1] for key in redis.scan_iter('data:*')]
cached_files = [f for f in os.listdir(tmp_dir)
if os.path.isfile(os.path.join(tmp_dir, f))]
for cached_file in cached_files:
if cached_file not in tracked_files:
sync.remove_file(os.path.join(tmp_dir, cached_file))
......@@ -7,7 +7,7 @@ import logging
from uuid import uuid4
from typing import List, Union
import manage
from fractalis.cleanup import janitor
from fractalis import app, redis, celery
from fractalis.data.etl import ETL
......@@ -136,7 +136,7 @@ class ETLHandler(metaclass=abc.ABCMeta):
task_ids = self.find_duplicates(data_tasks, descriptor)
for task_id in task_ids:
redis.delete('data:{}'.format(task_id))
manage.janitor.delay()
janitor.delay()
def find_duplicate_task_id(self, data_tasks: List[str],
descriptor: dict) -> Union[str, None]:
......
import os
from flask_script import Manager
from fractalis import app, redis, sync, celery
import fractalis.cleanup
from fractalis import app
manager = Manager(app)
@celery.task
@manager.command
def janitor():
"""Ideally this is maintained by a systemd service to cleanup redis and the
file system while Fractalis is running.
"""
tmp_dir = app.config['FRACTALIS_TMP_DIR']
tracked_files = [key.split(':')[1] for key in redis.scan_iter('data:*')]
cached_files = [f for f in os.listdir(tmp_dir)
if os.path.isfile(os.path.join(tmp_dir, f))]
for cached_file in cached_files:
if cached_file not in tracked_files:
sync.remove_file(os.path.join(tmp_dir, cached_file))
fractalis.cleanup.janitor.delay()
if __name__ == "__main__":
......
......@@ -3,7 +3,7 @@
import os
from pathlib import Path
import manage
from fractalis.cleanup import janitor
from fractalis import app, redis
......@@ -14,7 +14,7 @@ class TestManage:
tmp_dir = app.config['FRACTALIS_TMP_DIR']
os.makedirs(tmp_dir, exist_ok=True)
Path(os.path.join(tmp_dir, 'abc')).touch()
manage.janitor()
janitor()
assert not os.path.exists(os.path.join(tmp_dir, 'abc'))
def test_janitor_does_not_remove_tracked_files(self):
......@@ -22,5 +22,5 @@ class TestManage:
os.makedirs(tmp_dir, exist_ok=True)
Path(os.path.join(tmp_dir, 'abc')).touch()
redis.set('data:abc', '')
manage.janitor()
janitor()
assert os.path.exists(os.path.join(tmp_dir, 'abc'))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment