cleanup.py 942 Bytes
Newer Older
1 2 3 4 5 6 7 8 9 10
import os

from fractalis import app, redis, sync, celery


@celery.task
def janitor():
    """Ideally this is maintained by a systemd service to cleanup redis and the
    file system while Fractalis is running.
    """
Sascha Herzinger's avatar
Sascha Herzinger committed
11 12 13
    data_dir = os.path.join(app.config['FRACTALIS_TMP_DIR'], 'data')
    cached_files = [f for f in os.listdir(data_dir)
                    if os.path.isfile(os.path.join(data_dir, f))]
14
    tracked_files = [key.split(':')[1] for key in redis.scan_iter('data:*')]
Sascha Herzinger's avatar
Sascha Herzinger committed
15 16

    # clean cached files
17 18
    for cached_file in cached_files:
        if cached_file not in tracked_files:
Sascha Herzinger's avatar
Sascha Herzinger committed
19 20 21 22 23 24 25 26
            sync.remove_file(os.path.join(data_dir, cached_file))

    # clean tracked files
    for task_id in tracked_files:
        path = os.path.join(data_dir, task_id)
        async_result = celery.AsyncResult(task_id)
        if async_result.state == 'SUCCESS' and not os.path.exists(path):
            redis.delete('data:{}'.format(task_id))