Commit 94636db1 authored by Sascha Herzinger's avatar Sascha Herzinger
Browse files

Implemented sync tests and fixed 2 uncovered bugs. I love tests.

parent 6d542d68
Pipeline #1783 passed with stage
in 1 minute and 40 seconds
......@@ -16,7 +16,9 @@ from fractalis import app
@celery.task
def remove_expired_redis_entries() -> None:
"""Remove expired entries from the redis DB."""
"""Remove 'data' entries from the redis DB for which 'last_access' lays back
longer than the timedelta defined in 'FRACTALIS_CACHE_EXP'.
"""
cache_expr = app.config['FRACTALIS_CACHE_EXP']
redis_data = redis.hgetall(name='data')
for key in redis_data:
......@@ -34,8 +36,8 @@ def remove_untracked_data_files() -> None:
"""Remove files that have no record in the redis DB"""
tmp_dir = app.config['FRACTALIS_TMP_DIR']
data_dir = os.path.join(tmp_dir, 'data')
redis_data = redis.hdel('data')
for file_path in iglob(os.path.join(data_dir, '*.py')):
redis_data = redis.hgetall('data')
for file_path in iglob(os.path.join(data_dir, '*')):
# check if file tracked by redis
is_tracked = False
for key in redis_data:
......@@ -49,7 +51,8 @@ def remove_untracked_data_files() -> None:
def cleanup_all() -> None:
"""Reset redis and the filesystem. This is only useful for testing and
should !!!NEVER!!! be used otherwise."""
should !!!NEVER!!! be used otherwise.
"""
redis.flushall()
tmp_dir = app.config['FRACTALIS_TMP_DIR']
if os.path.exists(tmp_dir):
......
......@@ -2,16 +2,95 @@
components synchronized.
"""
from fractalis import app, redis, sync
import os
import json
import time
from pathlib import Path
from uuid import uuid4
import pytest
from fractalis import app, sync
class TestSync:
def test_expired_entries_removed(self):
assert False
@pytest.fixture()
def redis(self):
from fractalis import redis
yield redis
sync.cleanup_all()
def test_expired_entries_removed(self, redis):
data_obj = {
'file_path': 'foo',
'last_access': 0 # last access = 1970
}
redis.hset(name='data', key=123, value=json.dumps(data_obj))
assert redis.hget(name='data', key=123)
sync.remove_expired_redis_entries()
assert not redis.hget(name='data', key=123)
def test_not_expired_entries_not_removed(self, redis):
data_obj = {
'file_path': 'foo',
'last_access': time.time() # now
}
redis.hset(name='data', key=123, value=json.dumps(data_obj))
assert redis.hget(name='data', key=123)
sync.remove_expired_redis_entries()
assert redis.hget(name='data', key=123)
def test_expired_tracked_files_removed(self, redis):
tmp_dir = app.config['FRACTALIS_TMP_DIR']
data_dir = os.path.join(tmp_dir, 'data')
file_path = os.path.join(data_dir, str(uuid4()))
data_obj = {
'file_path': file_path,
'last_access': 0 # last access = 1970
}
os.makedirs(data_dir, exist_ok=True)
Path(file_path).touch()
redis.hset(name='data', key=123, value=json.dumps(data_obj))
assert os.path.exists(file_path)
sync.remove_expired_redis_entries()
assert not os.path.exists(file_path)
def test_not_expired_tracked_files_not_removed(self, redis):
tmp_dir = app.config['FRACTALIS_TMP_DIR']
data_dir = os.path.join(tmp_dir, 'data')
file_path = os.path.join(data_dir, str(uuid4()))
data_obj = {
'file_path': file_path,
'last_access': time.time()
}
os.makedirs(data_dir, exist_ok=True)
Path(file_path).touch()
redis.hset(name='data', key=123, value=json.dumps(data_obj))
assert os.path.exists(file_path)
sync.remove_expired_redis_entries()
assert os.path.exists(file_path)
def test_tracked_files_removed_after_redis_cleanup(self):
assert False
def test_untracked_files_removed(self):
tmp_dir = app.config['FRACTALIS_TMP_DIR']
data_dir = os.path.join(tmp_dir, 'data')
file_path = os.path.join(data_dir, str(uuid4()))
os.makedirs(data_dir, exist_ok=True)
Path(file_path).touch()
assert os.path.exists(file_path)
sync.remove_untracked_data_files()
assert not os.path.exists(file_path)
def test_untracked_removed(self):
assert False
\ No newline at end of file
def test_tracked_files_not_removed(self, redis):
tmp_dir = app.config['FRACTALIS_TMP_DIR']
data_dir = os.path.join(tmp_dir, 'data')
file_path = os.path.join(data_dir, str(uuid4()))
data_obj = {
'file_path': file_path
}
os.makedirs(data_dir, exist_ok=True)
Path(file_path).touch()
redis.hset(name='data', key=123, value=json.dumps(data_obj))
assert os.path.exists(file_path)
sync.remove_untracked_data_files()
assert os.path.exists(file_path)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment