diff --git a/smash/smash/settings.py b/smash/smash/settings.py
index c11fe16accedc21a7dcdf46f168e4305be7ec5ea..14f7a7ff96dc662eadb5b2216029619d0f16cc28 100644
--- a/smash/smash/settings.py
+++ b/smash/smash/settings.py
@@ -76,6 +76,7 @@ CRON_CLASSES = [
'web.views.kit.KitRequestEmailSendJob',
'web.redcap_connector.RedCapRefreshJob',
'web.views.voucher.ExpireVouchersJob',
+ 'web.importer.importer_cron_job.ImporterCronJob'
]
# Password validation
diff --git a/smash/web/importer/__init__.py b/smash/web/importer/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..eda42f3accab2707cf3151273f71db973c51f267
--- /dev/null
+++ b/smash/web/importer/__init__.py
@@ -0,0 +1,8 @@
+from csv_subject_import_reader import CsvSubjectImportReader
+from exporter import Exporter
+from exporter_cron_job import ExporterCronJob
+from importer import Importer
+from importer_cron_job import ImporterCronJob
+from subject_import_reader import SubjectImportReader
+
+__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob, Exporter, ExporterCronJob]
diff --git a/smash/web/importer/csv_subject_import_reader.py b/smash/web/importer/csv_subject_import_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..187b59aba9d98798f8630054828256aa3b80b07f
--- /dev/null
+++ b/smash/web/importer/csv_subject_import_reader.py
@@ -0,0 +1,74 @@
+import csv
+import datetime
+import logging
+
+from subject_import_reader import SubjectImportReader
+from web.models import StudySubject, Subject, Study
+from web.models.constants import GLOBAL_STUDY_ID
+
+CSV_DATE_FORMAT = "%d-%m-%Y"
+
+logger = logging.getLogger(__name__)
+
+
+class CsvSubjectImportReader(SubjectImportReader):
+ def __init__(self):
+ self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+
+ def load_data(self, filename):
+ study_subjects = []
+ with open(filename) as csv_file:
+ reader = csv.reader(csv_file, delimiter=',')
+ headers = next(reader, None)
+ for row in reader:
+ subject = Subject()
+ study_subject = StudySubject()
+ study_subject.subject = subject
+ study_subject.study = self.study
+ for header, value in zip(headers, row):
+ self.add_data(study_subject, header, value)
+ if study_subject.nd_number is None or study_subject.nd_number == "":
+ study_subject.nd_number = study_subject.screening_number
+ study_subjects.append(study_subject)
+ return study_subjects
+
+ def add_data(self, study_subject, column_name, value):
+ # type: (StudySubject, str, str) -> None
+ if column_name == "first_name":
+ study_subject.subject.first_name = self.get_new_value(study_subject.subject.first_name, column_name, value)
+ elif column_name == "last_name":
+ study_subject.subject.last_name = self.get_new_value(study_subject.subject.last_name, column_name, value)
+ elif column_name == "participant_id":
+ study_subject.screening_number = self.get_new_value(study_subject.screening_number, column_name, value)
+ elif column_name == "date_born":
+ study_subject.subject.date_born = self.get_new_date_value(study_subject.subject.date_born, column_name,
+ value)
+ else:
+ logger.warn("Don't know how to handle column " + column_name + " with data " + value)
+
+ def get_new_value(self, old_value, column_name, new_value):
+ # type: (unicode,unicode,unicode) -> unicode
+ if old_value is None or old_value == "":
+ return new_value
+ if new_value is None or new_value == "":
+ return old_value
+ logger.warn(
+ "Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
+ "). Latest value will be used")
+ return new_value
+
+ def get_new_date_value(self, old_value, column_name, new_value):
+ # type: (datetime,unicode,unicode) -> datetime
+ if old_value is None or old_value == "":
+ try:
+ result = datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ except ValueError:
+ logger.warn("Invalid date: " + new_value)
+ result = old_value
+ return result
+ if new_value is None or new_value == "":
+ return old_value
+ logger.warn(
+ "Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
+ "). Latest value will be used")
+ return datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
diff --git a/smash/web/importer/exporter.py b/smash/web/importer/exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..99b72d79a94443e3f04f4a7ad69f2c859da7b339
--- /dev/null
+++ b/smash/web/importer/exporter.py
@@ -0,0 +1,50 @@
+# coding=utf-8
+import csv
+import logging
+
+from warning_counter import MsgCounterHandler
+from web.models import StudySubject
+
+logger = logging.getLogger(__name__)
+
+
+class Exporter(object):
+ def __init__(self, filename):
+ # type: (str) -> None
+ self.filename = filename
+ self.exported_count = 0
+ self.warning_count = 0
+
+ def execute(self):
+ self.exported_count = 0
+ self.warning_count = 0
+
+ warning_counter = MsgCounterHandler()
+ logging.getLogger('').addHandler(warning_counter)
+
+ with open(self.filename, 'w') as csv_file:
+ data = self.get_subjects_as_array()
+ writer = csv.writer(csv_file, quotechar=str(u'"'), quoting=csv.QUOTE_ALL)
+ for row in data:
+ writer.writerow([s.encode("utf-8") for s in row])
+ self.exported_count += 1
+
+ if "WARNING" in warning_counter.level2count:
+ self.warning_count = warning_counter.level2count["WARNING"]
+ logging.getLogger('').removeHandler(warning_counter)
+
+ def get_summary(self):
+ result = "<p>Number of entries: <b>" + str(self.exported_count) + "</b></p>"
+ style = ''
+ if self.warning_count > 0:
+ style = ' color="brown" '
+ result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"
+
+ return result
+
+ def get_subjects_as_array(self):
+ result = []
+ study_subjects = StudySubject.objects.filter(excluded=True)
+ for study_subject in study_subjects:
+ result.append([study_subject.nd_number])
+ return result
diff --git a/smash/web/importer/exporter_cron_job.py b/smash/web/importer/exporter_cron_job.py
new file mode 100644
index 0000000000000000000000000000000000000000..90ce95a604070e873332ec0c8610180808694ece
--- /dev/null
+++ b/smash/web/importer/exporter_cron_job.py
@@ -0,0 +1,46 @@
+# coding=utf-8
+import logging
+import traceback
+
+import timeout_decorator
+from django.conf import settings
+from django_cron import CronJobBase, Schedule
+
+from exporter import Exporter
+from web.models.constants import CRON_JOB_TIMEOUT
+from ..smash_email import EmailSender
+
+logger = logging.getLogger(__name__)
+
+
+class ExporterCronJob(CronJobBase):
+ RUN_EVERY_MINUTES = 60 * 24
+ schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
+ code = 'web.import_daily_job' # a unique code
+
+ @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
+ def do(self):
+ email_title = "Daily export"
+ email_recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
+
+ filename = getattr(settings, "DAILY_EXPORT_FILE", None)
+
+ if filename is None:
+ logger.info("Exporting subjects skipped. File not defined ")
+ return "export file not defined"
+ logger.info("Exporting subjects from file: " + filename)
+ try:
+ exporter = Exporter(settings.DAILY_EXPORT_FILE)
+ exporter.execute()
+ email_body = exporter.get_summary()
+ EmailSender().send_email(email_title,
+ "<h3>Data was successfully exported to file: " + filename + "</h3>" + email_body,
+ email_recipients)
+ return "export is successful"
+
+ except:
+ tb = traceback.format_exc()
+ EmailSender().send_email(email_title,
+ "<h3><font color='red'>There was a problem with exporting data to file: " + filename + "</font></h3><pre>" + tb + "</pre>",
+ email_recipients)
+ return "export crashed"
diff --git a/smash/web/importer/importer.py b/smash/web/importer/importer.py
new file mode 100644
index 0000000000000000000000000000000000000000..2c7702f0de38fb9a29156e87b207b382f7a7b1ff
--- /dev/null
+++ b/smash/web/importer/importer.py
@@ -0,0 +1,161 @@
+# coding=utf-8
+import logging
+import sys
+import traceback
+
+from django.conf import settings
+from django.contrib.auth.models import User
+
+from subject_import_reader import SubjectImportReader
+from warning_counter import MsgCounterHandler
+from web.models import StudySubject, Subject, Provenance, Worker
+from web.models.constants import GLOBAL_STUDY_ID
+
+logger = logging.getLogger(__name__)
+
+
+class Importer(object):
+ def __init__(self, filename, reader):
+ # type: (str, SubjectImportReader) -> None
+ self.filename = filename
+ self.reader = reader
+ self.added_count = 0
+ self.problematic_count = 0
+ self.merged_count = 0
+ self.warning_count = 0
+ self.study_subjects = []
+
+ self.importer_user = None
+
+ importer_user_name = getattr(settings, "IMPORTER_USER", None)
+ if importer_user_name is not None:
+ user = User.objects.filter(username=importer_user_name)
+ if user is None:
+ logger.warn("User does not exist: " + importer_user_name)
+ else:
+ self.importer_user = Worker.objects.filter(user=user)
+
+ def execute(self):
+ self.added_count = 0
+ self.problematic_count = 0
+ self.merged_count = 0
+ self.warning_count = 0
+
+ warning_counter = MsgCounterHandler()
+ logging.getLogger('').addHandler(warning_counter)
+
+ self.study_subjects = self.reader.load_data(self.filename)
+ for study_subject in self.study_subjects:
+ try:
+ if study_subject.study is None:
+ self.problematic_count += 1
+ logger.warn("Empty study found. Ignoring")
+ continue
+ elif study_subject.study.id != GLOBAL_STUDY_ID:
+ self.problematic_count += 1
+ logger.warn("Empty study found. Ignoring: " + study_subject.study.id)
+ continue
+ else:
+ self.import_study_subject(study_subject)
+ except:
+ self.problematic_count += 1
+ traceback.print_exc(file=sys.stdout)
+ logger.error("Problem with importing study subject: " + study_subject.screening_number)
+ if "WARNING" in warning_counter.level2count:
+ self.warning_count = warning_counter.level2count["WARNING"]
+ logging.getLogger('').removeHandler(warning_counter)
+
+ def import_study_subject(self, study_subject):
+ # type: (StudySubject) -> None
+ db_study_subjects = StudySubject.objects.filter(screening_number=study_subject.screening_number)
+ if db_study_subjects.count() > 0:
+ db_study_subject = db_study_subjects.first()
+ for field in Subject._meta.get_fields():
+ if field.get_internal_type() == "CharField" or field.get_internal_type() == "DateField" or field.get_internal_type() is "BooleanField":
+ new_value = getattr(study_subject.subject, field.name)
+ if new_value is not None and new_value != "":
+ old_value = getattr(db_study_subject.subject, field.name)
+ description = '{} changed from "{}" to "{}"'.format(field, old_value, new_value)
+ p = Provenance(modified_table=Subject._meta.db_table,
+ modified_table_id=db_study_subject.subject.id,
+ modification_author=self.importer_user,
+ previous_value=old_value,
+ new_value=new_value,
+ modification_description=description,
+ modified_field=field,
+ )
+ setattr(db_study_subject.subject, field.name, new_value)
+ p.save()
+
+ db_study_subject.subject.save()
+
+ for field in StudySubject._meta.get_fields():
+ if field.get_internal_type() == "CharField" or field.get_internal_type() == "DateField" or field.get_internal_type() is "BooleanField":
+ new_value = getattr(study_subject, field.name)
+ if new_value is not None and new_value != "":
+ old_value = getattr(db_study_subject, field.name)
+ description = '{} changed from "{}" to "{}"'.format(field, old_value, new_value)
+ p = Provenance(modified_table=Subject._meta.db_table,
+ modified_table_id=db_study_subject.id,
+ modification_author=self.importer_user,
+ previous_value=old_value,
+ new_value=new_value,
+ modification_description=description,
+ modified_field=field,
+ )
+ setattr(db_study_subject, field.name, new_value)
+ p.save()
+ db_study_subject.save()
+ self.merged_count += 1
+ else:
+ study_subject.subject.save()
+ study_subject.subject = Subject.objects.filter(id=study_subject.subject.id)[0]
+ study_subject.save()
+
+ for field in Subject._meta.get_fields():
+ if field.get_internal_type() == "CharField" or field.get_internal_type() == "DateField" or field.get_internal_type() is "BooleanField":
+ new_value = getattr(study_subject.subject, field.name)
+ if new_value is not None and new_value != "":
+ description = '{} changed from "{}" to "{}"'.format(field, '', new_value)
+ p = Provenance(modified_table=Subject._meta.db_table,
+ modified_table_id=study_subject.subject.id,
+ modification_author=self.importer_user,
+ previous_value='',
+ new_value=new_value,
+ modification_description=description,
+ modified_field=field,
+ )
+ p.save()
+
+ for field in StudySubject._meta.get_fields():
+ if field.get_internal_type() == "CharField" or field.get_internal_type() == "DateField" or field.get_internal_type() is "BooleanField":
+ new_value = getattr(study_subject, field.name)
+ if new_value is not None and new_value != "":
+ description = '{} changed from "{}" to "{}"'.format(field, '', new_value)
+ p = Provenance(modified_table=Subject._meta.db_table,
+ modified_table_id=study_subject.id,
+ modification_author=self.importer_user,
+ previous_value='',
+ new_value=new_value,
+ modification_description=description,
+ modified_field=field,
+ )
+ p.save()
+
+ self.added_count += 1
+
+ def get_summary(self):
+ result = "<p>Number of entries: <b>" + str(len(self.study_subjects)) + "</b></p>" + \
+ "<p>Number of successfully added entries: <b>" + str(self.added_count) + "</b></p>" + \
+ "<p>Number of successfully merged entries: <b>" + str(self.merged_count) + "</b></p>"
+ style = ''
+ if self.problematic_count > 0:
+ style = ' color="red" '
+ result += "<p><font " + style + ">Number of problematic entries: <b>" + str(
+ self.problematic_count) + "</b></font></p>"
+ style = ''
+ if self.warning_count > 0:
+ style = ' color="brown" '
+ result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"
+
+ return result
diff --git a/smash/web/importer/importer_cron_job.py b/smash/web/importer/importer_cron_job.py
new file mode 100644
index 0000000000000000000000000000000000000000..bfb75d105c4d0db0daaea06034fa1a82946b182b
--- /dev/null
+++ b/smash/web/importer/importer_cron_job.py
@@ -0,0 +1,61 @@
+# coding=utf-8
+import datetime
+import logging
+import os
+import os.path
+import traceback
+
+import timeout_decorator
+from django.conf import settings
+from django_cron import CronJobBase, Schedule
+
+from csv_subject_import_reader import CsvSubjectImportReader
+from importer import Importer
+from web.models.constants import CRON_JOB_TIMEOUT
+from ..smash_email import EmailSender
+
+logger = logging.getLogger(__name__)
+
+
+class ImporterCronJob(CronJobBase):
+ RUN_EVERY_MINUTES = 60 * 24
+ schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
+ code = 'web.import_daily_job' # a unique code
+
+ @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
+ def do(self):
+ email_title = "Daily import"
+ email_recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
+
+ filename = getattr(settings, "DAILY_IMPORT_FILE", None)
+
+ if filename is None:
+ logger.info("Importing subjects skipped. File not defined ")
+ return "import file not defined"
+ logger.info("Importing subjects from file: " + filename)
+ if not os.path.isfile(filename):
+ EmailSender().send_email(email_title,
+ "<h3><font color='red'>File with imported data is not available in the system: " + filename + "</font></h3>",
+ email_recipients)
+ return "import file not found"
+ try:
+ importer = Importer(settings.DAILY_IMPORT_FILE, CsvSubjectImportReader())
+ importer.execute()
+ email_body = importer.get_summary()
+ EmailSender().send_email(email_title,
+ "<h3>Data was successfully imported from file: " + filename + "</h3>" + email_body,
+ email_recipients)
+ self.backup_file(filename)
+ return "import is successful"
+
+ except:
+ tb = traceback.format_exc()
+ EmailSender().send_email(email_title,
+ "<h3><font color='red'>There was a problem with importing data from file: " + filename + "</font></h3><pre>" + tb + "</pre>",
+ email_recipients)
+ return "import crashed"
+
+ def backup_file(self, filename):
+ new_file = filename + "-" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") + ".bac"
+ os.rename(filename, new_file)
+ return
diff --git a/smash/web/importer/subject_import_reader.py b/smash/web/importer/subject_import_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bc7f898ea24a1af7e99b32bca1c8a095635a38d
--- /dev/null
+++ b/smash/web/importer/subject_import_reader.py
@@ -0,0 +1,7 @@
+from web.models.study_subject import StudySubject
+
+
+class SubjectImportReader:
+ def load_data(self, filename):
+ # type: (str) -> List [StudySubject]
+ pass
diff --git a/smash/web/importer/warning_counter.py b/smash/web/importer/warning_counter.py
new file mode 100644
index 0000000000000000000000000000000000000000..8500d6a34303f5e2368f536ccd9c206069174eae
--- /dev/null
+++ b/smash/web/importer/warning_counter.py
@@ -0,0 +1,15 @@
+import logging
+
+class MsgCounterHandler(logging.Handler):
+ level2count = None
+
+ def __init__(self, *args, **kwargs):
+ super(MsgCounterHandler, self).__init__(*args, **kwargs)
+ self.level2count = {}
+
+ def emit(self, record):
+ l = record.levelname
+ if l not in self.level2count:
+ self.level2count[l] = 0
+ print(l)
+ self.level2count[l] += 1
diff --git a/smash/web/tests/data/import.csv b/smash/web/tests/data/import.csv
new file mode 100644
index 0000000000000000000000000000000000000000..b13e437a14950691eb4f1daaa7b9cc2a609f8843
--- /dev/null
+++ b/smash/web/tests/data/import.csv
@@ -0,0 +1,2 @@
+first_name,last_name,participant_id,date_born
+Piotr,Gawron,Cov-000001,01-02-2020
\ No newline at end of file
diff --git a/smash/web/tests/data/import_date_of_birth.csv b/smash/web/tests/data/import_date_of_birth.csv
new file mode 100644
index 0000000000000000000000000000000000000000..43d9adb8ac452c865ffcbdb96446f22847873c61
--- /dev/null
+++ b/smash/web/tests/data/import_date_of_birth.csv
@@ -0,0 +1,4 @@
+first_name,last_name,participant_id,date_born
+Piotr,Gawron,Cov-000001,
+Piotr,Gawron,Cov-000002,invalid
+Piotr,Gawron,Cov-000003,2222-20-20
\ No newline at end of file
diff --git a/smash/web/tests/importer/__init__.py b/smash/web/tests/importer/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/smash/web/tests/importer/mock_reader.py b/smash/web/tests/importer/mock_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5aedb09dd658512fcbcfbe8a80913e6f144026f
--- /dev/null
+++ b/smash/web/tests/importer/mock_reader.py
@@ -0,0 +1,13 @@
+import logging
+
+from web.importer.subject_import_reader import SubjectImportReader
+
+logger = logging.getLogger(__name__)
+
+
+class MockReader(SubjectImportReader):
+ def __init__(self, study_subjects):
+ self.study_subjects = study_subjects
+
+ def load_data(self, filename):
+ return self.study_subjects
diff --git a/smash/web/tests/importer/test_csv_subject_import_reader.py b/smash/web/tests/importer/test_csv_subject_import_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..a80f323469577c13837ee6bf83c6ed1baebd004c
--- /dev/null
+++ b/smash/web/tests/importer/test_csv_subject_import_reader.py
@@ -0,0 +1,37 @@
+# coding=utf-8
+
+import logging
+
+from django.test import TestCase
+
+from web.importer import CsvSubjectImportReader
+from web.tests.functions import get_resource_path
+
+logger = logging.getLogger(__name__)
+
+
+class TestCsvReader(TestCase):
+
+ def test_load_data(self):
+ filename = get_resource_path('import.csv')
+ study_subjects = CsvSubjectImportReader().load_data(filename)
+ self.assertEqual(1, len(study_subjects))
+ study_subject = study_subjects[0]
+ self.assertEqual("Piotr", study_subject.subject.first_name)
+ self.assertEqual("Gawron", study_subject.subject.last_name)
+ self.assertEqual("Cov-000001", study_subject.screening_number)
+ self.assertEqual("Cov-000001", study_subject.nd_number)
+
+ self.assertEqual(1, study_subject.subject.date_born.day)
+ self.assertEqual(2, study_subject.subject.date_born.month)
+ self.assertEqual(2020, study_subject.subject.date_born.year)
+
+ self.assertIsNotNone(study_subject.study)
+
+ def test_load_problematic_dates(self):
+ filename = get_resource_path('import_date_of_birth.csv')
+ study_subjects = CsvSubjectImportReader().load_data(filename)
+ self.assertEqual(3, len(study_subjects))
+ self.assertIsNone(study_subjects[0].subject.date_born)
+ self.assertIsNone(study_subjects[1].subject.date_born)
+ self.assertIsNone(study_subjects[2].subject.date_born)
diff --git a/smash/web/tests/importer/test_exporter.py b/smash/web/tests/importer/test_exporter.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a552443b9ba73818783f213a5af0ced1b50a890
--- /dev/null
+++ b/smash/web/tests/importer/test_exporter.py
@@ -0,0 +1,40 @@
+# coding=utf-8
+
+import datetime
+import logging
+
+from django.test import TestCase
+
+from web.tests.functions import create_study_subject
+from web.importer import Exporter
+from web.models import Subject, StudySubject, Study, Provenance
+from web.models.constants import GLOBAL_STUDY_ID
+
+logger = logging.getLogger(__name__)
+
+
+class TestExporter(TestCase):
+
+ def setUp(self):
+ self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+
+ def test_export_not_excluded(self):
+ create_study_subject()
+
+ exporter = Exporter(filename="empty.csv")
+ exporter.execute()
+
+ self.assertEqual(0, exporter.exported_count)
+ self.assertEqual(0, exporter.warning_count)
+
+ def test_export_excluded(self):
+ subject = create_study_subject()
+ subject.excluded=True
+ subject.save()
+
+ exporter = Exporter(filename="empty.csv")
+ exporter.execute()
+
+ self.assertEqual(1, exporter.exported_count)
+ self.assertEqual(0, exporter.warning_count)
+
diff --git a/smash/web/tests/importer/test_exporter_cron_job.py b/smash/web/tests/importer/test_exporter_cron_job.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1b44a37fbb2eec72ddceeef88d43d2928f689c4
--- /dev/null
+++ b/smash/web/tests/importer/test_exporter_cron_job.py
@@ -0,0 +1,45 @@
+# coding=utf-8
+
+import logging
+import tempfile
+
+from django.conf import settings
+from django.test import TestCase
+
+from web.importer import ExporterCronJob
+
+logger = logging.getLogger(__name__)
+from django.core import mail
+from django_cron.models import CronJobLog
+
+
+class TestCronJobExporter(TestCase):
+
+ def setUp(self):
+ setattr(settings, "DAILY_EXPORT_FILE", None)
+
+ def tearDown(self):
+ setattr(settings, "DAILY_EXPORT_FILE", None)
+
+ def test_import_without_configuration(self):
+ CronJobLog.objects.all().delete()
+
+ job = ExporterCronJob()
+
+ status = job.do()
+
+ self.assertEqual("export file not defined", status)
+ self.assertEqual(0, len(mail.outbox))
+
+ def test_import(self):
+ new_file, tmp = tempfile.mkstemp()
+
+ setattr(settings, "DAILY_EXPORT_FILE", tmp)
+ CronJobLog.objects.all().delete()
+
+ job = ExporterCronJob()
+
+ status = job.do()
+
+ self.assertEqual("export is successful", status)
+ self.assertEqual(1, len(mail.outbox))
diff --git a/smash/web/tests/importer/test_importer.py b/smash/web/tests/importer/test_importer.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5e95a1ea900a4a1742abece15e00cfea13160d2
--- /dev/null
+++ b/smash/web/tests/importer/test_importer.py
@@ -0,0 +1,110 @@
+# coding=utf-8
+
+import datetime
+import logging
+
+from django.test import TestCase
+
+from mock_reader import MockReader
+from web.tests.functions import create_study_subject
+from web.importer import Importer
+from web.models import Subject, StudySubject, Study, Provenance
+from web.models.constants import GLOBAL_STUDY_ID
+
+logger = logging.getLogger(__name__)
+
+
+class TestImporter(TestCase):
+
+ def setUp(self):
+ self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+
+ def test_import_new_subject(self):
+ study_subjects = []
+ subject = Subject()
+ subject.first_name = "Piotr"
+ subject.last_name = "Gawron"
+ study_subject = StudySubject()
+ study_subject.screening_number = "Cov-123456"
+ study_subject.subject = subject
+ study_subject.study = self.study
+ study_subjects.append(study_subject)
+
+ importer = Importer(filename="empty.csv", reader=MockReader(study_subjects))
+ subject_counter = Subject.objects.count()
+ study_subject_counter = StudySubject.objects.count()
+ provenance_counter = Provenance.objects.count()
+ importer.execute()
+
+ self.assertEqual(subject_counter + 1, Subject.objects.count())
+ self.assertEqual(study_subject_counter + 1, StudySubject.objects.count())
+ self.assertNotEqual(provenance_counter, Provenance.objects.count())
+
+ self.assertEqual(1, importer.added_count)
+ self.assertEqual(0, importer.problematic_count)
+ self.assertEqual(0, importer.warning_count)
+
+ def test_import_invalid(self):
+ study_subjects = []
+ study_subject = StudySubject()
+ study_subject.screening_number = "Cov-123456"
+ study_subject.study = self.study
+ study_subjects.append(study_subject)
+
+ importer = Importer(filename="empty.csv", reader=MockReader(study_subjects))
+ study_subject_counter = StudySubject.objects.count()
+ importer.execute()
+
+ self.assertEqual(study_subject_counter, StudySubject.objects.count())
+
+ self.assertEqual(0, importer.added_count)
+ self.assertEqual(1, importer.problematic_count)
+
+ def test_import_no_study(self):
+ study_subjects = []
+ study_subject = StudySubject()
+ study_subject.screening_number = "Cov-123456"
+ study_subjects.append(study_subject)
+
+ importer = Importer(filename="empty.csv", reader=MockReader(study_subjects))
+ study_subject_counter = StudySubject.objects.count()
+ importer.execute()
+
+ self.assertEqual(study_subject_counter, StudySubject.objects.count())
+
+ self.assertEqual(0, importer.added_count)
+ self.assertEqual(1, importer.problematic_count)
+
+ def test_import_merge_subject(self):
+ existing_study_subject = create_study_subject()
+ study_subjects = []
+ subject = Subject()
+ subject.first_name = "XYZ"
+ subject.last_name = "AAA"
+ subject.date_born = datetime.datetime.now()
+ study_subject = StudySubject()
+ study_subject.screening_number = existing_study_subject.screening_number
+ study_subject.subject = subject
+ study_subject.study = self.study
+ study_subjects.append(study_subject)
+
+ importer = Importer(filename="empty.csv", reader=MockReader(study_subjects))
+ provenance_counter = Provenance.objects.count()
+ subject_counter = Subject.objects.count()
+ study_subject_counter = StudySubject.objects.count()
+ importer.execute()
+
+ self.assertEqual(subject_counter, Subject.objects.count())
+ self.assertEqual(study_subject_counter, StudySubject.objects.count())
+ self.assertNotEqual(provenance_counter, Provenance.objects.count())
+
+ self.assertEqual(0, importer.added_count)
+ self.assertEqual(1, importer.merged_count)
+ self.assertEqual(0, importer.problematic_count)
+ self.assertEqual(0, importer.warning_count)
+
+ existing_study_subject = StudySubject.objects.filter(id=existing_study_subject.id)[0]
+ self.assertEquals(existing_study_subject.subject.first_name, subject.first_name)
+ self.assertEquals(existing_study_subject.subject.last_name, subject.last_name)
+ self.assertEquals(existing_study_subject.subject.date_born.strftime("%Y-%m-%d"),
+ subject.date_born.strftime("%Y-%m-%d"))
diff --git a/smash/web/tests/importer/test_importer_cron_job.py b/smash/web/tests/importer/test_importer_cron_job.py
new file mode 100644
index 0000000000000000000000000000000000000000..0eec9f6acb0969b11f703639ee961f665b71ad13
--- /dev/null
+++ b/smash/web/tests/importer/test_importer_cron_job.py
@@ -0,0 +1,46 @@
+# coding=utf-8
+
+import logging
+import tempfile
+from shutil import copyfile
+
+from django.conf import settings
+from django.test import TestCase
+
+from web.importer import ImporterCronJob
+from web.tests.functions import get_resource_path
+
+logger = logging.getLogger(__name__)
+from django.core import mail
+from django_cron.models import CronJobLog
+
+
+class TestCronJobImporter(TestCase):
+
+ def tearDown(self):
+ setattr(settings, "DAILY_IMPORT_FILE", None)
+
+ def test_import_without_configuration(self):
+ CronJobLog.objects.all().delete()
+
+ job = ImporterCronJob()
+
+ status = job.do()
+
+ self.assertEqual("import file not defined", status)
+ self.assertEqual(0, len(mail.outbox))
+
+ def test_import(self):
+ filename = get_resource_path('import.csv')
+ new_file, tmp = tempfile.mkstemp()
+ copyfile(filename, tmp)
+
+ setattr(settings, "DAILY_IMPORT_FILE", tmp)
+ CronJobLog.objects.all().delete()
+
+ job = ImporterCronJob()
+
+ status = job.do()
+
+ self.assertEqual("import is successful", status)
+ self.assertEqual(1, len(mail.outbox))