diff --git a/smash/web/importer/__init__.py b/smash/web/importer/__init__.py
index eda42f3accab2707cf3151273f71db973c51f267..99f8f1dedcee0573a8c4d440f6b98d97e8d973ea 100644
--- a/smash/web/importer/__init__.py
+++ b/smash/web/importer/__init__.py
@@ -1,8 +1,10 @@
from csv_subject_import_reader import CsvSubjectImportReader
+from csv_tns_subject_import_reader import TnsCsvSubjectImportReader
from exporter import Exporter
from exporter_cron_job import ExporterCronJob
from importer import Importer
from importer_cron_job import ImporterCronJob
from subject_import_reader import SubjectImportReader
-__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob, Exporter, ExporterCronJob]
+__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob, Exporter, ExporterCronJob,
+ TnsCsvSubjectImportReader]
diff --git a/smash/web/importer/csv_tns_subject_import_reader.py b/smash/web/importer/csv_tns_subject_import_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..638c798cf7782f80407b9a77c712defdc78ef6ff
--- /dev/null
+++ b/smash/web/importer/csv_tns_subject_import_reader.py
@@ -0,0 +1,81 @@
+import csv
+import datetime
+import logging
+
+from subject_import_reader import SubjectImportReader
+from web.models import StudySubject, Subject, Study
+from web.models.constants import GLOBAL_STUDY_ID
+
+CSV_DATE_FORMAT = "%d/%m/%Y"
+
+logger = logging.getLogger(__name__)
+
+
+class TnsCsvSubjectImportReader(SubjectImportReader):
+ def __init__(self):
+ self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+
+ def load_data(self, filename):
+ study_subjects = []
+ with open(filename) as csv_file:
+ reader = csv.reader(csv_file, delimiter=';')
+ headers = next(reader, None)
+ for row in reader:
+ subject = Subject()
+ study_subject = StudySubject()
+ study_subject.subject = subject
+ study_subject.study = self.study
+ for header, value in zip(headers, row):
+ self.add_data(study_subject, header, value)
+ if study_subject.nd_number is None or study_subject.nd_number == "":
+ study_subject.nd_number = study_subject.screening_number
+ study_subjects.append(study_subject)
+ return study_subjects
+
+ def add_data(self, study_subject, column_name, value):
+ # type: (StudySubject, str, str) -> None
+ if column_name == "firstname":
+ study_subject.subject.first_name = self.get_new_value(study_subject.subject.first_name, column_name, value)
+ elif column_name == "lastname":
+ study_subject.subject.last_name = self.get_new_value(study_subject.subject.last_name, column_name, value)
+ elif column_name == "donor_id":
+ study_subject.screening_number = self.get_new_value(study_subject.screening_number, column_name, value)
+ elif column_name == "phonenr":
+ study_subject.subject.phone_number = self.get_new_value(study_subject.subject.phone_number, column_name,
+ value)
+ elif column_name == "treatingphysician":
+ if value is not None and value != "":
+ value = "Treating physician: " + value
+ study_subject.comments = self.get_new_value(study_subject.comments, column_name, value)
+ elif column_name == "dateofbirth":
+ study_subject.subject.date_born = self.get_new_date_value(study_subject.subject.date_born, column_name,
+ value)
+ else:
+ logger.warn("Don't know how to handle column " + column_name + " with data " + value)
+
+ def get_new_value(self, old_value, column_name, new_value):
+ # type: (unicode,unicode,unicode) -> unicode
+ if old_value is None or old_value == "":
+ return new_value
+ if new_value is None or new_value == "":
+ return old_value
+ logger.warn(
+ "Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
+ "). Latest value will be used")
+ return new_value
+
+ def get_new_date_value(self, old_value, column_name, new_value):
+ # type: (datetime,unicode,unicode) -> datetime
+ if old_value is None or old_value == "":
+ try:
+ result = datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+ except ValueError:
+ logger.warn("Invalid date: " + new_value)
+ result = old_value
+ return result
+ if new_value is None or new_value == "":
+ return old_value
+ logger.warn(
+ "Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
+ "). Latest value will be used")
+ return datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
diff --git a/smash/web/importer/exporter.py b/smash/web/importer/exporter.py
index 99b72d79a94443e3f04f4a7ad69f2c859da7b339..ebdf57df450478390716dc09161814b57f30c882 100644
--- a/smash/web/importer/exporter.py
+++ b/smash/web/importer/exporter.py
@@ -24,7 +24,7 @@ class Exporter(object):
with open(self.filename, 'w') as csv_file:
data = self.get_subjects_as_array()
- writer = csv.writer(csv_file, quotechar=str(u'"'), quoting=csv.QUOTE_ALL)
+ writer = csv.writer(csv_file, quotechar=str(u'"'))
for row in data:
writer.writerow([s.encode("utf-8") for s in row])
self.exported_count += 1
diff --git a/smash/web/importer/importer.py b/smash/web/importer/importer.py
index 2c7702f0de38fb9a29156e87b207b382f7a7b1ff..6c22dd9ad358cac02925815fbde4a1460d1f95c3 100644
--- a/smash/web/importer/importer.py
+++ b/smash/web/importer/importer.py
@@ -67,7 +67,7 @@ class Importer(object):
def import_study_subject(self, study_subject):
# type: (StudySubject) -> None
- db_study_subjects = StudySubject.objects.filter(screening_number=study_subject.screening_number)
+ db_study_subjects = StudySubject.objects.filter(nd_number=study_subject.nd_number)
if db_study_subjects.count() > 0:
db_study_subject = db_study_subjects.first()
for field in Subject._meta.get_fields():
diff --git a/smash/web/importer/importer_cron_job.py b/smash/web/importer/importer_cron_job.py
index bfb75d105c4d0db0daaea06034fa1a82946b182b..36df103ae0e3442b83549020bfee058a8e5dd634 100644
--- a/smash/web/importer/importer_cron_job.py
+++ b/smash/web/importer/importer_cron_job.py
@@ -9,7 +9,7 @@ import timeout_decorator
from django.conf import settings
from django_cron import CronJobBase, Schedule
-from csv_subject_import_reader import CsvSubjectImportReader
+from csv_tns_subject_import_reader import TnsCsvSubjectImportReader
from importer import Importer
from web.models.constants import CRON_JOB_TIMEOUT
from ..smash_email import EmailSender
@@ -18,8 +18,8 @@ logger = logging.getLogger(__name__)
class ImporterCronJob(CronJobBase):
- RUN_EVERY_MINUTES = 60 * 24
- schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
+ RUN_AT_TIMES = ['23:55']
+ schedule = Schedule(run_at_times=RUN_AT_TIMES)
code = 'web.import_daily_job' # a unique code
@timeout_decorator.timeout(CRON_JOB_TIMEOUT)
@@ -39,7 +39,7 @@ class ImporterCronJob(CronJobBase):
email_recipients)
return "import file not found"
try:
- importer = Importer(settings.DAILY_IMPORT_FILE, CsvSubjectImportReader())
+ importer = Importer(settings.DAILY_IMPORT_FILE, TnsCsvSubjectImportReader())
importer.execute()
email_body = importer.get_summary()
EmailSender().send_email(email_title,
diff --git a/smash/web/templates/_base.html b/smash/web/templates/_base.html
index ecc810b2bced77a1747bfe65c65e711ff1933541..015a026b1caf69c931fd05f65a2f51565e189320 100644
--- a/smash/web/templates/_base.html
+++ b/smash/web/templates/_base.html
@@ -260,7 +260,7 @@ desired effect
{% block footer %}
<!-- To the right -->
<div class="pull-right hidden-xs">
- Version: <strong>0.14.0</strong> (1 Apr 2020)
+ Version: <strong>0.15.0</strong> (7 Apr 2020)
</div>
<!-- Default to the left -->
diff --git a/smash/web/tests/data/tns_import.csv b/smash/web/tests/data/tns_import.csv
new file mode 100644
index 0000000000000000000000000000000000000000..588d1661d5f54369ed643d22a56db3c0c1784281
--- /dev/null
+++ b/smash/web/tests/data/tns_import.csv
@@ -0,0 +1,4 @@
+donor_id;firstname;lastname;dateofbirth;phonenr;treatingphysician
+Cov-000001;John;Doe;01/01/1977;555555;Gregory House
+Cov-000002;John2;Doe2;01/02/1977;621000000;Gregory House2
+Cov-000003;John2;Doe2;01/03/1977;691000000;Gregory House3
\ No newline at end of file
diff --git a/smash/web/tests/importer/test_importer.py b/smash/web/tests/importer/test_importer.py
index a5e95a1ea900a4a1742abece15e00cfea13160d2..602a5f299970826d89558c611e935914577d7a7e 100644
--- a/smash/web/tests/importer/test_importer.py
+++ b/smash/web/tests/importer/test_importer.py
@@ -84,6 +84,7 @@ class TestImporter(TestCase):
subject.date_born = datetime.datetime.now()
study_subject = StudySubject()
study_subject.screening_number = existing_study_subject.screening_number
+ study_subject.nd_number = existing_study_subject.nd_number
study_subject.subject = subject
study_subject.study = self.study
study_subjects.append(study_subject)
diff --git a/smash/web/tests/importer/test_importer_cron_job.py b/smash/web/tests/importer/test_importer_cron_job.py
index 0eec9f6acb0969b11f703639ee961f665b71ad13..8686ac27053f422180497245e397b76ef893f56d 100644
--- a/smash/web/tests/importer/test_importer_cron_job.py
+++ b/smash/web/tests/importer/test_importer_cron_job.py
@@ -31,7 +31,7 @@ class TestCronJobImporter(TestCase):
self.assertEqual(0, len(mail.outbox))
def test_import(self):
- filename = get_resource_path('import.csv')
+ filename = get_resource_path('tns_import.csv')
new_file, tmp = tempfile.mkstemp()
copyfile(filename, tmp)
diff --git a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bc8e31c6fe699b37b9f12df87e08c61aef54c43
--- /dev/null
+++ b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
@@ -0,0 +1,31 @@
+# coding=utf-8
+
+import logging
+
+from django.test import TestCase
+
+from web.importer import TnsCsvSubjectImportReader
+from web.tests.functions import get_resource_path
+
+logger = logging.getLogger(__name__)
+
+
+class TestTnsCsvReader(TestCase):
+
+ def test_load_data(self):
+ filename = get_resource_path('tns_import.csv')
+ study_subjects = TnsCsvSubjectImportReader().load_data(filename)
+ self.assertEqual(3, len(study_subjects))
+ study_subject = study_subjects[1]
+ self.assertEqual("John2", study_subject.subject.first_name)
+ self.assertEqual("Doe2", study_subject.subject.last_name)
+ self.assertEqual("Cov-000002", study_subject.screening_number)
+ self.assertEqual("Cov-000002", study_subject.nd_number)
+ self.assertEqual("621000000", study_subject.subject.phone_number)
+ self.assertTrue("Gregory House2" in study_subject.comments)
+
+ self.assertEqual(1, study_subject.subject.date_born.day)
+ self.assertEqual(2, study_subject.subject.date_born.month)
+ self.assertEqual(1977, study_subject.subject.date_born.year)
+
+ self.assertIsNotNone(study_subject.study)