Merge branch 'tns-importer' into 'master'

tns-csv importer implemented

See merge request NCER-PD/scheduling-system!235

smash/web/importer/__init__.py

 from csv_subject_import_reader import CsvSubjectImportReader
+from csv_tns_subject_import_reader import TnsCsvSubjectImportReader
 from exporter import Exporter
 from exporter_cron_job import ExporterCronJob
 from importer import Importer
 from importer_cron_job import ImporterCronJob
 from subject_import_reader import SubjectImportReader
 
-__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob, Exporter, ExporterCronJob]
+__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob, Exporter, ExporterCronJob,
+           TnsCsvSubjectImportReader]

smash/web/importer/csv_tns_subject_import_reader.py

+import csv
+import datetime
+import logging
+
+from subject_import_reader import SubjectImportReader
+from web.models import StudySubject, Subject, Study
+from web.models.constants import GLOBAL_STUDY_ID
+
+CSV_DATE_FORMAT = "%d/%m/%Y"
+
+logger = logging.getLogger(__name__)
+
+
+class TnsCsvSubjectImportReader(SubjectImportReader):
+    def __init__(self):
+        self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+
+    def load_data(self, filename):
+        study_subjects = []
+        with open(filename) as csv_file:
+            reader = csv.reader(csv_file, delimiter=';')
+            headers = next(reader, None)
+            for row in reader:
+                subject = Subject()
+                study_subject = StudySubject()
+                study_subject.subject = subject
+                study_subject.study = self.study
+                for header, value in zip(headers, row):
+                    self.add_data(study_subject, header, value)
+                if study_subject.nd_number is None or study_subject.nd_number == "":
+                    study_subject.nd_number = study_subject.screening_number
+                study_subjects.append(study_subject)
+        return study_subjects
+
+    def add_data(self, study_subject, column_name, value):
+        # type: (StudySubject, str, str) ->  None
+        if column_name == "firstname":
+            study_subject.subject.first_name = self.get_new_value(study_subject.subject.first_name, column_name, value)
+        elif column_name == "lastname":
+            study_subject.subject.last_name = self.get_new_value(study_subject.subject.last_name, column_name, value)
+        elif column_name == "donor_id":
+            study_subject.screening_number = self.get_new_value(study_subject.screening_number, column_name, value)
+        elif column_name == "phonenr":
+            study_subject.subject.phone_number = self.get_new_value(study_subject.subject.phone_number, column_name,
+                                                                    value)
+        elif column_name == "treatingphysician":
+            if value is not None and value != "":
+                value = "Treating physician: " + value
+            study_subject.comments = self.get_new_value(study_subject.comments, column_name, value)
+        elif column_name == "dateofbirth":
+            study_subject.subject.date_born = self.get_new_date_value(study_subject.subject.date_born, column_name,
+                                                                      value)
+        else:
+            logger.warn("Don't know how to handle column " + column_name + " with data " + value)
+
+    def get_new_value(self, old_value, column_name, new_value):
+        # type: (unicode,unicode,unicode) ->  unicode
+        if old_value is None or old_value == "":
+            return new_value
+        if new_value is None or new_value == "":
+            return old_value
+        logger.warn(
+            "Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
+            "). Latest value will be used")
+        return new_value
+
+    def get_new_date_value(self, old_value, column_name, new_value):
+        # type: (datetime,unicode,unicode) ->  datetime
+        if old_value is None or old_value == "":
+            try:
+                result = datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)
+            except ValueError:
+                logger.warn("Invalid date: " + new_value)
+                result = old_value
+            return result
+        if new_value is None or new_value == "":
+            return old_value
+        logger.warn(
+            "Contradicting entries in csv file for column: " + column_name + "(" + new_value + "," + old_value +
+            "). Latest value will be used")
+        return datetime.datetime.strptime(new_value, CSV_DATE_FORMAT)

smash/web/importer/exporter.py

@@ -24,7 +24,7 @@ class Exporter(object):
 
         with open(self.filename, 'w') as csv_file:
             data = self.get_subjects_as_array()
-            writer = csv.writer(csv_file, quotechar=str(u'"'), quoting=csv.QUOTE_ALL)
+            writer = csv.writer(csv_file, quotechar=str(u'"'))
             for row in data:
                 writer.writerow([s.encode("utf-8") for s in row])
                 self.exported_count += 1

smash/web/importer/importer.py

@@ -67,7 +67,7 @@ class Importer(object):
 
     def import_study_subject(self, study_subject):
         # type: (StudySubject) -> None
-        db_study_subjects = StudySubject.objects.filter(screening_number=study_subject.screening_number)
+        db_study_subjects = StudySubject.objects.filter(nd_number=study_subject.nd_number)
         if db_study_subjects.count() > 0:
             db_study_subject = db_study_subjects.first()
             for field in Subject._meta.get_fields():

smash/web/importer/importer_cron_job.py

@@ -9,7 +9,7 @@ import timeout_decorator
 from django.conf import settings
 from django_cron import CronJobBase, Schedule
 
-from csv_subject_import_reader import CsvSubjectImportReader
+from csv_tns_subject_import_reader import TnsCsvSubjectImportReader
 from importer import Importer
 from web.models.constants import CRON_JOB_TIMEOUT
 from ..smash_email import EmailSender
@@ -18,8 +18,8 @@ logger = logging.getLogger(__name__)
 
 
 class ImporterCronJob(CronJobBase):
-    RUN_EVERY_MINUTES = 60 * 24
-    schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
+    RUN_AT_TIMES = ['23:55']
+    schedule = Schedule(run_at_times=RUN_AT_TIMES)
     code = 'web.import_daily_job'  # a unique code
 
     @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
@@ -39,7 +39,7 @@ class ImporterCronJob(CronJobBase):
                                      email_recipients)
             return "import file not found"
         try:
-            importer = Importer(settings.DAILY_IMPORT_FILE, CsvSubjectImportReader())
+            importer = Importer(settings.DAILY_IMPORT_FILE, TnsCsvSubjectImportReader())
             importer.execute()
             email_body = importer.get_summary()
             EmailSender().send_email(email_title,

smash/web/templates/_base.html

@@ -260,7 +260,7 @@ desired effect
         {% block footer %}
             <!-- To the right -->
             <div class="pull-right hidden-xs">
-                Version: <strong>0.14.0</strong> (1 Apr 2020)
+                Version: <strong>0.15.0</strong> (7 Apr 2020)
             </div>
 
             <!-- Default to the left -->

smash/web/tests/data/tns_import.csv

+donor_id;firstname;lastname;dateofbirth;phonenr;treatingphysician
+Cov-000001;John;Doe;01/01/1977;555555;Gregory House
+Cov-000002;John2;Doe2;01/02/1977;621000000;Gregory House2
+Cov-000003;John2;Doe2;01/03/1977;691000000;Gregory House3
\ No newline at end of file

smash/web/tests/importer/test_importer.py

@@ -84,6 +84,7 @@ class TestImporter(TestCase):
         subject.date_born = datetime.datetime.now()
         study_subject = StudySubject()
         study_subject.screening_number = existing_study_subject.screening_number
+        study_subject.nd_number = existing_study_subject.nd_number
         study_subject.subject = subject
         study_subject.study = self.study
         study_subjects.append(study_subject)

smash/web/tests/importer/test_importer_cron_job.py

@@ -31,7 +31,7 @@ class TestCronJobImporter(TestCase):
         self.assertEqual(0, len(mail.outbox))
 
     def test_import(self):
-        filename = get_resource_path('import.csv')
+        filename = get_resource_path('tns_import.csv')
         new_file, tmp = tempfile.mkstemp()
         copyfile(filename, tmp)
 

smash/web/tests/importer/test_tns_csv_subject_import_reader.py

+# coding=utf-8
+
+import logging
+
+from django.test import TestCase
+
+from web.importer import TnsCsvSubjectImportReader
+from web.tests.functions import get_resource_path
+
+logger = logging.getLogger(__name__)
+
+
+class TestTnsCsvReader(TestCase):
+
+    def test_load_data(self):
+        filename = get_resource_path('tns_import.csv')
+        study_subjects = TnsCsvSubjectImportReader().load_data(filename)
+        self.assertEqual(3, len(study_subjects))
+        study_subject = study_subjects[1]
+        self.assertEqual("John2", study_subject.subject.first_name)
+        self.assertEqual("Doe2", study_subject.subject.last_name)
+        self.assertEqual("Cov-000002", study_subject.screening_number)
+        self.assertEqual("Cov-000002", study_subject.nd_number)
+        self.assertEqual("621000000", study_subject.subject.phone_number)
+        self.assertTrue("Gregory House2" in study_subject.comments)
+
+        self.assertEqual(1, study_subject.subject.date_born.day)
+        self.assertEqual(2, study_subject.subject.date_born.month)
+        self.assertEqual(1977, study_subject.subject.date_born.year)
+
+        self.assertIsNotNone(study_subject.study)