diff --git a/smash/smash/settings.py b/smash/smash/settings.py
index 14f7a7ff96dc662eadb5b2216029619d0f16cc28..7cd843ec0ac63af7806278d35bb724be47864c62 100644
--- a/smash/smash/settings.py
+++ b/smash/smash/settings.py
@@ -76,7 +76,9 @@ CRON_CLASSES = [
'web.views.kit.KitRequestEmailSendJob',
'web.redcap_connector.RedCapRefreshJob',
'web.views.voucher.ExpireVouchersJob',
- 'web.importer.importer_cron_job.ImporterCronJob'
+ 'web.importer.exporter_cron_job.ExporterCronJob',
+ 'web.importer.importer_cron_job.SubjectImporterCronJob',
+ 'web.importer.importer_cron_job.VisitImporterCronJob'
]
# Password validation
diff --git a/smash/web/importer/__init__.py b/smash/web/importer/__init__.py
index 99f8f1dedcee0573a8c4d440f6b98d97e8d973ea..f60887a0d48238dced793d829f0dd82eaabb9cb8 100644
--- a/smash/web/importer/__init__.py
+++ b/smash/web/importer/__init__.py
@@ -1,10 +1,12 @@
from csv_subject_import_reader import CsvSubjectImportReader
from csv_tns_subject_import_reader import TnsCsvSubjectImportReader
+from csv_tns_visit_import_reader import TnsCsvVisitImportReader
from exporter import Exporter
from exporter_cron_job import ExporterCronJob
from importer import Importer
-from importer_cron_job import ImporterCronJob
+from importer_cron_job import SubjectImporterCronJob, VisitImporterCronJob
from subject_import_reader import SubjectImportReader
+from warning_counter import MsgCounterHandler
-__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob, Exporter, ExporterCronJob,
- TnsCsvSubjectImportReader]
+__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, SubjectImporterCronJob, VisitImporterCronJob,
+ Exporter, ExporterCronJob, TnsCsvSubjectImportReader, TnsCsvVisitImportReader, MsgCounterHandler]
diff --git a/smash/web/importer/csv_tns_visit_import_reader.py b/smash/web/importer/csv_tns_visit_import_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..185ac5b02436feccf34410aa9aeae068f3e745e5
--- /dev/null
+++ b/smash/web/importer/csv_tns_visit_import_reader.py
@@ -0,0 +1,116 @@
+import csv
+import datetime
+import logging
+
+import pytz
+from django.conf import settings
+
+from warning_counter import MsgCounterHandler
+from web.models import StudySubject, Study, Visit, Appointment, AppointmentType, Location, AppointmentTypeLink
+from web.models.constants import GLOBAL_STUDY_ID
+
+CSV_DATE_FORMAT = "%d/%m/%Y"
+
+logger = logging.getLogger(__name__)
+
+
+class TnsCsvVisitImportReader:
+ def __init__(self):
+ self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
+ appointment_code = getattr(settings, "IMPORT_APPOINTMENT_TYPE", "SAMPLES")
+
+ appointment_types = AppointmentType.objects.filter(code=appointment_code)
+ if len(appointment_types) > 0:
+ self.appointment_type = appointment_types[0]
+ else:
+ logger.warn("Appointment type does not exist: " + appointment_code)
+ self.appointment_type = None
+ self.problematic_count = 0
+ self.processed_count = 0
+ self.warning_count = 0
+
+ def load_data(self, filename):
+ warning_counter = MsgCounterHandler()
+ logging.getLogger('').addHandler(warning_counter)
+
+ result = []
+ with open(filename) as csv_file:
+ reader = csv.reader(csv_file, delimiter=';')
+ headers = next(reader, None)
+ for row in reader:
+ try:
+ data = {}
+ for h, v in zip(headers, row):
+ data[h] = v
+ nd_number = data['donor_id']
+ study_subjects = StudySubject.objects.filter(nd_number=nd_number)
+ if len(study_subjects) == 0:
+ raise NotImplementedError
+ study_subject = study_subjects[0]
+ visit_number = data['visit_id']
+ visit_number = int(visit_number) + 1
+ visits = Visit.objects.filter(subject=study_subject, visit_number=visit_number)
+
+ if len(visits) > 0:
+ raise NotImplementedError
+
+ date = self.extract_date(data['dateofvisit'])
+ visit = Visit.objects.create(subject=study_subject, visit_number=visit_number, datetime_begin=date,
+ datetime_end=date + datetime.timedelta(days=14))
+ visit.save()
+ result.append(visit)
+
+ location = self.extract_location(data['adressofvisit'])
+
+ appointment = Appointment.objects.create(visit=visit, length=60, datetime_when=date,
+ location=location)
+ if self.appointment_type is not None:
+ AppointmentTypeLink.objects.create(appointment_id=appointment.id,
+ appointment_type=self.appointment_type)
+ self.processed_count += 1
+ except:
+ self.problematic_count += 1
+ logger.warn("Problematic data: " + ';'.join(row))
+
+ if "WARNING" in warning_counter.level2count:
+ self.warning_count = warning_counter.level2count["WARNING"]
+ logging.getLogger('').removeHandler(warning_counter)
+
+ return result
+
+ def extract_date(self, text):
+ # type: (unicode) -> datetime
+
+ # by default use day after tomorrow
+ result = datetime.datetime.now().replace(hour=9, minute=0) + datetime.timedelta(days=2)
+ try:
+ year = int(text[:4])
+ month = int(text[4:6])
+ day = int(text[6:8])
+ result = result.replace(year=year, month=month, day=day, tzinfo=pytz.UTC)
+ except ValueError:
+ logger.warn("Invalid date: " + text)
+ return result
+
+ def extract_location(self, text):
+ # type: (unicode) -> Location
+
+ locations = Location.objects.filter(name=text)
+ if len(locations) > 0:
+ return locations[0]
+ else:
+ logger.warn("Location with name does not exist: " + text)
+ return Location.objects.create(name=text)
+
+ def get_summary(self):
+ result = "<p>Number of successfully added appointments: <b>" + str(self.processed_count) + "</b></p>"
+ style = ''
+ if self.problematic_count > 0:
+ style = ' color="red" '
+ result += "<p><font " + style + ">Number of problematic appointments: <b>" + str(
+ self.problematic_count) + "</b></font></p>"
+ style = ''
+ if self.warning_count > 0:
+ style = ' color="brown" '
+ result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"
+ return result
diff --git a/smash/web/importer/importer_cron_job.py b/smash/web/importer/importer_cron_job.py
index ef0bfd123cc493d004298edd52b3c5b641abae36..670130d4d047e252744cea08f6b026582bdc5f76 100644
--- a/smash/web/importer/importer_cron_job.py
+++ b/smash/web/importer/importer_cron_job.py
@@ -10,6 +10,7 @@ from django.conf import settings
from django_cron import CronJobBase, Schedule
from csv_tns_subject_import_reader import TnsCsvSubjectImportReader
+from csv_tns_visit_import_reader import TnsCsvVisitImportReader
from importer import Importer
from web.models.constants import CRON_JOB_TIMEOUT
from ..smash_email import EmailSender
@@ -17,14 +18,14 @@ from ..smash_email import EmailSender
logger = logging.getLogger(__name__)
-class ImporterCronJob(CronJobBase):
- RUN_AT_TIMES = getattr(settings, "IMPORT_RUN_AT", ['23:55'])
+class SubjectImporterCronJob(CronJobBase):
+ RUN_AT_TIMES = getattr(settings, "IMPORT_RUN_AT", ['23:45'])
schedule = Schedule(run_at_times=RUN_AT_TIMES)
- code = 'web.import_daily_job' # a unique code
+ code = 'web.import_subjects_daily_job' # a unique code
@timeout_decorator.timeout(CRON_JOB_TIMEOUT)
def do(self):
- email_title = "Daily import"
+ email_title = "Subjects daily import"
email_recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
filename = getattr(settings, "DAILY_IMPORT_FILE", None)
@@ -59,3 +60,47 @@ class ImporterCronJob(CronJobBase):
new_file = filename + "-" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") + ".bac"
os.rename(filename, new_file)
return
+
+
+class VisitImporterCronJob(CronJobBase):
+ RUN_AT_TIMES = getattr(settings, "IMPORT_RUN_AT", ['23:55'])
+ schedule = Schedule(run_at_times=RUN_AT_TIMES)
+ code = 'web.import_visits_daily_job' # a unique code
+
+ @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
+ def do(self):
+ email_title = "Visits daily import"
+ email_recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
+
+ filename = getattr(settings, "DAILY_VISIT_IMPORT_FILE", None)
+
+ if filename is None:
+ logger.info("Importing visits skipped. File not defined ")
+ return "import file not defined"
+ logger.info("Importing visits from file: " + filename)
+ if not os.path.isfile(filename):
+ EmailSender().send_email(email_title,
+ "<h3><font color='red'>File with imported data is not available in the system: " + filename + "</font></h3>",
+ email_recipients)
+ return "import file not found"
+ try:
+ importer = TnsCsvVisitImportReader()
+ importer.load_data(filename)
+ email_body = importer.get_summary()
+ EmailSender().send_email(email_title,
+ "<h3>Data was successfully imported from file: " + filename + "</h3>" + email_body,
+ email_recipients)
+ self.backup_file(filename)
+ return "import is successful"
+
+ except:
+ tb = traceback.format_exc()
+ EmailSender().send_email(email_title,
+ "<h3><font color='red'>There was a problem with importing data from file: " + filename + "</font></h3><pre>" + tb + "</pre>",
+ email_recipients)
+ return "import crashed"
+
+ def backup_file(self, filename):
+ new_file = filename + "-" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") + ".bac"
+ os.rename(filename, new_file)
+ return
diff --git a/smash/web/migrations/0157_auto_20200414_0909.py b/smash/web/migrations/0157_auto_20200414_0909.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1c759826e330b4299e320882a29413dd3768265
--- /dev/null
+++ b/smash/web/migrations/0157_auto_20200414_0909.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2020-04-14 09:09
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('web', '0156_auto_20200406_1207'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='location',
+ name='name',
+ field=models.CharField(max_length=256),
+ ),
+ ]
diff --git a/smash/web/models/location.py b/smash/web/models/location.py
index 5720f26897115e0148138e130988a8b2cbfd9181..217a699d2c0495d7a373ec3b3be68881f6265ccf 100644
--- a/smash/web/models/location.py
+++ b/smash/web/models/location.py
@@ -6,7 +6,7 @@ class Location(models.Model):
class Meta:
app_label = 'web'
- name = models.CharField(max_length=20)
+ name = models.CharField(max_length=256)
color = models.CharField(max_length=20,
verbose_name='Calendar appointment color',
diff --git a/smash/web/tests/data/tns_import.csv b/smash/web/tests/data/tns_import.csv
deleted file mode 100644
index 588d1661d5f54369ed643d22a56db3c0c1784281..0000000000000000000000000000000000000000
--- a/smash/web/tests/data/tns_import.csv
+++ /dev/null
@@ -1,4 +0,0 @@
-donor_id;firstname;lastname;dateofbirth;phonenr;treatingphysician
-Cov-000001;John;Doe;01/01/1977;555555;Gregory House
-Cov-000002;John2;Doe2;01/02/1977;621000000;Gregory House2
-Cov-000003;John2;Doe2;01/03/1977;691000000;Gregory House3
\ No newline at end of file
diff --git a/smash/web/tests/data/tns_subjects_import.csv b/smash/web/tests/data/tns_subjects_import.csv
new file mode 100644
index 0000000000000000000000000000000000000000..4ae65947d840665eaec52d4320b35be5f57397e1
--- /dev/null
+++ b/smash/web/tests/data/tns_subjects_import.csv
@@ -0,0 +1,4 @@
+donor_id;firstname;lastname;dateofbirth;phonenr;treatingphysician;sig_firstname;sig_lastname;representative
+cov-000111;John;Doe;01/01/1977;555555;Gregory House;John;Doe;Mario Doe
+cov-222333;John2;Doe2;01/02/1977;621000000;Gregory House2;John 2;Doe 2;
+cov-444444;John2;Doe2;01/03/1977;691000000;Gregory House3;John 3;Doe 3;Elsa Doe
\ No newline at end of file
diff --git a/smash/web/tests/data/tns_vouchers_import.csv b/smash/web/tests/data/tns_vouchers_import.csv
new file mode 100644
index 0000000000000000000000000000000000000000..5ea0fe27ede1bcef29aca8082d3fc20c14772ed8
--- /dev/null
+++ b/smash/web/tests/data/tns_vouchers_import.csv
@@ -0,0 +1,4 @@
+donor_id;visit_id;dateofvisit;adressofvisit
+cov-000111;0;20200410;"Laboratoires réunis 23 Route de Diekirch 6555 Bollendorf-Pont"
+cov-222333;0;20200410;PickenDoheem
+cov-444444;0;20200410;"Ketterthill 1-3, rue de la Continentale 4917 Bascharage"
\ No newline at end of file
diff --git a/smash/web/tests/importer/test_importer_cron_job.py b/smash/web/tests/importer/test_importer_cron_job.py
index 8686ac27053f422180497245e397b76ef893f56d..861a839e47c75317571f0008fad464be61b57419 100644
--- a/smash/web/tests/importer/test_importer_cron_job.py
+++ b/smash/web/tests/importer/test_importer_cron_job.py
@@ -7,7 +7,7 @@ from shutil import copyfile
from django.conf import settings
from django.test import TestCase
-from web.importer import ImporterCronJob
+from web.importer import SubjectImporterCronJob
from web.tests.functions import get_resource_path
logger = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ class TestCronJobImporter(TestCase):
def test_import_without_configuration(self):
CronJobLog.objects.all().delete()
- job = ImporterCronJob()
+ job = SubjectImporterCronJob()
status = job.do()
@@ -31,14 +31,14 @@ class TestCronJobImporter(TestCase):
self.assertEqual(0, len(mail.outbox))
def test_import(self):
- filename = get_resource_path('tns_import.csv')
+ filename = get_resource_path('tns_subjects_import.csv')
new_file, tmp = tempfile.mkstemp()
copyfile(filename, tmp)
setattr(settings, "DAILY_IMPORT_FILE", tmp)
CronJobLog.objects.all().delete()
- job = ImporterCronJob()
+ job = SubjectImporterCronJob()
status = job.do()
diff --git a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
index 0bc8e31c6fe699b37b9f12df87e08c61aef54c43..a713f5b1b581491bda3ecebcbeaf859edede4a62 100644
--- a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
+++ b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
@@ -10,17 +10,17 @@ from web.tests.functions import get_resource_path
logger = logging.getLogger(__name__)
-class TestTnsCsvReader(TestCase):
+class TestTnsCsvSubjectReader(TestCase):
def test_load_data(self):
- filename = get_resource_path('tns_import.csv')
+ filename = get_resource_path('tns_subjects_import.csv')
study_subjects = TnsCsvSubjectImportReader().load_data(filename)
self.assertEqual(3, len(study_subjects))
study_subject = study_subjects[1]
self.assertEqual("John2", study_subject.subject.first_name)
self.assertEqual("Doe2", study_subject.subject.last_name)
- self.assertEqual("Cov-000002", study_subject.screening_number)
- self.assertEqual("Cov-000002", study_subject.nd_number)
+ self.assertEqual("cov-222333", study_subject.screening_number)
+ self.assertEqual("cov-222333", study_subject.nd_number)
self.assertEqual("621000000", study_subject.subject.phone_number)
self.assertTrue("Gregory House2" in study_subject.comments)
diff --git a/smash/web/tests/importer/test_tns_csv_visit_import_reader.py b/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..d93f37bf78186808d37f7628bb33dfc6f9d21b27
--- /dev/null
+++ b/smash/web/tests/importer/test_tns_csv_visit_import_reader.py
@@ -0,0 +1,52 @@
+# coding=utf-8
+
+import logging
+
+from django.conf import settings
+from django.test import TestCase
+
+from web.importer import TnsCsvVisitImportReader, MsgCounterHandler
+from web.models import Appointment, Visit
+from web.tests.functions import get_resource_path, create_study_subject, create_appointment_type, create_location
+
+logger = logging.getLogger(__name__)
+
+
+class TestTnsCsvSubjectReader(TestCase):
+ def setUp(self):
+ self.warning_counter = MsgCounterHandler()
+ logging.getLogger('').addHandler(self.warning_counter)
+ setattr(settings, "IMPORT_APPOINTMENT_TYPE", "SAMPLE_2")
+ create_appointment_type(code = "SAMPLE_2")
+
+
+ def tearDown(self):
+ setattr(settings, "IMPORT_APPOINTMENT_TYPE", None)
+ logging.getLogger('').removeHandler(self.warning_counter)
+
+ def test_load_data(self):
+ create_study_subject(nd_number='cov-000111')
+ create_study_subject(nd_number='cov-222333')
+ create_study_subject(nd_number='cov-444444')
+
+ create_location(name=u"Laboratoires réunis 23 Route de Diekirch 6555 Bollendorf-Pont")
+ create_location(name=u"PickenDoheem")
+ create_location(name=u"Ketterthill 1-3, rue de la Continentale 4917 Bascharage")
+
+ filename = get_resource_path('tns_vouchers_import.csv')
+ visits = TnsCsvVisitImportReader().load_data(filename)
+ self.assertEqual(3, len(visits))
+ visit = Visit.objects.filter(id=visits[0].id)[0]
+ self.assertEqual("cov-000111", visit.subject.nd_number)
+ self.assertEqual(1, visit.visit_number)
+
+ appointment = Appointment.objects.filter(visit=visit)[0]
+ self.assertEqual(u"Laboratoires réunis 23 Route de Diekirch 6555 Bollendorf-Pont",
+ appointment.location.name)
+
+ self.assertEqual(10, appointment.datetime_when.day)
+ self.assertEqual(4, appointment.datetime_when.month)
+ self.assertEqual(2020, appointment.datetime_when.year)
+
+ if "WARNING" in self.warning_counter.level2count:
+ self.assertEquals(0, self.warning_counter.level2count["WARNING"])