From 66d890f1cf02f48d5425bb031ac0d61580d17dd1 Mon Sep 17 00:00:00 2001
From: Piotr Gawron <piotr.gawron@uni.lu>
Date: Wed, 18 Nov 2020 12:30:18 +0100
Subject: [PATCH] SubjectImportReader requires SubjectImportData parameter
---
.../web/importer/csv_subject_import_reader.py | 5 +-
.../importer/csv_tns_subject_import_reader.py | 9 ++-
smash/web/importer/importer.py | 4 +-
smash/web/importer/importer_cron_job.py | 73 ++++++++++---------
smash/web/importer/subject_import_reader.py | 6 ++
.../0180_visitimportdata_migration.py | 29 +++++++-
.../test_csv_subject_import_reader.py | 15 ++--
.../tests/importer/test_importer_cron_job.py | 18 ++---
.../test_tns_csv_subject_import_reader.py | 10 ++-
9 files changed, 107 insertions(+), 62 deletions(-)
diff --git a/smash/web/importer/csv_subject_import_reader.py b/smash/web/importer/csv_subject_import_reader.py
index 9f95f0c3..77c71ab6 100644
--- a/smash/web/importer/csv_subject_import_reader.py
+++ b/smash/web/importer/csv_subject_import_reader.py
@@ -3,7 +3,7 @@ import datetime
import logging
from .subject_import_reader import SubjectImportReader
-from web.models import StudySubject, Subject, Study
+from web.models import StudySubject, Subject, Study, SubjectImportData
from web.models.constants import GLOBAL_STUDY_ID
CSV_DATE_FORMAT = "%d-%m-%Y"
@@ -12,7 +12,8 @@ logger = logging.getLogger(__name__)
class CsvSubjectImportReader(SubjectImportReader):
- def __init__(self):
+ def __init__(self, import_data: SubjectImportData):
+ super().__init__(import_data)
self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
def load_data(self, filename):
diff --git a/smash/web/importer/csv_tns_subject_import_reader.py b/smash/web/importer/csv_tns_subject_import_reader.py
index c20862be..38c539a9 100644
--- a/smash/web/importer/csv_tns_subject_import_reader.py
+++ b/smash/web/importer/csv_tns_subject_import_reader.py
@@ -1,11 +1,11 @@
+import codecs
import csv
import datetime
import logging
-import codecs
+from web.models import StudySubject, Subject, Study, SubjectImportData
+from web.models.constants import GLOBAL_STUDY_ID
from .subject_import_reader import SubjectImportReader
-from ..models import StudySubject, Subject, Study
-from ..models.constants import GLOBAL_STUDY_ID
CSV_DATE_FORMAT = "%d/%m/%Y"
@@ -13,7 +13,8 @@ logger = logging.getLogger(__name__)
class TnsCsvSubjectImportReader(SubjectImportReader):
- def __init__(self):
+ def __init__(self, import_data: SubjectImportData):
+ super().__init__(import_data)
self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
def load_data(self, filename):
diff --git a/smash/web/importer/importer.py b/smash/web/importer/importer.py
index 00d2bf1d..95b42414 100644
--- a/smash/web/importer/importer.py
+++ b/smash/web/importer/importer.py
@@ -14,8 +14,7 @@ logger = logging.getLogger(__name__)
class Importer(object):
- def __init__(self, filename, reader):
- # type: (str, SubjectImportReader) -> None
+ def __init__(self, filename, reader: SubjectImportReader):
self.filename = filename
self.reader = reader
self.added_count = 0
@@ -34,7 +33,6 @@ class Importer(object):
else:
self.importer_user = Worker.objects.filter(user=user)
-
def execute(self):
self.added_count = 0
self.problematic_count = 0
diff --git a/smash/web/importer/importer_cron_job.py b/smash/web/importer/importer_cron_job.py
index ce0bdfa8..9eadfafb 100644
--- a/smash/web/importer/importer_cron_job.py
+++ b/smash/web/importer/importer_cron_job.py
@@ -9,9 +9,8 @@ import timeout_decorator
from django.db import OperationalError, ProgrammingError
from django_cron import CronJobBase, Schedule
-from web.models import ConfigurationItem, Study, VisitImportData
-from web.models.constants import CRON_JOB_TIMEOUT, DEFAULT_FROM_EMAIL, DAILY_SUBJECT_IMPORT_FILE, \
- SUBJECT_IMPORT_RUN_AT, GLOBAL_STUDY_ID
+from web.models import ConfigurationItem, Study, VisitImportData, SubjectImportData
+from web.models.constants import CRON_JOB_TIMEOUT, DEFAULT_FROM_EMAIL, GLOBAL_STUDY_ID
from web.smash_email import EmailSender
from .csv_tns_subject_import_reader import TnsCsvSubjectImportReader
from .csv_tns_visit_import_reader import TnsCsvVisitImportReader
@@ -23,46 +22,55 @@ logger = logging.getLogger(__name__)
class SubjectImporterCronJob(CronJobBase):
RUN_AT_TIMES = []
try:
- item = ConfigurationItem.objects.filter(type=SUBJECT_IMPORT_RUN_AT).first()
+ item = SubjectImportData.objects.filter(study_id=GLOBAL_STUDY_ID).first()
if item is not None:
- RUN_AT_TIMES = item.value.split(';')
+ RUN_AT_TIMES = item.run_at_times.split(';')
except (OperationalError, ProgrammingError): # sqlite and postgres throw different errors here
logger.debug('Looks like db is not initialized')
schedule = Schedule(run_at_times=RUN_AT_TIMES)
code = 'web.import_subjects_daily_job' # a unique code
+ def __init__(self, study_id=GLOBAL_STUDY_ID):
+ super().__init__()
+ self.study = Study.objects.get(pk=study_id)
+
@timeout_decorator.timeout(CRON_JOB_TIMEOUT)
def do(self):
email_title = "Subjects daily import"
email_recipients = ConfigurationItem.objects.get(type=DEFAULT_FROM_EMAIL).value
- filename = ConfigurationItem.objects.get(type=DAILY_SUBJECT_IMPORT_FILE).value
-
- if filename is None or filename == '':
- logger.info("Importing subjects skipped. File not defined ")
- return "import file not defined"
- logger.info("Importing subjects from file: " + filename)
- if not os.path.isfile(filename):
- EmailSender().send_email(email_title,
- "<h3><font color='red'>File with imported data is not available in the system: " + filename + "</font></h3>",
- email_recipients)
- return "import file not found"
- try:
- importer = Importer(filename, TnsCsvSubjectImportReader())
- importer.execute()
- email_body = importer.get_summary()
- EmailSender().send_email(email_title,
- "<h3>Data was successfully imported from file: " + filename + "</h3>" + email_body,
- email_recipients)
- backup_file(filename)
- return "import is successful"
-
- except:
- tb = traceback.format_exc()
- EmailSender().send_email(email_title,
- "<h3><font color='red'>There was a problem with importing data from file: " + filename + "</font></h3><pre>" + tb + "</pre>",
- email_recipients)
- return "import crashed"
+ for import_data in SubjectImportData.objects.filter(study=self.study).all():
+ if import_data.filename is None or import_data.filename == '':
+ logger.info("Importing subjects skipped. File not defined ")
+ return "import file not defined"
+ filename = import_data.get_absolute_file_path()
+ logger.info("Importing subjects from file: " + filename)
+ if not os.path.isfile(filename):
+ content = "<h3><font color='red'>File with imported data is not available in the system: " + \
+ filename + "</font></h3>"
+ EmailSender().send_email(email_title,
+ content,
+ email_recipients)
+ return "import file not found"
+ # noinspection PyBroadException
+ try:
+ importer = Importer(filename, TnsCsvSubjectImportReader(import_data))
+ importer.execute()
+ email_body = importer.get_summary()
+ EmailSender().send_email(email_title,
+ "<h3>Data was successfully imported from file: " + filename + "</h3>" +
+ email_body,
+ email_recipients)
+ backup_file(filename)
+ return "import is successful"
+
+ except BaseException:
+ tb = traceback.format_exc()
+ EmailSender().send_email(email_title,
+ "<h3><font color='red'>There was a problem with importing data from file: " +
+ filename + "</font></h3><pre>" + tb + "</pre>",
+ email_recipients)
+ return "import crashed"
class VisitImporterCronJob(CronJobBase):
@@ -116,7 +124,6 @@ class VisitImporterCronJob(CronJobBase):
"<h3><font color='red'>There was a problem with importing data from file: "
+ filename + "</font></h3><pre>" + tb + "</pre>",
email_recipients)
- print(tb)
return "import crashed"
diff --git a/smash/web/importer/subject_import_reader.py b/smash/web/importer/subject_import_reader.py
index 0bc7f898..187c3d03 100644
--- a/smash/web/importer/subject_import_reader.py
+++ b/smash/web/importer/subject_import_reader.py
@@ -1,7 +1,13 @@
+from typing import List
+
+from web.models import SubjectImportData
from web.models.study_subject import StudySubject
class SubjectImportReader:
+ def __init__(self, import_data: SubjectImportData):
+ self.import_data = import_data
+
def load_data(self, filename):
# type: (str) -> List [StudySubject]
pass
diff --git a/smash/web/migrations/0180_visitimportdata_migration.py b/smash/web/migrations/0180_visitimportdata_migration.py
index 364f5f73..8e13d955 100644
--- a/smash/web/migrations/0180_visitimportdata_migration.py
+++ b/smash/web/migrations/0180_visitimportdata_migration.py
@@ -1,7 +1,7 @@
from django.db import migrations
from web.models.constants import IMPORTER_USER, GLOBAL_STUDY_ID, IMPORT_APPOINTMENT_TYPE, DAILY_VISIT_IMPORT_FILE, \
- VISIT_IMPORT_RUN_AT
+ VISIT_IMPORT_RUN_AT, DAILY_SUBJECT_IMPORT_FILE, SUBJECT_IMPORT_RUN_AT
def get_val(apps, item_type: str):
@@ -47,6 +47,32 @@ def create_visit_import_data(apps, schema_editor):
entry.save()
+# noinspection PyUnusedLocal
+def create_subject_import_data(apps, schema_editor):
+ # noinspection PyPep8Naming
+ SubjectImportData = apps.get_model("web", "SubjectImportData")
+ # noinspection PyPep8Naming
+ Study = apps.get_model("web", "Study")
+ # noinspection PyPep8Naming
+ Worker = apps.get_model("web", "Worker")
+
+ entry = SubjectImportData()
+ entry.study = Study.objects.get(pk=GLOBAL_STUDY_ID)
+ importer_user_name = get_val(apps, IMPORTER_USER)
+ import_file = get_val(apps, DAILY_SUBJECT_IMPORT_FILE)
+ import_file_run_at = get_val(apps, SUBJECT_IMPORT_RUN_AT)
+
+ if importer_user_name is not None:
+ workers = Worker.objects.filter(user__username=importer_user_name)
+
+ if len(workers) > 0:
+ entry.worker = workers[0]
+
+ entry.filename = import_file
+ entry.run_at_times = import_file_run_at
+ entry.save()
+
+
class Migration(migrations.Migration):
dependencies = [
('web', '0179_visitimportdata'),
@@ -54,4 +80,5 @@ class Migration(migrations.Migration):
operations = [
migrations.RunPython(create_visit_import_data),
+ migrations.RunPython(create_subject_import_data),
]
diff --git a/smash/web/tests/importer/test_csv_subject_import_reader.py b/smash/web/tests/importer/test_csv_subject_import_reader.py
index a80f3234..3f11f773 100644
--- a/smash/web/tests/importer/test_csv_subject_import_reader.py
+++ b/smash/web/tests/importer/test_csv_subject_import_reader.py
@@ -4,17 +4,21 @@ import logging
from django.test import TestCase
+from web.models import SubjectImportData
from web.importer import CsvSubjectImportReader
-from web.tests.functions import get_resource_path
+from web.tests.functions import get_resource_path, get_test_study
logger = logging.getLogger(__name__)
class TestCsvReader(TestCase):
+ def setUp(self):
+ self.study_import_data = SubjectImportData.objects.create(study=get_test_study())
def test_load_data(self):
- filename = get_resource_path('import.csv')
- study_subjects = CsvSubjectImportReader().load_data(filename)
+ self.study_import_data.filename = get_resource_path('import.csv')
+ study_subjects = CsvSubjectImportReader(self.study_import_data).load_data(self.study_import_data.filename)
+
self.assertEqual(1, len(study_subjects))
study_subject = study_subjects[0]
self.assertEqual("Piotr", study_subject.subject.first_name)
@@ -29,8 +33,9 @@ class TestCsvReader(TestCase):
self.assertIsNotNone(study_subject.study)
def test_load_problematic_dates(self):
- filename = get_resource_path('import_date_of_birth.csv')
- study_subjects = CsvSubjectImportReader().load_data(filename)
+ self.study_import_data.filename = get_resource_path('import_date_of_birth.csv')
+ study_subjects = CsvSubjectImportReader(self.study_import_data).load_data(self.study_import_data.filename)
+
self.assertEqual(3, len(study_subjects))
self.assertIsNone(study_subjects[0].subject.date_born)
self.assertIsNone(study_subjects[1].subject.date_born)
diff --git a/smash/web/tests/importer/test_importer_cron_job.py b/smash/web/tests/importer/test_importer_cron_job.py
index e1b95470..55ae4b09 100644
--- a/smash/web/tests/importer/test_importer_cron_job.py
+++ b/smash/web/tests/importer/test_importer_cron_job.py
@@ -11,8 +11,7 @@ from django.test import TestCase
from django_cron.models import CronJobLog
from web.importer import SubjectImporterCronJob, VisitImporterCronJob
-from web.models import ConfigurationItem, Visit, VisitImportData
-from web.models.constants import DAILY_SUBJECT_IMPORT_FILE
+from web.models import Visit, VisitImportData, SubjectImportData
from web.tests.functions import get_resource_path, get_test_study, create_appointment_type, create_worker
logger = logging.getLogger(__name__)
@@ -27,9 +26,11 @@ class TestCronJobImporter(TestCase):
self.visit_import_data = VisitImportData.objects.create(study=self.study,
appointment_type=create_appointment_type(),
import_worker=create_worker())
+ self.subject_import_data = SubjectImportData.objects.create(study=self.study,
+ import_worker=create_worker())
+ CronJobLog.objects.all().delete()
def test_import_without_configuration(self):
- CronJobLog.objects.all().delete()
job = SubjectImporterCronJob()
@@ -43,14 +44,12 @@ class TestCronJobImporter(TestCase):
new_file, tmp = tempfile.mkstemp()
copyfile(filename, tmp)
- conf = ConfigurationItem.objects.get(type=DAILY_SUBJECT_IMPORT_FILE)
- conf.value = tmp
- conf.save()
- CronJobLog.objects.all().delete()
+ settings.ETL_ROOT = os.path.dirname(tmp)
- job = SubjectImporterCronJob()
+ self.subject_import_data.filename = os.path.basename(tmp)
+ self.subject_import_data.save()
- status = job.do()
+ status = SubjectImporterCronJob(self.study.id).do()
self.assertEqual("import is successful", status)
self.assertEqual(1, len(mail.outbox))
@@ -64,7 +63,6 @@ class TestCronJobImporter(TestCase):
self.visit_import_data.filename = os.path.basename(tmp)
self.visit_import_data.save()
- CronJobLog.objects.all().delete()
job = VisitImporterCronJob(study_id=self.study.id)
diff --git a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
index e9d46d49..28e5bad5 100644
--- a/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
+++ b/smash/web/tests/importer/test_tns_csv_subject_import_reader.py
@@ -4,9 +4,10 @@ import logging
from django.test import TestCase
-from web.importer import TnsCsvSubjectImportReader
-from web.tests.functions import get_resource_path
from web.importer import MsgCounterHandler
+from web.importer import TnsCsvSubjectImportReader
+from web.models import SubjectImportData
+from web.tests.functions import get_resource_path, get_test_study
logger = logging.getLogger(__name__)
@@ -14,6 +15,7 @@ logger = logging.getLogger(__name__)
class TestTnsCsvSubjectReader(TestCase):
def setUp(self):
+ self.study_import_data = SubjectImportData.objects.create(study=get_test_study())
self.warning_counter = MsgCounterHandler()
logging.getLogger('').addHandler(self.warning_counter)
@@ -21,8 +23,8 @@ class TestTnsCsvSubjectReader(TestCase):
logging.getLogger('').removeHandler(self.warning_counter)
def test_load_data(self):
- filename = get_resource_path('tns_subjects_import.csv')
- study_subjects = TnsCsvSubjectImportReader().load_data(filename)
+ self.study_import_data.filename = get_resource_path('tns_subjects_import.csv')
+ study_subjects = TnsCsvSubjectImportReader(self.study_import_data).load_data(self.study_import_data.filename)
self.assertEqual(3, len(study_subjects))
study_subject = study_subjects[1]
self.assertEqual("John2", study_subject.subject.first_name)
--
GitLab