From 86e61c840b2912448996037b5d04285e83a2f0d8 Mon Sep 17 00:00:00 2001
From: Piotr Gawron <piotr.gawron@uni.lu>
Date: Sun, 5 Apr 2020 14:28:00 +0200
Subject: [PATCH] cron job added
---
smash/web/importer/__init__.py | 3 +-
smash/web/importer/importer.py | 23 ++++++++-
smash/web/importer/importer_cron_job.py | 49 +++++++++++++++++++
smash/web/importer/warning_counter.py | 15 ++++++
smash/web/tests/importer/mock_reader.py | 4 ++
smash/web/tests/importer/test_importer.py | 1 +
.../tests/importer/test_importer_cron_job.py | 40 +++++++++++++++
7 files changed, 132 insertions(+), 3 deletions(-)
create mode 100644 smash/web/importer/importer_cron_job.py
create mode 100644 smash/web/importer/warning_counter.py
create mode 100644 smash/web/tests/importer/test_importer_cron_job.py
diff --git a/smash/web/importer/__init__.py b/smash/web/importer/__init__.py
index 76e28e32..785e17b2 100644
--- a/smash/web/importer/__init__.py
+++ b/smash/web/importer/__init__.py
@@ -1,5 +1,6 @@
from importer import Importer
from subject_import_reader import SubjectImportReader
from csv_subject_import_reader import CsvSubjectImportReader
+from importer_cron_job import ImporterCronJob
-__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader]
+__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob]
diff --git a/smash/web/importer/importer.py b/smash/web/importer/importer.py
index 2639f670..ab4427fd 100644
--- a/smash/web/importer/importer.py
+++ b/smash/web/importer/importer.py
@@ -4,6 +4,7 @@ import sys
import traceback
from subject_import_reader import SubjectImportReader
+from warning_counter import MsgCounterHandler
from web.models import StudySubject, Subject
from web.models.constants import GLOBAL_STUDY_ID
@@ -17,13 +18,21 @@ class Importer(object):
self.reader = reader
self.added_count = 0
self.problematic_count = 0
+ self.merged_count = 0
+ self.warning_count = 0
+ self.study_subjects = []
def execute(self):
self.added_count = 0
self.problematic_count = 0
- study_subjects = self.reader.load_data(self.filename)
+ self.merged_count = 0
+ self.warning_count = 0
- for study_subject in study_subjects:
+ warning_counter = MsgCounterHandler()
+ logging.getLogger('').addHandler(warning_counter)
+
+ self.study_subjects = self.reader.load_data(self.filename)
+ for study_subject in self.study_subjects:
try:
if study_subject.study is None:
self.problematic_count += 1
@@ -39,6 +48,9 @@ class Importer(object):
self.problematic_count += 1
traceback.print_exc(file=sys.stdout)
logger.error("Problem with importing study subject: " + study_subject.screening_number)
+ if "WARNING" in warning_counter.level2count:
+ self.warning_count = warning_counter.level2count["WARNING"]
+ logging.getLogger('').removeHandler(warning_counter)
def import_study_subject(self, study_subject):
# type: (StudySubject) -> None
@@ -51,3 +63,10 @@ class Importer(object):
study_subject.subject = Subject.objects.filter(id=study_subject.subject.id)[0]
study_subject.save()
self.added_count += 1
+
+ def get_summary(self):
+ return "<p>Number of entries: <b>" + str(len(self.study_subjects)) + "</b></p>" + \
+ "<p>Number of successfully added entries: <b>" + str(self.added_count) + "</b></p>" + \
+ "<p>Number of successfully merged entries: <b>" + str(self.merged_count) + "</b></p>" + \
+ "<p>Number of problematic entries: <b>" + str(self.problematic_count) + "</b></p>" + \
+ "<p>Number of raised warnings: <b>" + str(self.warning_count) + "</b></p>"
diff --git a/smash/web/importer/importer_cron_job.py b/smash/web/importer/importer_cron_job.py
new file mode 100644
index 00000000..3a9b0d97
--- /dev/null
+++ b/smash/web/importer/importer_cron_job.py
@@ -0,0 +1,49 @@
+# coding=utf-8
+import logging
+import os.path
+import traceback
+
+import timeout_decorator
+from django.conf import settings
+from django_cron import CronJobBase, Schedule
+
+from csv_subject_import_reader import CsvSubjectImportReader
+from importer import Importer
+from web.models.constants import CRON_JOB_TIMEOUT
+from ..smash_email import EmailSender
+
+logger = logging.getLogger(__name__)
+
+
+class ImporterCronJob(CronJobBase):
+ RUN_EVERY_MINUTES = 60 * 24
+ schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
+ code = 'web.import_daily_job' # a unique code
+
+ @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
+ def do(self):
+ title = "Daily import"
+ recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
+
+ filename = getattr(settings, "DAILY_IMPORT_FILE", None)
+ if filename is None:
+ return "import file not defined"
+ if not os.path.isfile(filename):
+ EmailSender().send_email(title,
+ "<h1>File with imported data is not available in the system: " + filename + "</h1>",
+ recipients)
+ try:
+ importer = Importer(settings.DAILY_IMPORT_FILE, CsvSubjectImportReader())
+ importer.execute()
+ body = importer.get_summary()
+ EmailSender().send_email(title,
+ "<h1>Data was successfully imported from file: " + filename + "</h1>" + body,
+ recipients)
+ return "import is successful"
+
+ except:
+ tb = traceback.format_exc()
+ EmailSender().send_email(title,
+ "<h1>There was a problem with importing data from file: " + filename + "</h1>" + tb,
+ recipients)
+ return "import crashed"
diff --git a/smash/web/importer/warning_counter.py b/smash/web/importer/warning_counter.py
new file mode 100644
index 00000000..8500d6a3
--- /dev/null
+++ b/smash/web/importer/warning_counter.py
@@ -0,0 +1,15 @@
+import logging
+
+class MsgCounterHandler(logging.Handler):
+ level2count = None
+
+ def __init__(self, *args, **kwargs):
+ super(MsgCounterHandler, self).__init__(*args, **kwargs)
+ self.level2count = {}
+
+ def emit(self, record):
+ l = record.levelname
+ if l not in self.level2count:
+ self.level2count[l] = 0
+ print(l)
+ self.level2count[l] += 1
diff --git a/smash/web/tests/importer/mock_reader.py b/smash/web/tests/importer/mock_reader.py
index f5435429..a5aedb09 100644
--- a/smash/web/tests/importer/mock_reader.py
+++ b/smash/web/tests/importer/mock_reader.py
@@ -1,5 +1,9 @@
+import logging
+
from web.importer.subject_import_reader import SubjectImportReader
+logger = logging.getLogger(__name__)
+
class MockReader(SubjectImportReader):
def __init__(self, study_subjects):
diff --git a/smash/web/tests/importer/test_importer.py b/smash/web/tests/importer/test_importer.py
index 320e6bf7..da4bb9c3 100644
--- a/smash/web/tests/importer/test_importer.py
+++ b/smash/web/tests/importer/test_importer.py
@@ -38,6 +38,7 @@ class TestImporter(TestCase):
self.assertEqual(1, importer.added_count)
self.assertEqual(0, importer.problematic_count)
+ self.assertEqual(0, importer.warning_count)
def test_import_invalid(self):
study_subjects = []
diff --git a/smash/web/tests/importer/test_importer_cron_job.py b/smash/web/tests/importer/test_importer_cron_job.py
new file mode 100644
index 00000000..f748f902
--- /dev/null
+++ b/smash/web/tests/importer/test_importer_cron_job.py
@@ -0,0 +1,40 @@
+# coding=utf-8
+
+import logging
+
+from django.conf import settings
+from django.test import TestCase
+
+from web.importer import ImporterCronJob
+from web.tests.functions import get_resource_path
+
+logger = logging.getLogger(__name__)
+from django.core import mail
+from django_cron.models import CronJobLog
+
+
+class TestCronJobImporter(TestCase):
+
+ def test_import_without_configuration(self):
+ CronJobLog.objects.all().delete()
+
+ job = ImporterCronJob()
+
+ status = job.do()
+
+ self.assertEqual("import file not defined", status)
+ self.assertEqual(0, len(mail.outbox))
+
+ def test_import(self):
+ filename = get_resource_path('import.csv')
+ setattr(settings, "DAILY_IMPORT_FILE", filename)
+ CronJobLog.objects.all().delete()
+
+ job = ImporterCronJob()
+
+ status = job.do()
+
+ self.assertEqual("import is successful", status)
+ self.assertEqual(1, len(mail.outbox))
+
+ setattr(settings, "DAILY_IMPORT_FILE", None)
--
GitLab