cron job added

smash/web/importer/__init__.py

 from importer import Importer
 from subject_import_reader import SubjectImportReader
 from csv_subject_import_reader import CsvSubjectImportReader
+from importer_cron_job import ImporterCronJob
 
-__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader]
+__all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, ImporterCronJob]

smash/web/importer/importer.py

@@ -4,6 +4,7 @@ import sys
 import traceback
 
 from subject_import_reader import SubjectImportReader
+from warning_counter import MsgCounterHandler
 from web.models import StudySubject, Subject
 from web.models.constants import GLOBAL_STUDY_ID
 
@@ -17,13 +18,21 @@ class Importer(object):
         self.reader = reader
         self.added_count = 0
         self.problematic_count = 0
+        self.merged_count = 0
+        self.warning_count = 0
+        self.study_subjects = []
 
     def execute(self):
         self.added_count = 0
         self.problematic_count = 0
-        study_subjects = self.reader.load_data(self.filename)
+        self.merged_count = 0
+        self.warning_count = 0
 
-        for study_subject in study_subjects:
+        warning_counter = MsgCounterHandler()
+        logging.getLogger('').addHandler(warning_counter)
+
+        self.study_subjects = self.reader.load_data(self.filename)
+        for study_subject in self.study_subjects:
             try:
                 if study_subject.study is None:
                     self.problematic_count += 1
@@ -39,6 +48,9 @@ class Importer(object):
                 self.problematic_count += 1
                 traceback.print_exc(file=sys.stdout)
                 logger.error("Problem with importing study subject: " + study_subject.screening_number)
+        if "WARNING" in warning_counter.level2count:
+            self.warning_count = warning_counter.level2count["WARNING"]
+        logging.getLogger('').removeHandler(warning_counter)
 
     def import_study_subject(self, study_subject):
         # type: (StudySubject) -> None
@@ -51,3 +63,10 @@ class Importer(object):
             study_subject.subject = Subject.objects.filter(id=study_subject.subject.id)[0]
             study_subject.save()
             self.added_count += 1
+
+    def get_summary(self):
+        return "<p>Number of entries: <b>" + str(len(self.study_subjects)) + "</b></p>" + \
+               "<p>Number of successfully added entries: <b>" + str(self.added_count) + "</b></p>" + \
+               "<p>Number of successfully merged entries: <b>" + str(self.merged_count) + "</b></p>" + \
+               "<p>Number of problematic entries: <b>" + str(self.problematic_count) + "</b></p>" + \
+               "<p>Number of raised warnings: <b>" + str(self.warning_count) + "</b></p>"

smash/web/importer/importer_cron_job.py

+# coding=utf-8
+import logging
+import os.path
+import traceback
+
+import timeout_decorator
+from django.conf import settings
+from django_cron import CronJobBase, Schedule
+
+from csv_subject_import_reader import CsvSubjectImportReader
+from importer import Importer
+from web.models.constants import CRON_JOB_TIMEOUT
+from ..smash_email import EmailSender
+
+logger = logging.getLogger(__name__)
+
+
+class ImporterCronJob(CronJobBase):
+    RUN_EVERY_MINUTES = 60 * 24
+    schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
+    code = 'web.import_daily_job'  # a unique code
+
+    @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
+    def do(self):
+        title = "Daily import"
+        recipients = getattr(settings, "DEFAULT_FROM_EMAIL", None)
+
+        filename = getattr(settings, "DAILY_IMPORT_FILE", None)
+        if filename is None:
+            return "import file not defined"
+        if not os.path.isfile(filename):
+            EmailSender().send_email(title,
+                                     "<h1>File with imported data is not available in the system: " + filename + "</h1>",
+                                     recipients)
+        try:
+            importer = Importer(settings.DAILY_IMPORT_FILE, CsvSubjectImportReader())
+            importer.execute()
+            body = importer.get_summary()
+            EmailSender().send_email(title,
+                                     "<h1>Data was successfully imported from file: " + filename + "</h1>" + body,
+                                     recipients)
+            return "import is successful"
+
+        except:
+            tb = traceback.format_exc()
+            EmailSender().send_email(title,
+                                     "<h1>There was a problem with importing data from file: " + filename + "</h1>" + tb,
+                                     recipients)
+            return "import crashed"

smash/web/importer/warning_counter.py

+import logging
+
+class MsgCounterHandler(logging.Handler):
+    level2count = None
+
+    def __init__(self, *args, **kwargs):
+        super(MsgCounterHandler, self).__init__(*args, **kwargs)
+        self.level2count = {}
+
+    def emit(self, record):
+        l = record.levelname
+        if l not in self.level2count:
+            self.level2count[l] = 0
+        print(l)
+        self.level2count[l] += 1

smash/web/tests/importer/mock_reader.py

+import logging
+
 from web.importer.subject_import_reader import SubjectImportReader
 
+logger = logging.getLogger(__name__)
+
 
 class MockReader(SubjectImportReader):
     def __init__(self, study_subjects):

smash/web/tests/importer/test_importer.py

@@ -38,6 +38,7 @@ class TestImporter(TestCase):
 
         self.assertEqual(1, importer.added_count)
         self.assertEqual(0, importer.problematic_count)
+        self.assertEqual(0, importer.warning_count)
 
     def test_import_invalid(self):
         study_subjects = []

smash/web/tests/importer/test_importer_cron_job.py

+# coding=utf-8
+
+import logging
+
+from django.conf import settings
+from django.test import TestCase
+
+from web.importer import ImporterCronJob
+from web.tests.functions import get_resource_path
+
+logger = logging.getLogger(__name__)
+from django.core import mail
+from django_cron.models import CronJobLog
+
+
+class TestCronJobImporter(TestCase):
+
+    def test_import_without_configuration(self):
+        CronJobLog.objects.all().delete()
+
+        job = ImporterCronJob()
+
+        status = job.do()
+
+        self.assertEqual("import file not defined", status)
+        self.assertEqual(0, len(mail.outbox))
+
+    def test_import(self):
+        filename = get_resource_path('import.csv')
+        setattr(settings, "DAILY_IMPORT_FILE", filename)
+        CronJobLog.objects.all().delete()
+
+        job = ImporterCronJob()
+
+        status = job.do()
+
+        self.assertEqual("import is successful", status)
+        self.assertEqual(1, len(mail.outbox))
+
+        setattr(settings, "DAILY_IMPORT_FILE", None)