old convoluted exporter removed

smash/smash/settings.py

@@ -83,8 +83,6 @@ CRON_CLASSES = [
     'web.views.virus_mail.KitRequestEmailSendJob',
     'web.redcap_connector.RedCapRefreshJob',
     'web.views.voucher.ExpireVouchersJob',
-    'web.importer.exporter_cron_job.SubjectExporterCronJob',
-    'web.importer.exporter_cron_job.VisitExporterCronJob',
     'web.importer.importer_cron_job.SubjectImporterCronJob',
     'web.importer.importer_cron_job.VisitImporterCronJob'
 ]

smash/web/importer/__init__.py

 from .csv_subject_import_reader import CsvSubjectImportReader
 from .csv_visit_import_reader import CsvVisitImportReader
-from .exporter import SubjectExporter, VisitExporter
-from .exporter_cron_job import SubjectExporterCronJob, VisitExporterCronJob
 from .importer import Importer
 from .importer_cron_job import SubjectImporterCronJob, VisitImporterCronJob
 from .subject_import_reader import SubjectImportReader
 from .warning_counter import MsgCounterHandler
 
 __all__ = [Importer, SubjectImportReader, CsvSubjectImportReader, SubjectImporterCronJob, VisitImporterCronJob,
-           SubjectExporter, VisitExporter, SubjectExporterCronJob, VisitExporterCronJob,
            CsvVisitImportReader, MsgCounterHandler]

smash/web/importer/exporter.py

-# coding=utf-8
-import csv
-import logging
-import sys
-import traceback
-
-from .warning_counter import MsgCounterHandler
-from web.models import StudySubject, Appointment
-
-logger = logging.getLogger(__name__)
-
-
-class SubjectExporter(object):
-    def __init__(self, filename):
-        # type: (str) -> None
-        self.filename = filename
-        self.exported_count = 0
-        self.warning_count = 0
-
-    def execute(self):
-        self.exported_count = 0
-        self.warning_count = 0
-
-        warning_counter = MsgCounterHandler()
-        logging.getLogger('').addHandler(warning_counter)
-
-        with open(self.filename, 'w') as csv_file:
-            data = self.get_subjects_as_array()
-            writer = csv.writer(csv_file, quotechar=str('"'))
-            for row in data:
-                writer.writerow([s.encode("utf-8") for s in row])
-                self.exported_count += 1
-
-        if "WARNING" in warning_counter.level2count:
-            self.warning_count = warning_counter.level2count["WARNING"]
-        logging.getLogger('').removeHandler(warning_counter)
-
-    def get_summary(self):
-        result = "<p>Number of entries: <b>" + str(self.exported_count) + "</b></p>"
-        style = ''
-        if self.warning_count > 0:
-            style = ' color="brown" '
-        result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"
-
-        return result
-
-    def get_subjects_as_array(self):
-        result = []
-        study_subjects = StudySubject.objects.filter(excluded=True)
-        for study_subject in study_subjects:
-            result.append([study_subject.nd_number])
-        return result
-
-
-class VisitExporter(object):
-    def __init__(self, filename):
-        # type: (str) -> None
-        self.filename = filename
-        self.exported_count = 0
-        self.warning_count = 0
-
-    def execute(self):
-        self.exported_count = 0
-        self.warning_count = 0
-
-        warning_counter = MsgCounterHandler()
-        logging.getLogger('').addHandler(warning_counter)
-
-        with open(self.filename, 'w') as csv_file:
-            data = self.get_appointments_as_array()
-            writer = csv.writer(csv_file, quotechar=str('"'))
-            for row in data:
-                writer.writerow([s.encode("utf-8") for s in row])
-                self.exported_count += 1
-
-        if "WARNING" in warning_counter.level2count:
-            self.warning_count = warning_counter.level2count["WARNING"]
-        logging.getLogger('').removeHandler(warning_counter)
-
-    def get_summary(self):
-        result = "<p>Number of entries: <b>" + str(self.exported_count) + "</b></p>"
-        style = ''
-        if self.warning_count > 0:
-            style = ' color="brown" '
-        result += "<p><font " + style + ">Number of raised warnings: <b>" + str(self.warning_count) + "</b></font></p>"
-
-        return result
-
-    def get_appointments_as_array(self):
-        result = []
-        appointments = Appointment.objects.filter(status=Appointment.APPOINTMENT_STATUS_FINISHED)
-        for appointment in appointments:
-            try:
-                if appointment.visit is not None:
-                    result.append([appointment.visit.subject.nd_number, str(appointment.visit.visit_number - 1)])
-            except:
-                traceback.print_exc(file=sys.stdout)
-                logger.warning("Problem with exporting appointment: " + str(appointment.id))
-        return result

smash/web/importer/exporter_cron_job.py

-# coding=utf-8
-import logging
-import traceback
-
-import timeout_decorator
-from django.db import OperationalError, ProgrammingError
-from django_cron import CronJobBase, Schedule
-
-from web.models import ConfigurationItem
-from web.models.constants import CRON_JOB_TIMEOUT, DEFAULT_FROM_EMAIL, DAILY_SUBJECT_EXPORT_FILE, \
-    DAILY_VISIT_EXPORT_FILE, SUBJECT_EXPORT_RUN_AT, VISIT_EXPORT_RUN_AT
-from web.smash_email import EmailSender
-from .exporter import SubjectExporter, VisitExporter
-
-logger = logging.getLogger(__name__)
-
-
-class SubjectExporterCronJob(CronJobBase):
-    RUN_AT_TIMES = []
-    try:
-        item = ConfigurationItem.objects.filter(type=SUBJECT_EXPORT_RUN_AT).first()
-        if item is not None:
-            RUN_AT_TIMES = item.value.split(';')
-    except (OperationalError, ProgrammingError):  # sqlite and postgres throw different errors here
-        logger.debug('Looks like db is not initialized')
-
-    schedule = Schedule(run_at_times=RUN_AT_TIMES)
-    code = 'web.export_subject_daily_job'  # a unique code
-
-    @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
-    def do(self):
-        email_title = "Daily subject export"
-        email_recipients = ConfigurationItem.objects.get(type=DEFAULT_FROM_EMAIL).value
-
-        filename = ConfigurationItem.objects.get(type=DAILY_SUBJECT_EXPORT_FILE).value
-
-        if filename is None or filename == '':
-            logger.info("Exporting subjects skipped. File not defined ")
-            return "export file not defined"
-        logger.info("Exporting subjects to file: " + filename)
-        try:
-            exporter = SubjectExporter(filename)
-            exporter.execute()
-            email_body = exporter.get_summary()
-            EmailSender().send_email(email_title,
-                                     "<h3>Data was successfully exported to file: " + filename + "</h3>" + email_body,
-                                     email_recipients)
-            return "export is successful"
-
-        except:
-            tb = traceback.format_exc()
-            EmailSender().send_email(email_title,
-                                     "<h3><font color='red'>There was a problem with exporting data to file: " + filename + "</font></h3><pre>" + tb + "</pre>",
-                                     email_recipients)
-            return "export crashed"
-
-
-class VisitExporterCronJob(CronJobBase):
-    RUN_AT_TIMES = []
-    try:
-        item = ConfigurationItem.objects.filter(type=VISIT_EXPORT_RUN_AT).first()
-        if item is not None:
-            RUN_AT_TIMES = item.value.split(';')
-    except (OperationalError, ProgrammingError):  # sqlite and postgres throw different errors here
-        logger.debug('Looks like db is not initialized')
-    schedule = Schedule(run_at_times=RUN_AT_TIMES)
-    code = 'web.export_visit_daily_job'  # a unique code
-
-    @timeout_decorator.timeout(CRON_JOB_TIMEOUT)
-    def do(self):
-        email_title = "Daily visit export"
-        email_recipients = ConfigurationItem.objects.get(type=DEFAULT_FROM_EMAIL).value
-
-        filename = ConfigurationItem.objects.get(type=DAILY_VISIT_EXPORT_FILE).value
-
-        if filename is None or filename == '':
-            logger.info("Exporting visit skipped. File not defined ")
-            return "export file not defined"
-        logger.info("Exporting visits to file: " + filename)
-        try:
-            exporter = VisitExporter(filename)
-            exporter.execute()
-            email_body = exporter.get_summary()
-            EmailSender().send_email(email_title,
-                                     "<h3>Data was successfully exported to file: " + filename + "</h3>" + email_body,
-                                     email_recipients)
-            return "export is successful"
-
-        except:
-            tb = traceback.format_exc()
-            EmailSender().send_email(email_title,
-                                     "<h3><font color='red'>There was a problem with exporting data to file: " + filename + "</font></h3><pre>" + tb + "</pre>",
-                                     email_recipients)
-            return "export crashed"

smash/web/migrations/0176_configurationitem_local_setting_clean.py

@@ -2,10 +2,8 @@
 from django.conf import settings
 from django.db import migrations
 
-from web.models.constants import DEFAULT_FROM_EMAIL, DAILY_SUBJECT_IMPORT_FILE, DAILY_SUBJECT_EXPORT_FILE, \
-    DAILY_VISIT_IMPORT_FILE, DAILY_VISIT_EXPORT_FILE, SUBJECT_IMPORT_RUN_AT, SUBJECT_EXPORT_RUN_AT, \
-    VISIT_EXPORT_RUN_AT, VISIT_IMPORT_RUN_AT, IMPORTER_USER, IMPORT_APPOINTMENT_TYPE, NEXMO_API_KEY, NEXMO_API_SECRET, \
-    NEXMO_DEFAULT_FROM, LOGIN_PAGE_BACKGROUND_IMAGE
+from web.models.constants import DEFAULT_FROM_EMAIL, IMPORTER_USER, IMPORT_APPOINTMENT_TYPE, NEXMO_API_KEY, \
+    NEXMO_API_SECRET, NEXMO_DEFAULT_FROM, LOGIN_PAGE_BACKGROUND_IMAGE
 
 
 def create_item(apps, item_type, value, name):
@@ -28,29 +26,29 @@ def configuration_items(apps, schema_editor):
                 "Default email address used in the from field when sending emails")
 
     subject_import_file = getattr(settings, "DAILY_SUBJECT_IMPORT_FILE", '')
-    create_item(apps, DAILY_SUBJECT_IMPORT_FILE, subject_import_file, "File used to import subjects automatically")
+    create_item(apps, "DAILY_SUBJECT_IMPORT_FILE", subject_import_file, "File used to import subjects automatically")
 
     subject_export_file = getattr(settings, "DAILY_SUBJECT_EXPORT_FILE", '')
-    create_item(apps, DAILY_SUBJECT_EXPORT_FILE, subject_export_file, "File used to export subjects automatically")
+    create_item(apps, "DAILY_SUBJECT_EXPORT_FILE", subject_export_file, "File used to export subjects automatically")
 
     visit_import_file = getattr(settings, "DAILY_VISIT_IMPORT_FILE", '')
-    create_item(apps, DAILY_VISIT_IMPORT_FILE, visit_import_file, "File used to import visits automatically")
+    create_item(apps, "DAILY_VISIT_IMPORT_FILE", visit_import_file, "File used to import visits automatically")
 
     visit_export_file = getattr(settings, "DAILY_VISIT_EXPORT_FILE", '')
-    create_item(apps, DAILY_VISIT_EXPORT_FILE, visit_export_file, "File used to export visits automatically")
+    create_item(apps, "DAILY_VISIT_EXPORT_FILE", visit_export_file, "File used to export visits automatically")
 
     subject_import_run_at_times = getattr(settings, "SUBJECT_IMPORT_RUN_AT", ['23:45'])
-    create_item(apps, SUBJECT_IMPORT_RUN_AT, ';'.join(subject_import_run_at_times),
+    create_item(apps, "SUBJECT_IMPORT_RUN_AT", ';'.join(subject_import_run_at_times),
                 "At what times should the subject importer run")
 
     export_run_at_times = getattr(settings, "EXPORT_RUN_AT", ['23:55'])
-    create_item(apps, SUBJECT_EXPORT_RUN_AT, ';'.join(export_run_at_times),
+    create_item(apps, "SUBJECT_EXPORT_RUN_AT", ';'.join(export_run_at_times),
                 "At what times should the subject exporter run")
-    create_item(apps, VISIT_EXPORT_RUN_AT, ';'.join(export_run_at_times),
+    create_item(apps, "VISIT_EXPORT_RUN_AT", ';'.join(export_run_at_times),
                 "At what times should the visit exporter run")
 
     visit_import_run_at_times = getattr(settings, "VISIT_IMPORT_RUN_AT", ['23:55'])
-    create_item(apps, VISIT_IMPORT_RUN_AT, ';'.join(visit_import_run_at_times),
+    create_item(apps, "VISIT_IMPORT_RUN_AT", ';'.join(visit_import_run_at_times),
                 "At what times should the visit importer run")
 
     importer_user_name = getattr(settings, "IMPORTER_USER", '')

smash/web/migrations/0180_visitimportdata_migration.py

 from django.db import migrations
 
-from web.models.constants import IMPORTER_USER, GLOBAL_STUDY_ID, IMPORT_APPOINTMENT_TYPE, DAILY_VISIT_IMPORT_FILE, \
-    VISIT_IMPORT_RUN_AT, DAILY_SUBJECT_IMPORT_FILE, SUBJECT_IMPORT_RUN_AT
+from web.models.constants import GLOBAL_STUDY_ID
 
 
 def get_val(apps, item_type: str):
@@ -13,6 +12,12 @@ def get_val(apps, item_type: str):
     return items[0].value
 
 
+def delete_config_option(apps, item_type: str):
+    # noinspection PyPep8Naming
+    ConfigurationItem = apps.get_model("web", "ConfigurationItem")
+    ConfigurationItem.objects.filter(type=item_type).delete()
+
+
 # noinspection PyUnusedLocal
 def create_visit_import_data(apps, schema_editor):
     # noinspection PyPep8Naming
@@ -26,10 +31,10 @@ def create_visit_import_data(apps, schema_editor):
 
     entry = VisitImportData()
     entry.study = Study.objects.get(pk=GLOBAL_STUDY_ID)
-    importer_user_name = get_val(apps, IMPORTER_USER)
-    appointment_type_name = get_val(apps, IMPORT_APPOINTMENT_TYPE)
-    import_file = get_val(apps, DAILY_VISIT_IMPORT_FILE)
-    import_file_run_at = get_val(apps, VISIT_IMPORT_RUN_AT)
+    importer_user_name = get_val(apps, "IMPORTER_USER")
+    appointment_type_name = get_val(apps, "IMPORT_APPOINTMENT_TYPE")
+    import_file = get_val(apps, "DAILY_VISIT_IMPORT_FILE")
+    import_file_run_at = get_val(apps, "VISIT_IMPORT_RUN_AT")
 
     if importer_user_name is not None:
         workers = Worker.objects.filter(user__username=importer_user_name)
@@ -59,9 +64,9 @@ def create_subject_import_data(apps, schema_editor):
 
     entry = SubjectImportData()
     entry.study = Study.objects.get(pk=GLOBAL_STUDY_ID)
-    importer_user_name = get_val(apps, IMPORTER_USER)
-    import_file = get_val(apps, DAILY_SUBJECT_IMPORT_FILE)
-    import_file_run_at = get_val(apps, SUBJECT_IMPORT_RUN_AT)
+    importer_user_name = get_val(apps, "IMPORTER_USER")
+    import_file = get_val(apps, "DAILY_SUBJECT_IMPORT_FILE")
+    import_file_run_at = get_val(apps, "SUBJECT_IMPORT_RUN_AT")
 
     if importer_user_name is not None:
         workers = Worker.objects.filter(user__username=importer_user_name)
@@ -75,6 +80,17 @@ def create_subject_import_data(apps, schema_editor):
     entry.save()
 
 
+def remove_old_config(apps, schema_editor):
+    delete_config_option(apps, "DAILY_SUBJECT_IMPORT_FILE")
+    delete_config_option(apps, "DAILY_SUBJECT_EXPORT_FILE")
+    delete_config_option(apps, "DAILY_VISIT_IMPORT_FILE")
+    delete_config_option(apps, "DAILY_VISIT_EXPORT_FILE")
+    delete_config_option(apps, "SUBJECT_IMPORT_RUN_AT")
+    delete_config_option(apps, "SUBJECT_EXPORT_RUN_AT")
+    delete_config_option(apps, "VISIT_EXPORT_RUN_AT")
+    delete_config_option(apps, "VISIT_IMPORT_RUN_AT")
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ('web', '0179_visitimportdata'),
@@ -83,4 +99,5 @@ class Migration(migrations.Migration):
     operations = [
         migrations.RunPython(create_visit_import_data),
         migrations.RunPython(create_subject_import_data),
+        migrations.RunPython(remove_old_config),
     ]

smash/web/models/constants.py

@@ -58,14 +58,6 @@ KIT_DAILY_EMAIL_DAYS_PERIOD_TYPE = "KIT_DAILY_EMAIL_DAYS_PERIOD_TYPE"
 KIT_DAILY_EMAIL_TIME_FORMAT_TYPE = "KIT_DAILY_EMAIL_TIME_FORMAT_TYPE"
 VIRUS_EMAIL_HOUR_CONFIGURATION_TYPE = "VIRUS_EMAIL_HOUR_CONFIGURATION_TYPE"
 DEFAULT_FROM_EMAIL = "DEFAULT_FROM_EMAIL"
-DAILY_SUBJECT_IMPORT_FILE = "DAILY_SUBJECT_IMPORT_FILE"
-DAILY_SUBJECT_EXPORT_FILE = "DAILY_SUBJECT_EXPORT_FILE"
-DAILY_VISIT_IMPORT_FILE = "DAILY_VISIT_IMPORT_FILE"
-DAILY_VISIT_EXPORT_FILE = "DAILY_VISIT_EXPORT_FILE"
-SUBJECT_IMPORT_RUN_AT = "SUBJECT_IMPORT_RUN_AT"
-SUBJECT_EXPORT_RUN_AT = "SUBJECT_EXPORT_RUN_AT"
-VISIT_EXPORT_RUN_AT = "VISIT_EXPORT_RUN_AT"
-VISIT_IMPORT_RUN_AT = "VISIT_IMPORT_RUN_AT"
 IMPORTER_USER = "IMPORTER_USER"
 IMPORT_APPOINTMENT_TYPE = "IMPORT_APPOINTMENT_TYPE"
 NEXMO_API_KEY = "NEXMO_API_KEY"

smash/web/tests/importer/test_exporter.py

-# coding=utf-8
-
-import datetime
-import logging
-
-from django.test import TestCase
-
-from web.tests.functions import create_study_subject
-from web.importer import SubjectExporter
-from web.models import Subject, StudySubject, Study, Provenance
-from web.models.constants import GLOBAL_STUDY_ID
-
-logger = logging.getLogger(__name__)
-
-
-class TestExporter(TestCase):
-
-    def setUp(self):
-        self.study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
-
-    def test_export_not_excluded(self):
-        create_study_subject()
-
-        exporter = SubjectExporter(filename="empty.csv")
-        exporter.execute()
-
-        self.assertEqual(0, exporter.exported_count)
-        self.assertEqual(0, exporter.warning_count)
-
-    def test_export_excluded(self):
-        subject = create_study_subject()
-        subject.excluded=True
-        subject.save()
-
-        exporter = SubjectExporter(filename="empty.csv")
-        exporter.execute()
-
-        self.assertEqual(1, exporter.exported_count)
-        self.assertEqual(0, exporter.warning_count)
-

smash/web/tests/importer/test_exporter_cron_job.py

-# coding=utf-8
-
-import logging
-import tempfile
-
-from django.core import mail
-from django.test import TestCase
-from django_cron.models import CronJobLog
-
-from web.importer import SubjectExporterCronJob
-from web.models import ConfigurationItem
-from web.models.constants import DAILY_SUBJECT_EXPORT_FILE
-
-logger = logging.getLogger(__name__)
-
-
-class TestCronJobExporter(TestCase):
-
-    def test_import_without_configuration(self):
-        CronJobLog.objects.all().delete()
-
-        job = SubjectExporterCronJob()
-
-        status = job.do()
-
-        self.assertEqual("export file not defined", status)
-        self.assertEqual(0, len(mail.outbox))
-
-    def test_import(self):
-        new_file, tmp = tempfile.mkstemp()
-
-        item = ConfigurationItem.objects.get(type=DAILY_SUBJECT_EXPORT_FILE)
-        item.value = tmp
-        item.save()
-
-        CronJobLog.objects.all().delete()
-
-        job = SubjectExporterCronJob()
-
-        status = job.do()
-
-        self.assertEqual("export is successful", status)
-        self.assertEqual(1, len(mail.outbox))

smash/web/views/kit.py

@@ -8,18 +8,17 @@ import traceback
 
 import timeout_decorator
 from django.contrib import messages
-from django.utils.dateparse import parse_datetime
 from django.utils.timezone import make_aware
 from django_cron import CronJobBase, Schedule
 from django_cron.models import CronJobLog
 
-from .notifications import get_filter_locations, get_today_midnight_date
 from web.decorators import PermissionDecorator
 from web.models import ConfigurationItem, Language, Worker, Study
 from web.models.constants import KIT_RECIPIENT_EMAIL_CONFIGURATION_TYPE, KIT_DAILY_EMAIL_DAYS_PERIOD_TYPE, \
     KIT_EMAIL_HOUR_CONFIGURATION_TYPE, KIT_DAILY_EMAIL_TIME_FORMAT_TYPE, \
     KIT_EMAIL_DAY_OF_WEEK_CONFIGURATION_TYPE, CRON_JOB_TIMEOUT, GLOBAL_STUDY_ID
 from . import wrap_response
+from .notifications import get_filter_locations, get_today_midnight_date
 from ..forms import KitRequestForm
 from ..models import AppointmentType, Appointment
 from ..smash_email import EmailSender
@@ -236,6 +235,7 @@ def kit_requests_send_mail(request, start_date, end_date=None):
 class KitRequestEmailSendJob(CronJobBase):
     RUN_AT = []
 
+    # noinspection PyBroadException
     try:
         times = ConfigurationItem.objects.get(
             type=KIT_EMAIL_HOUR_CONFIGURATION_TYPE).value.split(";")
@@ -243,8 +243,8 @@ class KitRequestEmailSendJob(CronJobBase):
             # TODO it's a hack assuming that we are in CEST
             text = str((int(entry.split(":")[0]) + 22) % 24) + ":" + entry.split(":")[1]
             RUN_AT.append(text)
-    except:
-        logger.error("Cannot fetch data about email hour")
+    except BaseException:
+        logger.debug("Cannot fetch data about email hour")
     schedule = Schedule(run_at_times=RUN_AT)
     code = 'web.kit_request_weekly_email'  # a unique code
 

smash/web/views/virus_mail.py

@@ -115,6 +115,7 @@ def send_mail(data):
 class KitRequestEmailSendJob(CronJobBase):
     RUN_AT = []
 
+    # noinspection PyBroadException
     try:
         times = ConfigurationItem.objects.get(
             type=VIRUS_EMAIL_HOUR_CONFIGURATION_TYPE).value.split(";")
@@ -122,8 +123,8 @@ class KitRequestEmailSendJob(CronJobBase):
             # TODO it's a hack assuming that we are in CEST
             text = str((int(entry.split(":")[0]) + 22) % 24) + ":" + entry.split(":")[1]
             RUN_AT.append(text)
-    except:
-        logger.error("Cannot fetch data about email hour")
+    except BaseException as e:
+        logger.debug("Cannot fetch data about email hour")
     schedule = Schedule(run_at_times=RUN_AT)
     code = 'web.virus_daily_email'  # a unique code