Skip to content
Snippets Groups Projects
Commit 4eaf6946 authored by Piotr Gawron's avatar Piotr Gawron
Browse files

Merge branch 'error-logging' into 'master'

Error logging

See merge request NCER-PD/scheduling-system!301
parents 7ae0bdca c3f0c7b5
No related branches found
No related tags found
1 merge request!301Error logging
Pipeline #36817 passed
smasch (1.0.0~beta.3-1) unstable; urgency=low
* bug fix: upgrade from 0.15 version containing subjects failed
* bug fix: language import was not allowed
* bug fix: empty NEXMO_DEFAULT_FROM did not allow to send authentication
tokens using NEXMO (#364)
-- Piotr Gawron <piotr.gawron@uni.lu> Tue, 12 Jan 2021 10:00:00 +0200
-- Piotr Gawron <piotr.gawron@uni.lu> Mon, 25 Jan 2021 16:00:00 +0200
smasch (1.0.0~beta.2-1) unstable; urgency=low
......
from django import forms
from django.forms import ModelForm
from web.models import SubjectImportData, Subject, StudySubject
from web.models import SubjectImportData, Subject, StudySubject, Language
class SubjectImportDataEditForm(ModelForm):
......@@ -16,7 +16,8 @@ class SubjectImportDataEditForm(ModelForm):
for field in Subject._meta.get_fields():
if field.get_internal_type() == "CharField" or \
field.get_internal_type() == "DateField" or \
field.get_internal_type() == "TextField":
field.get_internal_type() == "TextField" or \
(field.get_internal_type() == "ForeignKey" and field.related_model in (Language,)):
field_id = Subject._meta.db_table + " - " + field.name
value = field.name
for mapping in instance.column_mappings.all():
......@@ -27,7 +28,8 @@ class SubjectImportDataEditForm(ModelForm):
for field in StudySubject._meta.get_fields():
if field.get_internal_type() == "CharField" or \
field.get_internal_type() == "DateField" or \
field.get_internal_type() == "TextField":
field.get_internal_type() == "TextField" or \
(field.get_internal_type() == "ForeignKey" and field.related_model in (Language,)):
field_id = StudySubject._meta.db_table + " - " + field.name
value = field.name
for mapping in instance.column_mappings.all():
......@@ -42,7 +44,8 @@ class SubjectImportDataEditForm(ModelForm):
for field in Subject._meta.get_fields():
if field.get_internal_type() == "CharField" or \
field.get_internal_type() == "DateField" or \
field.get_internal_type() == "TextField":
field.get_internal_type() == "TextField" or \
(field.get_internal_type() == "ForeignKey" and field.related_model in (Language,)):
field_id = Subject._meta.db_table + " - " + field.name
value = self[field_id].value()
instance.set_column_mapping(Subject, field.name, value)
......@@ -50,7 +53,8 @@ class SubjectImportDataEditForm(ModelForm):
for field in StudySubject._meta.get_fields():
if field.get_internal_type() == "CharField" or \
field.get_internal_type() == "DateField" or \
field.get_internal_type() == "TextField":
field.get_internal_type() == "TextField" or \
(field.get_internal_type() == "ForeignKey" and field.related_model in (Language,)):
field_id = StudySubject._meta.db_table + " - " + field.name
value = self[field_id].value()
instance.set_column_mapping(StudySubject, field.name, value)
......
......@@ -5,7 +5,7 @@ from typing import List, Type, Tuple
from django.db import models
from django.db.models import Field
from web.models import StudySubject, Subject, SubjectImportData
from web.models import StudySubject, Subject, SubjectImportData, Language
from .etl_common import EtlCommon
from .subject_import_reader import SubjectImportReader
......@@ -52,6 +52,9 @@ class CsvSubjectImportReader(SubjectImportReader):
if field.get_internal_type() == "DateField":
value = self.get_date(value)
if field.get_internal_type() == "ForeignKey":
value = self.get_value_for_foreign_field(field, value)
if table == Subject:
old_val = getattr(study_subject.subject, field.name)
setattr(study_subject.subject, field.name, self.get_new_value(old_val, value))
......@@ -61,6 +64,20 @@ class CsvSubjectImportReader(SubjectImportReader):
else:
logger.warning("Don't know how to handle column " + column_name + " with data " + value)
@staticmethod
def get_value_for_foreign_field(field, value):
if field.related_model == Language:
if value == "":
return None
else:
language = Language.objects.filter(name=value).first()
if language is None:
language = Language.objects.create(name=value)
return language
else:
logger.warning("Don't know how to handle type " + str(field.related_model))
return None
def get_table_and_field(self, column_name: str) -> Tuple[Type[models.Model], Field]:
return self.mappings.get(column_name, (None, None))
......@@ -68,7 +85,8 @@ class CsvSubjectImportReader(SubjectImportReader):
for field in object_type._meta.get_fields():
if field.get_internal_type() == "CharField" or \
field.get_internal_type() == "DateField" or \
field.get_internal_type() == "TextField":
field.get_internal_type() == "TextField" or \
(field.get_internal_type() == "ForeignKey" and field.related_model in (Language,)):
found = False
for mapping in self.import_data.column_mappings.all():
if mapping.table_name == object_type._meta.db_table and field.name == mapping.column_name:
......
......@@ -3,7 +3,7 @@ import logging
import sys
import traceback
from web.models import StudySubject, Subject
from web.models import StudySubject, Subject, Language
from .etl_common import EtlCommon
from .subject_import_reader import SubjectImportReader
from .warning_counter import MsgCounterHandler
......@@ -40,11 +40,11 @@ class Importer(EtlCommon):
continue
else:
self.import_study_subject(study_subject)
except BaseException:
except BaseException as e:
self.problematic_count += 1
traceback.print_exc(file=sys.stdout)
logger.error("Problem with importing study subject: " + str(study_subject.screening_number) + "," +
str(study_subject.nd_number))
str(study_subject.nd_number), exc_info=e)
if "WARNING" in warning_counter.level2count:
self.warning_count = warning_counter.level2count["WARNING"]
logging.getLogger('').removeHandler(warning_counter)
......@@ -55,7 +55,9 @@ class Importer(EtlCommon):
for field in Subject._meta.get_fields():
if field.get_internal_type() == "CharField" or \
field.get_internal_type() == "DateField" or \
field.get_internal_type() is "BooleanField":
field.get_internal_type() == "BooleanField" or \
field.get_internal_type() == "TextField" or \
(field.get_internal_type() == "ForeignKey" and field.related_model in (Language,)):
old_value = getattr(db_study_subject.subject, field.name)
new_value = self.get_new_value(old_value, getattr(study_subject.subject, field.name))
self.create_provenance_and_change_data(db_study_subject.subject, field.name, new_value, Subject)
......@@ -64,7 +66,9 @@ class Importer(EtlCommon):
for field in StudySubject._meta.get_fields():
if field.get_internal_type() == "CharField" or \
field.get_internal_type() == "DateField" or \
field.get_internal_type() is "BooleanField":
field.get_internal_type() == "BooleanField" or \
field.get_internal_type() == "TextField" or \
(field.get_internal_type() == "ForeignKey" and field.related_model in (Language,)):
old_value = getattr(db_study_subject, field.name)
new_value = self.get_new_value(old_value, getattr(study_subject, field.name))
self.create_provenance_and_change_data(db_study_subject, field.name, new_value, StudySubject)
......
......@@ -66,6 +66,8 @@ def configuration_items(apps, schema_editor):
create_item(apps, NEXMO_API_SECRET, nexmo_api_secret, "NEXMO API SECRET")
nexmo_api_from = getattr(settings, 'NEXMO_DEFAULT_FROM', '')
if nexmo_api_from is None or nexmo_api_from == "":
nexmo_api_from = "SMASCH"
create_item(apps, NEXMO_DEFAULT_FROM, nexmo_api_from, "The sender of the message from NEXMO (phone number or text)")
background_image = getattr(settings, "LOGIN_PAGE_BACKGROUND_IMAGE", "background.jpg")
......
......@@ -32,6 +32,8 @@ class Nexmo:
api_secret = ConfigurationItem.objects.get(type=NEXMO_API_SECRET).value
self.client = nexmo.Client(key=api_key, secret=api_secret)
self.default_from = ConfigurationItem.objects.get(type=NEXMO_DEFAULT_FROM).value
if self.default_from is None or self.default_from == "":
self.default_from = "SMASCH"
def send_sms(self, device, token):
body = 'Your authentication token is %s' % token
......
first_name,last_name,participant_id,default_written_communication_language
Piotr,Gawron,Cov-000001,English
Piotr,Gawron,Cov-000002,Polish
Piotr,Gawron,Cov-000003,
\ No newline at end of file
......@@ -51,6 +51,18 @@ class TestCsvReader(TestCase):
self.assertIsNone(study_subjects[1].subject.date_born)
self.assertIsNone(study_subjects[2].subject.date_born)
def test_load_language(self):
self.subject_import_data.filename = get_resource_path('import_language.csv')
study_subjects = CsvSubjectImportReader(self.subject_import_data).load_data()
for study_subject in study_subjects:
study_subject.subject.save()
study_subject.save()
self.assertEqual(3, len(study_subjects))
self.assertIsNotNone(study_subjects[0].subject.default_written_communication_language)
self.assertIsNotNone(study_subjects[1].subject.default_written_communication_language)
self.assertIsNone(study_subjects[2].subject.default_written_communication_language)
def test_load_data_for_tns(self):
self.subject_import_data = SubjectImportData.objects.create(study=get_test_study(),
date_format="%d/%m/%Y",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment