diff --git a/smash/db_scripts/create_dummy_data.py b/smash/db_scripts/create_dummy_data.py index b6a8ab5324685770bd4c06b381cf03df16cdce2b..0e44344b67b6cef660a5b3b259e53f1aa1353743 100644 --- a/smash/db_scripts/create_dummy_data.py +++ b/smash/db_scripts/create_dummy_data.py @@ -437,13 +437,17 @@ class smashProvider(BaseProvider): virus_test_1_updated = fake.date_between(start_date='-30d', end_date='-4d') #inconclusive results have a date virus_test_1_updated = choice([None, virus_test_1_updated], 1, p=[0.7, 0.3])[0] + virus_test_1_collection_date = choice([None, virus_test_1_updated], 1, p=[0.7, 0.3])[0] elif virus_test_1 == True: virus_test_1_updated = fake.date_between(start_date='-30d', end_date='-4d') + virus_test_1_collection_date = fake.date_between(start_date='-30d', end_date='-4d') else: virus_test_1_updated = fake.date_between(start_date='-30d', end_date='-4d') + virus_test_1_collection_date = fake.date_between(start_date='-30d', end_date='-4d') study_subject, _ = StudySubject.objects.update_or_create(nd_number=nd_number, subject=subject, virus_test_1=virus_test_1, virus_test_1_updated=virus_test_1_updated, + virus_test_1_collection_date=virus_test_1_collection_date, defaults={'default_location': default_location, 'type': type, 'screening_number': screening_number, 'study': study}) diff --git a/smash/web/api_views/subject.py b/smash/web/api_views/subject.py index 05a96565baf77d46c1c9cf9158f3a107fa9aaffe..5a8ff67a8451e21a599018ab3155601c8e5ea61b 100644 --- a/smash/web/api_views/subject.py +++ b/smash/web/api_views/subject.py @@ -95,9 +95,14 @@ def get_subject_columns(request, subject_list_type): 'virus_test_{}'.format(one_based_idx), #always starts in 1 study_subject_columns, 'yes_no_null_inconclusive_filter', study.columns) - add_column(result, - "Visit {} RT-PCR date".format(virus_visit_number), - "virus_test_{}_updated".format(one_based_idx), + add_column(result, + "Visit {} RT-PCR update date".format(virus_visit_number), + "virus_test_{}_updated".format(one_based_idx), + study_subject_columns, + "virus_test_date", study.columns) + add_column(result, + "Visit {} RT-PCR collection date".format(virus_visit_number), + "virus_test_{}_collection_date".format(one_based_idx), study_subject_columns, "virus_test_date", study.columns) @@ -220,6 +225,16 @@ def get_subjects_order(subjects_to_be_ordered, order_column, order_direction, co result = subjects_to_be_ordered.order_by(order_direction + 'virus_test_4_updated') elif order_column == "virus_test_5_updated": result = subjects_to_be_ordered.order_by(order_direction + 'virus_test_5_updated') + elif order_column == "virus_test_1_collection_date": + result = subjects_to_be_ordered.order_by(order_direction + 'virus_test_1_collection_date') + elif order_column == "virus_test_2_collection_date": + result = subjects_to_be_ordered.order_by(order_direction + 'virus_test_2_collection_date') + elif order_column == "virus_test_3_collection_date": + result = subjects_to_be_ordered.order_by(order_direction + 'virus_test_3_collection_date') + elif order_column == "virus_test_4_collection_date": + result = subjects_to_be_ordered.order_by(order_direction + 'virus_test_4_collection_date') + elif order_column == "virus_test_5_collection_date": + result = subjects_to_be_ordered.order_by(order_direction + 'virus_test_5_collection_date') else: logger.warn("Unknown sort column: " + str(order_column)) return result @@ -529,6 +544,11 @@ def serialize_subject(study_subject): "virus_test_3_updated": study_subject.virus_test_3_updated, "virus_test_4_updated": study_subject.virus_test_4_updated, "virus_test_5_updated": study_subject.virus_test_5_updated, + "virus_test_1_collection_date": study_subject.virus_test_1_collection_date, + "virus_test_2_collection_date": study_subject.virus_test_2_collection_date, + "virus_test_3_collection_date": study_subject.virus_test_3_collection_date, + "virus_test_4_collection_date": study_subject.virus_test_4_collection_date, + "virus_test_5_collection_date": study_subject.virus_test_5_collection_date, "postponed": bool_to_yes_no(study_subject.postponed), "brain_donation_agreement": bool_to_yes_no(study_subject.brain_donation_agreement), "excluded": bool_to_yes_no(study_subject.excluded), diff --git a/smash/web/forms/study_subject_forms.py b/smash/web/forms/study_subject_forms.py index bf3a78441409eab3d978470843fc0a2df64cbffc..39f2fb0ec6182be2fc580543b754c3d3074151e2 100644 --- a/smash/web/forms/study_subject_forms.py +++ b/smash/web/forms/study_subject_forms.py @@ -34,8 +34,13 @@ class StudySubjectForm(ModelForm): for one_based_idx, virus_visit_number in enumerate(virus_visit_numbers, 1): field = 'virus_test_{}'.format(one_based_idx) self.fields[field].label = 'Visit {} RT-PCR'.format(virus_visit_number) + date_field = 'virus_test_{}_updated'.format(one_based_idx) - self.fields[date_field].label = 'Visit {} RT-PCR date'.format(virus_visit_number) + self.fields[date_field].label = 'Visit {} RT-PCR result date'.format(virus_visit_number) + + collection_date_field = 'virus_test_{}_collection_date'.format(one_based_idx) + self.fields[collection_date_field].label = 'Visit {} RT-PCR collection date'.format(virus_visit_number) + for visit_number in range(1, 6): disable_virus_test_field(self, visit_number) self.update_virus_inconclusive_data(visit_number) @@ -172,10 +177,12 @@ class StudySubjectEditForm(StudySubjectForm): def disable_virus_test_field(self, visit_number): test_result_column_name = 'virus_test_{}'.format(visit_number) - test_date_column_name = 'virus_test_{}_updated'.format(visit_number) + test_result_date_column_name = 'virus_test_{}_updated'.format(visit_number) + test_collection_date_column_name = 'virus_test_{}_collection_date'.format(visit_number) self.fields[test_result_column_name].widget.attrs['readonly'] = True self.fields[test_result_column_name].widget.attrs['disabled'] = True - self.fields[test_date_column_name].widget.attrs['readonly'] = True + self.fields[test_result_date_column_name].widget.attrs['readonly'] = True + self.fields[test_collection_date_column_name].widget.attrs['readonly'] = True def get_study_from_args(kwargs): @@ -229,6 +236,11 @@ def prepare_study_subject_fields(fields, study): prepare_field(fields, study.columns, 'virus_test_3_updated') prepare_field(fields, study.columns, 'virus_test_4_updated') prepare_field(fields, study.columns, 'virus_test_5_updated') + prepare_field(fields, study.columns, 'virus_test_1_collection_date') + prepare_field(fields, study.columns, 'virus_test_2_collection_date') + prepare_field(fields, study.columns, 'virus_test_3_collection_date') + prepare_field(fields, study.columns, 'virus_test_4_collection_date') + prepare_field(fields, study.columns, 'virus_test_5_collection_date') def validate_subject_screening_number(self, cleaned_data): diff --git a/smash/web/migrations/0168_rename_radcap_field.py b/smash/web/migrations/0168_rename_radcap_field.py new file mode 100644 index 0000000000000000000000000000000000000000..be1108a551db354f331dd1c3e70ad4000884a3c6 --- /dev/null +++ b/smash/web/migrations/0168_rename_radcap_field.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +from web.models.constants import RED_CAP_SAMPLE_DATE_FIELD_TYPE, RED_CAP_KIT_ID_FIELD_TYPE + + +def create_item(apps, type, value, name): + # We can't import the ConfigurationItem model directly as it may be a newer + # version than this migration expects. We use the historical version. + ConfigurationItem = apps.get_model("web", "ConfigurationItem") + item = ConfigurationItem.objects.create() + item.type = type + item.value = value + item.name = name + item.save() + + +def configuration_items(apps, schema_editor): + create_item(apps, RED_CAP_SAMPLE_DATE_FIELD_TYPE, "", + "Redcap field for sample date in the visit") + + +class Migration(migrations.Migration): + dependencies = [ + ('web', '0167_auto_20200428_1110'), + ] + + operations = [ + migrations.RunSQL( + "update web_configurationitem set type = '" + RED_CAP_KIT_ID_FIELD_TYPE + "' where type = '" + RED_CAP_SAMPLE_DATE_FIELD_TYPE + "';"), + migrations.RunSQL( + "update web_configurationitem set name = 'Redcap field for sample kit id in the visit' where type = '" + RED_CAP_KIT_ID_FIELD_TYPE + "';"), + migrations.RunPython(configuration_items), + + ] diff --git a/smash/web/migrations/0169_auto_20200525_1123.py b/smash/web/migrations/0169_auto_20200525_1123.py new file mode 100644 index 0000000000000000000000000000000000000000..ecc26be110b775612d237ce09f3106c54637b69e --- /dev/null +++ b/smash/web/migrations/0169_auto_20200525_1123.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2020-05-25 11:23 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('web', '0168_rename_radcap_field'), + ] + + operations = [ + migrations.AddField( + model_name='studysubject', + name='virus_test_1_collection_date', + field=models.DateField(blank=True, null=True, verbose_name=b'Visit 1 virus collection date'), + ), + migrations.AddField( + model_name='studysubject', + name='virus_test_2_collection_date', + field=models.DateField(blank=True, null=True, verbose_name=b'Visit 2 virus collection date'), + ), + migrations.AddField( + model_name='studysubject', + name='virus_test_3_collection_date', + field=models.DateField(blank=True, null=True, verbose_name=b'Visit 3 virus collection date'), + ), + migrations.AddField( + model_name='studysubject', + name='virus_test_4_collection_date', + field=models.DateField(blank=True, null=True, verbose_name=b'Visit 4 virus collection date'), + ), + migrations.AddField( + model_name='studysubject', + name='virus_test_5_collection_date', + field=models.DateField(blank=True, null=True, verbose_name=b'Visit 5 virus collection date'), + ), + ] diff --git a/smash/web/migrations/0170_auto_20200525_1126.py b/smash/web/migrations/0170_auto_20200525_1126.py new file mode 100644 index 0000000000000000000000000000000000000000..4f70a352988530f0e3197b8c3a2f3643f24c9e30 --- /dev/null +++ b/smash/web/migrations/0170_auto_20200525_1126.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2020-05-25 11:26 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('web', '0169_auto_20200525_1123'), + ] + + operations = [ + migrations.AddField( + model_name='studycolumns', + name='virus_test_1_collection_date', + field=models.BooleanField(default=False, verbose_name=b'Visit 1 virus collection date'), + ), + migrations.AddField( + model_name='studycolumns', + name='virus_test_2_collection_date', + field=models.BooleanField(default=False, verbose_name=b'Visit 2 virus collection date'), + ), + migrations.AddField( + model_name='studycolumns', + name='virus_test_3_collection_date', + field=models.BooleanField(default=False, verbose_name=b'Visit 3 virus collection date'), + ), + migrations.AddField( + model_name='studycolumns', + name='virus_test_4_collection_date', + field=models.BooleanField(default=False, verbose_name=b'Visit 4 virus collection date'), + ), + migrations.AddField( + model_name='studycolumns', + name='virus_test_5_collection_date', + field=models.BooleanField(default=False, verbose_name=b'Visit 5 virus collection date'), + ), + ] diff --git a/smash/web/models/constants.py b/smash/web/models/constants.py index 51e7a70bcb8dc2f46892003cbe9fff9f67cba961..498c9a6a007731d41ed761c1a3d72c250c39a3df 100644 --- a/smash/web/models/constants.py +++ b/smash/web/models/constants.py @@ -65,6 +65,7 @@ RED_CAP_SEX_FIELD_TYPE = 'RED_CAP_SEX_FIELD_TYPE' RED_CAP_DATE_BORN_FIELD_TYPE = 'RED_CAP_DATE_BORN_FIELD_TYPE' RED_CAP_ND_NUMBER_FIELD_TYPE = 'RED_CAP_ND_NUMBER_FIELD_TYPE' RED_CAP_VIRUS_FIELD_TYPE = 'RED_CAP_VIRUS_FIELD_TYPE' +RED_CAP_KIT_ID_FIELD_TYPE = "RED_CAP_KIT_ID_FIELD_TYPE" RED_CAP_SAMPLE_DATE_FIELD_TYPE = "RED_CAP_SAMPLE_DATE_FIELD_TYPE" diff --git a/smash/web/models/study_columns.py b/smash/web/models/study_columns.py index d2e769ca52216cf9560933ec89d9902f3092220e..ac3954fa0d935044e729a504010df6c159cde1a7 100644 --- a/smash/web/models/study_columns.py +++ b/smash/web/models/study_columns.py @@ -162,3 +162,23 @@ class StudyColumns(models.Model): default=False, verbose_name='Visit 5 virus results date', ) + virus_test_1_collection_date = models.BooleanField( + default=False, + verbose_name='Visit 1 virus collection date', + ) + virus_test_2_collection_date = models.BooleanField( + default=False, + verbose_name='Visit 2 virus collection date', + ) + virus_test_3_collection_date = models.BooleanField( + default=False, + verbose_name='Visit 3 virus collection date', + ) + virus_test_4_collection_date = models.BooleanField( + default=False, + verbose_name='Visit 4 virus collection date', + ) + virus_test_5_collection_date = models.BooleanField( + default=False, + verbose_name='Visit 5 virus collection date', + ) diff --git a/smash/web/models/study_subject.py b/smash/web/models/study_subject.py index d802d8debb0e5d87dc0a54756bc9152349855cd9..99a0ebb8547f5eb41c3ebcbfb0ed290e9cde8c77 100644 --- a/smash/web/models/study_subject.py +++ b/smash/web/models/study_subject.py @@ -242,6 +242,26 @@ class StudySubject(models.Model): blank=True, null=True, ) + virus_test_1_collection_date = models.DateField(verbose_name='Visit 1 virus collection date', + blank=True, + null=True, + ) + virus_test_2_collection_date = models.DateField(verbose_name='Visit 2 virus collection date', + blank=True, + null=True, + ) + virus_test_3_collection_date = models.DateField(verbose_name='Visit 3 virus collection date', + blank=True, + null=True, + ) + virus_test_4_collection_date = models.DateField(verbose_name='Visit 4 virus collection date', + blank=True, + null=True, + ) + virus_test_5_collection_date = models.DateField(verbose_name='Visit 5 virus collection date', + blank=True, + null=True, + ) def sort_matched_screening_first(self, pattern, reverse=False): if self.screening_number is None: diff --git a/smash/web/redcap_connector.py b/smash/web/redcap_connector.py index 01ada9e9c12d07b945679485dbcd6aa6dc870d7e..0bd62843011f3da9538717c87f1ea873110b9508 100644 --- a/smash/web/redcap_connector.py +++ b/smash/web/redcap_connector.py @@ -11,12 +11,13 @@ from django.conf import settings from django.forms.models import model_to_dict from django_cron import CronJobBase, Schedule -from web.models import ConfigurationItem, StudySubject, Language, AppointmentType, Appointment, Visit, Study, Provenance, Worker, User +from web.models import ConfigurationItem, StudySubject, Language, AppointmentType, Appointment, Visit, Study, \ + Provenance, Worker, User from web.models.constants import REDCAP_TOKEN_CONFIGURATION_TYPE, \ REDCAP_BASE_URL_CONFIGURATION_TYPE, CRON_JOB_TIMEOUT, RED_CAP_LANGUAGE_4_FIELD_TYPE, RED_CAP_LANGUAGE_3_FIELD_TYPE, \ RED_CAP_LANGUAGE_2_FIELD_TYPE, RED_CAP_LANGUAGE_1_FIELD_TYPE, RED_CAP_MPOWER_ID_FIELD_TYPE, RED_CAP_DEAD_FIELD_TYPE, \ RED_CAP_SEX_FIELD_TYPE, RED_CAP_DATE_BORN_FIELD_TYPE, RED_CAP_ND_NUMBER_FIELD_TYPE, RED_CAP_VIRUS_FIELD_TYPE, \ - GLOBAL_STUDY_ID, RED_CAP_SAMPLE_DATE_FIELD_TYPE + GLOBAL_STUDY_ID, RED_CAP_SAMPLE_DATE_FIELD_TYPE, RED_CAP_KIT_ID_FIELD_TYPE from web.models.inconsistent_subject import InconsistentField, InconsistentSubject from web.models.missing_subject import MissingSubject @@ -45,6 +46,7 @@ class RedcapVisit(object): virus = None virus_inconclusive = False visit_number = 0 + virus_collection_date = None def different_string(string1, string2): @@ -55,6 +57,14 @@ def different_string(string1, string2): return string1.strip() != string2.strip() +def date_equals(date1, date2): + if date1 is None and date2 is None: + return True + if date1 is None or date2 is None: + return False + return date1.strftime("%Y-%m-%d") == date2.strftime("%Y-%m-%d") + + class RedcapConnector(object): def __init__(self): self.token = None @@ -82,6 +92,7 @@ class RedcapConnector(object): self.language_4_field = ConfigurationItem.objects.get(type=RED_CAP_LANGUAGE_4_FIELD_TYPE).value self.m_power_id_field = ConfigurationItem.objects.get(type=RED_CAP_MPOWER_ID_FIELD_TYPE).value self.virus_field = ConfigurationItem.objects.get(type=RED_CAP_VIRUS_FIELD_TYPE).value + self.sample_kit_id_field = ConfigurationItem.objects.get(type=RED_CAP_KIT_ID_FIELD_TYPE).value self.sample_date_field = ConfigurationItem.objects.get(type=RED_CAP_SAMPLE_DATE_FIELD_TYPE).value self.study = Study.objects.get(id=GLOBAL_STUDY_ID) @@ -199,7 +210,6 @@ class RedcapConnector(object): result.append(inconsistent_subject) if appointment_type_to_finish is not None: for visit in red_cap_subject.visits: - smasch_visits = Visit.objects.filter(visit_number=visit.visit_number, subject=subject) smasch_appointments = Appointment.objects.filter(visit__in=smasch_visits, appointment_types=appointment_type_to_finish, @@ -207,28 +217,35 @@ class RedcapConnector(object): for smasch_appointment in smasch_appointments: smasch_appointment.mark_as_finished() if smasch_appointment.visit.is_finished != True: - description = u'{} changed from "{}" to "{}"'.format('is_finished', smasch_appointment.visit.is_finished, True) + description = u'{} changed from "{}" to "{}"'.format('is_finished', + smasch_appointment.visit.is_finished, + True) p = Provenance(modified_table=Visit._meta.db_table, - modified_table_id=smasch_appointment.visit.id, - modification_author=self.importer_user, - previous_value=smasch_appointment.visit.is_finished, - new_value=True, - modification_description=description, - modified_field='is_finished') + modified_table_id=smasch_appointment.visit.id, + modification_author=self.importer_user, + previous_value=smasch_appointment.visit.is_finished, + new_value=True, + modification_description=description, + modified_field='is_finished') smasch_appointment.visit.is_finished = True smasch_appointment.visit.save() if visit.virus is not None or visit.virus_inconclusive: changes = None if visit.visit_number == 1 and subject.virus_test_1 != visit.virus: - changes = [('virus_test_1', visit.virus), ('virus_test_1_updated', datetime.datetime.now())] + changes = [('virus_test_1', visit.virus), + ('virus_test_1_updated', datetime.datetime.now())] if visit.visit_number == 2 and subject.virus_test_2 != visit.virus: - changes = [('virus_test_2', visit.virus), ('virus_test_2_updated', datetime.datetime.now())] + changes = [('virus_test_2', visit.virus), + ('virus_test_2_updated', datetime.datetime.now())] if visit.visit_number == 3 and subject.virus_test_3 != visit.virus: - changes = [('virus_test_3', visit.virus), ('virus_test_3_updated', datetime.datetime.now())] + changes = [('virus_test_3', visit.virus), + ('virus_test_3_updated', datetime.datetime.now())] if visit.visit_number == 4 and subject.virus_test_4 != visit.virus: - changes = [('virus_test_4', visit.virus), ('virus_test_4_updated', datetime.datetime.now())] + changes = [('virus_test_4', visit.virus), + ('virus_test_4_updated', datetime.datetime.now())] if visit.visit_number == 5 and subject.virus_test_5 != visit.virus: - changes = [('virus_test_5', visit.virus), ('virus_test_5_updated', datetime.datetime.now())] + changes = [('virus_test_5', visit.virus), + ('virus_test_5_updated', datetime.datetime.now())] if visit.visit_number == 1 and subject.virus_test_1_updated is None and visit.virus_inconclusive: changes = [('virus_test_1_updated', datetime.datetime.now())] if visit.visit_number == 2 and subject.virus_test_2_updated is None and visit.virus_inconclusive: @@ -239,18 +256,30 @@ class RedcapConnector(object): changes = [('virus_test_4_updated', datetime.datetime.now())] if visit.visit_number == 5 and subject.virus_test_5_updated is None and visit.virus_inconclusive: changes = [('virus_test_5_updated', datetime.datetime.now())] + + if visit.visit_number == 1 and not date_equals(subject.virus_test_1_collection_date , visit.virus_collection_date): + changes = [('virus_test_1_collection_date', visit.virus_collection_date)] + if visit.visit_number == 2 and subject.virus_test_2_collection_date != visit.virus_collection_date: + changes = [('virus_test_2_collection_date', visit.virus_collection_date)] + if visit.visit_number == 3 and subject.virus_test_3_collection_date != visit.virus_collection_date: + changes = [('virus_test_3_collection_date', visit.virus_collection_date)] + if visit.visit_number == 4 and subject.virus_test_4_collection_date != visit.virus_collection_date: + changes = [('virus_test_4_collection_date', visit.virus_collection_date)] + if visit.visit_number == 5 and subject.virus_test_5_collection_date != visit.virus_collection_date: + changes = [('virus_test_5_collection_date', visit.virus_collection_date)] + # if changes is not None: for field, new_value in changes: old_value = getattr(subject, field) description = u'{} changed from "{}" to "{}"'.format(field, old_value, new_value) p = Provenance(modified_table=StudySubject._meta.db_table, - modified_table_id=subject.id, - modification_author=self.importer_user, - previous_value=old_value, - new_value=new_value, - modification_description=description, - modified_field=field) + modified_table_id=subject.id, + modification_author=self.importer_user, + previous_value=old_value, + new_value=new_value, + modification_description=description, + modified_field=field) setattr(subject, field, new_value) p.save() subject.save() @@ -369,7 +398,17 @@ class RedcapConnector(object): elif row.get(self.virus_field) == "Inconclusive": visit.virus_inconclusive = True if self.sample_date_field != "": - if row.get(self.sample_date_field) != "": + date_str = row.get(self.sample_date_field) + if date_str is not None and date_str != "" and date_str != "Not done" and date_str != "Not known": + try: + visit.virus_collection_date = datetime.datetime.strptime(row.get(self.sample_date_field), + "%Y-%m-%d") + except ValueError: + logger.warn("Invalid date: " + row.get(self.sample_date_field)) + visit.virus_collection_date = None + + if self.sample_kit_id_field != "": + if row.get(self.sample_kit_id_field) != "": redcap_subject.visits.append(visit) result.append(redcap_subject) for i in range(1, 9): @@ -390,8 +429,8 @@ class RedcapConnector(object): visit.virus = False elif row.get(self.virus_field) == "Positive": visit.virus = True - if self.sample_date_field != "": - if row.get(self.sample_date_field) != "": + if self.sample_kit_id_field != "": + if row.get(self.sample_kit_id_field) != "": redcap_subject.visits.append(visit) return result @@ -441,6 +480,9 @@ class RedcapConnector(object): if self.virus_field != "": result['fields[' + str(field_number) + ']'] = self.virus_field field_number += 1 + if self.sample_kit_id_field != "": + result['fields[' + str(field_number) + ']'] = self.sample_kit_id_field + field_number += 1 if self.sample_date_field != "": result['fields[' + str(field_number) + ']'] = self.sample_date_field field_number += 1