Feature/import data pdp

Fixes #261 (closed), #203 (closed), #254 (closed)

Also important commit that must be merged (or at least cherry picked) since it fixes and error on the Voucher Tables: 5e979b18

I think it's done unless we need to make more changes to the data itself, like, adding the flags etc. Which we can do on the website instead of using the program, unless we want a more general program for importing information. Right now I would consider the program basically ad-hoc for this specific case. Since some information must be hardcoded like the VouchersTypes or the format of the Voucher References to get the issue date and voucher type.

Added excluded property to studySubject model. Regarding this, I saw a comment which says: "Marking a subject resigned or deceased cannot be undone.". Why not? and, should resigned option have the same behaviour?

smash/web/api_views/subject.py

@@ -19,8 +19,7 @@ logger = logging.getLogger(__name__)
 
 # noinspection PyUnusedLocal
 def cities(request):
-    result_subjects = Subject.objects.filter(
-        city__isnull=False).values_list('city').distinct()
+    result_subjects = Subject.objects.filter(city__isnull=False).values_list('city').distinct()
     return JsonResponse({
         "cities": [x[0] for x in result_subjects]
     })
@@ -28,8 +27,7 @@ def cities(request):
 
 # noinspection PyUnusedLocal
 def referrals(request):
-    result_subjects = StudySubject.objects.filter(
-        referral__isnull=False).values_list('referral').distinct()
+    result_subjects = StudySubject.objects.filter(referral__isnull=False).values_list('referral').distinct()
     return JsonResponse({
         "referrals": [x[0] for x in result_subjects]
     })
@@ -37,8 +35,7 @@ def referrals(request):
 
 def get_subject_columns(request, subject_list_type):
     study = Study.objects.filter(id=GLOBAL_STUDY_ID)[0]
-    study_subject_lists = StudySubjectList.objects.filter(
-        study=study, type=subject_list_type)
+    study_subject_lists = StudySubjectList.objects.filter(study=study, type=subject_list_type)
     if len(study_subject_lists) > 0:
         study_subject_list = study_subject_lists[0]
         subject_columns = study_subject_list.visible_subject_columns
@@ -49,44 +46,30 @@ def get_subject_columns(request, subject_list_type):
         study_subject_columns = StudyColumns()
 
     result = []
-    add_column(result, "ND", "nd_number", study_subject_columns,
-               "string_filter", study.columns)
-    add_column(result, "Screening", "screening_number",
-               study_subject_columns, "string_filter", study.columns)
-    add_column(result, "First name", "first_name",
-               subject_columns, "string_filter")
-    add_column(result, "Last name", "last_name",
-               subject_columns, "string_filter")
-    add_column(result, "Social Security Number",
-               "social_security_number", subject_columns, "string_filter")
+    add_column(result, "ND", "nd_number", study_subject_columns, "string_filter", study.columns)
+    add_column(result, "Screening", "screening_number", study_subject_columns, "string_filter", study.columns)
+    add_column(result, "First name", "first_name", subject_columns, "string_filter")
+    add_column(result, "Last name", "last_name", subject_columns, "string_filter")
+    add_column(result, "Social Security Number", "social_security_number", subject_columns, "string_filter")
     add_column(result, "Date of birth", "date_born", subject_columns, None)
-    add_column(result, "Contact on", "datetime_contact_reminder",
-               study_subject_columns, None, study.columns)
-    add_column(result, "Last contact attempt",
-               "last_contact_attempt", study_subject_list, None)
-    add_column(result, "Referred by", "referral",
-               study_subject_columns, "string_filter", study.columns)
+    add_column(result, "Contact on", "datetime_contact_reminder", study_subject_columns, None, study.columns)
+    add_column(result, "Last contact attempt", "last_contact_attempt", study_subject_list, None)
+    add_column(result, "Referred by", "referral", study_subject_columns, "string_filter", study.columns)
     add_column(result, "Health partner name", "health_partner_first_name", None, "string_filter",
                add_param=study.columns.health_partner,
                visible_param=study_subject_columns.health_partner)
     add_column(result, "Health partner last name", "health_partner_last_name", None, "string_filter",
                add_param=study.columns.health_partner,
                visible_param=study_subject_columns.health_partner)
-    add_column(result, "Location", "default_location",
-               study_subject_columns, "location_filter", study.columns)
+    add_column(result, "Location", "default_location", study_subject_columns, "location_filter", study.columns)
     add_column(result, "Flying team location", "flying_team", study_subject_columns, "flying_team_filter",
                study.columns)
     add_column(result, "Deceased", "dead", subject_columns, "yes_no_filter")
-    add_column(result, "Resigned", "resigned",
-               study_subject_columns, "yes_no_filter", study.columns)
-    add_column(result, "Postponed", "postponed",
-               study_subject_columns, "yes_no_filter", study.columns)
-    add_column(result, "Excluded", "excluded",
-               study_subject_columns, "yes_no_filter", study.columns)
-    add_column(result, "Info sent", "information_sent",
-               study_subject_columns, "yes_no_filter", study.columns)
-    add_column(result, "Type", "type", study_subject_columns,
-               "type_filter", study.columns)
+    add_column(result, "Resigned", "resigned", study_subject_columns, "yes_no_filter", study.columns)
+    add_column(result, "Postponed", "postponed", study_subject_columns, "yes_no_filter", study.columns)
+    add_column(result, "Excluded", "excluded", study_subject_columns, "yes_no_filter", study.columns)
+    add_column(result, "Info sent", "information_sent", study_subject_columns, "yes_no_filter", study.columns)
+    add_column(result, "Type", "type", study_subject_columns, "type_filter", study.columns)
     add_column(result, "Edit", "edit", None, None, sortable=False)
     for visit_number in range(1, 9):
         visit_key = "visit_" + str(visit_number)
@@ -122,11 +105,9 @@ def get_subjects_order(subjects_to_be_ordered, order_column, order_direction, co
     else:
         order_direction = "-"
     if order_column == "first_name":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'subject__first_name')
+        result = subjects_to_be_ordered.order_by(order_direction + 'subject__first_name')
     elif order_column == "last_name":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'subject__last_name')
+        result = subjects_to_be_ordered.order_by(order_direction + 'subject__last_name')
     elif order_column == "nd_number":
         result = subjects_to_be_ordered.order_by(order_direction + 'nd_number')
     elif order_column == "referral":
@@ -137,31 +118,23 @@ def get_subjects_order(subjects_to_be_ordered, order_column, order_direction, co
         else:
             pattern = column_filters[u'screening_number']
         result = subjects_to_be_ordered.all()
-        result = sorted(result, key=lambda t: t.sort_matched_screening_first(
-            pattern, reverse=order_direction == '-'), reverse=order_direction == '-')
+        result = sorted(result, key=lambda t: t.sort_matched_screening_first(pattern, reverse=order_direction == '-'), reverse = order_direction == '-' )
     elif order_column == "default_location":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'default_location')
+        result = subjects_to_be_ordered.order_by(order_direction + 'default_location')
     elif order_column == "flying_team":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'flying_team')
+        result = subjects_to_be_ordered.order_by(order_direction + 'flying_team')
     elif order_column == "dead":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'subject__dead')
+        result = subjects_to_be_ordered.order_by(order_direction + 'subject__dead')
     elif order_column == "resigned":
         result = subjects_to_be_ordered.order_by(order_direction + 'resigned')
     elif order_column == "information_sent":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'information_sent')
+        result = subjects_to_be_ordered.order_by(order_direction + 'information_sent')
     elif order_column == "health_partner_first_name":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'health_partner__first_name')
+        result = subjects_to_be_ordered.order_by(order_direction + 'health_partner__first_name')
     elif order_column == "health_partner_last_name":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'health_partner__last_name')
+        result = subjects_to_be_ordered.order_by(order_direction + 'health_partner__last_name')
     elif order_column == "social_security_number":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'subject__social_security_number')
+        result = subjects_to_be_ordered.order_by(order_direction + 'subject__social_security_number')
     elif order_column == "postponed":
         result = subjects_to_be_ordered.order_by(order_direction + 'postponed')
     elif order_column == "excluded":
@@ -171,19 +144,16 @@ def get_subjects_order(subjects_to_be_ordered, order_column, order_direction, co
     elif order_column == "id":
         result = subjects_to_be_ordered.order_by(order_direction + 'id')
     elif order_column == "date_born":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'subject__date_born')
+        result = subjects_to_be_ordered.order_by(order_direction + 'subject__date_born')
     elif order_column == "datetime_contact_reminder":
-        result = subjects_to_be_ordered.order_by(
-            order_direction + 'datetime_contact_reminder')
+        result = subjects_to_be_ordered.order_by(order_direction + 'datetime_contact_reminder')
     elif order_column == "last_contact_attempt":
         # noinspection SpellCheckingInspection
         result = subjects_to_be_ordered.annotate(sort_contact_attempt=Max("contactattempt__datetime_when")).order_by(
             order_direction + 'sort_contact_attempt')
     elif str(order_column).startswith("visit_"):
         visit_number = get_visit_number_from_visit_x_string(order_column)
-        result = order_by_visit(subjects_to_be_ordered,
-                                order_direction, visit_number)
+        result = order_by_visit(subjects_to_be_ordered, order_direction, visit_number)
     else:
         logger.warn("Unknown sort column: " + str(order_column))
     return result
@@ -199,10 +169,8 @@ def filter_by_visit(result, visit_number, visit_type):
     datetime_begin_filter = 'visit_' + str(visit_number) + '_datetime_begin'
     datetime_end_filter = 'visit_' + str(visit_number) + '_datetime_end'
     is_finished_filter = 'visit_' + str(visit_number) + '_is_finished'
-    finished_appointments_filter = 'visit_' + \
-        str(visit_number) + '_finished_appointments'
-    scheduled_appointments_filter = 'visit_' + \
-        str(visit_number) + '_scheduled_appointments'
+    finished_appointments_filter = 'visit_' + str(visit_number) + '_finished_appointments'
+    scheduled_appointments_filter = 'visit_' + str(visit_number) + '_scheduled_appointments'
 
     # this is hack... instead of providing True/False value this field contain 1/0 value, the problem is that we need
     # to provide aggregate function for the interacting parameter
@@ -214,13 +182,11 @@ def filter_by_visit(result, visit_number, visit_type):
 
     # number of finished appointments
     result = result.annotate(**{finished_appointments_filter: Count(Case(When(
-        Q(visit__appointment__status=Appointment.APPOINTMENT_STATUS_FINISHED) & Q(
-            visit__visit_number=visit_number),
+        Q(visit__appointment__status=Appointment.APPOINTMENT_STATUS_FINISHED) & Q(visit__visit_number=visit_number),
         then=1)))})
     # number of scheduled appointments
     result = result.annotate(**{scheduled_appointments_filter: Count(Case(When(
-        Q(visit__appointment__status=Appointment.APPOINTMENT_STATUS_SCHEDULED) & Q(
-            visit__visit_number=visit_number),
+        Q(visit__appointment__status=Appointment.APPOINTMENT_STATUS_SCHEDULED) & Q(visit__visit_number=visit_number),
         then=1)))})
 
     # when visit starts
@@ -253,8 +219,7 @@ def filter_by_visit(result, visit_number, visit_type):
             filter(**{datetime_end_filter + "__gt": get_today_midnight_date()}). \
             filter(**{scheduled_appointments_filter: 0})
     elif visit_type == "UPCOMING":
-        result = result.filter(
-            **{datetime_begin_filter + "__gt": get_today_midnight_date()})
+        result = result.filter(**{datetime_begin_filter + "__gt": get_today_midnight_date()})
 
     return result
 
@@ -280,8 +245,6 @@ def get_subjects_filtered(subjects_to_be_filtered, filters):
             result = result.filter(resigned=(value == "true"))
         elif column == "postponed":
             result = result.filter(postponed=(value == "true"))
-        elif column == "excluded":
-            result = result.filter(excluded=(value == "true"))
         elif column == "information_sent":
             result = result.filter(information_sent=(value == "true"))
         elif column == "health_partner_first_name":
@@ -289,8 +252,7 @@ def get_subjects_filtered(subjects_to_be_filtered, filters):
         elif column == "health_partner_last_name":
             result = result.filter(health_partner__last_name__icontains=value)
         elif column == "social_security_number":
-            result = result.filter(
-                subject__social_security_number__icontains=value)
+            result = result.filter(subject__social_security_number__icontains=value)
         elif column == "default_location":
             result = result.filter(default_location=value)
         elif column == "flying_team":
@@ -314,16 +276,14 @@ def get_subjects_filtered(subjects_to_be_filtered, filters):
 
 def subjects(request, type):
     try:
-        # id of the query from dataTable:
-        # https://datatables.net/manual/server-side
+        # id of the query from dataTable: https://datatables.net/manual/server-side
         draw = int(request.GET.get("draw", "-1"))
 
         start = int(request.GET.get("start", "0"))
         length = int(request.GET.get("length", "10"))
         order = int(request.GET.get("order[0][column]", "0"))
         order_dir = request.GET.get("order[0][dir]", "asc")
-        order_column = request.GET.get(
-            "columns[" + str(order) + "][data]", "last_name")
+        order_column = request.GET.get("columns[" + str(order) + "][data]", "last_name")
 
         filters = get_filters_for_data_table_request(request)
 
@@ -332,8 +292,7 @@ def subjects(request, type):
         count = all_subjects.count()
 
         filtered_subjects = get_subjects_filtered(all_subjects, filters)
-        ordered_subjects = get_subjects_order(
-            filtered_subjects, order_column, order_dir, column_filters=dict(filters))
+        ordered_subjects = get_subjects_order(filtered_subjects, order_column, order_dir, column_filters=dict(filters))
         sliced_subjects = ordered_subjects[start:(start + length)]
 
         result_subjects = sliced_subjects
@@ -367,8 +326,7 @@ def types(request):
 def serialize_subject(study_subject):
     location = location_to_str(study_subject.default_location)
     flying_team = flying_team_to_str(study_subject.flying_team)
-    visits = Visit.objects.filter(
-        subject=study_subject).order_by('visit_number')
+    visits = Visit.objects.filter(subject=study_subject).order_by('visit_number')
     serialized_visits = []
     for visit in visits:
         if visit.datetime_begin < get_today_midnight_date():
@@ -400,10 +358,8 @@ def serialize_subject(study_subject):
             "datetime_start": serialize_date(visit.datetime_begin),
             "datetime_end": serialize_date(visit.datetime_end),
         })
-    contact_reminder = serialize_datetime(
-        study_subject.datetime_contact_reminder)
-    contact_attempts = ContactAttempt.objects.filter(
-        subject=study_subject).order_by("-datetime_when")
+    contact_reminder = serialize_datetime(study_subject.datetime_contact_reminder)
+    contact_attempts = ContactAttempt.objects.filter(subject=study_subject).order_by("-datetime_when")
     if len(contact_attempts) > 0:
         last_contact_attempt = contact_attempts[0]
         last_contact_attempt_string = serialize_datetime(last_contact_attempt.datetime_when) + "<br/>" + unicode(
@@ -441,4 +397,4 @@ def serialize_subject(study_subject):
         "id": study_subject.id,
         "visits": serialized_visits,
     }
-    return result
+    return result
\ No newline at end of file