Merge branch '124-connection-to-redcap' into 'master'

Simple connection to RED Cap

See merge request !65

.gitignore

@@ -21,6 +21,7 @@ appointment-import/tmp.sql
 out
 .idea
 smash/smash.log
+smash/smash/smash.log
 
 
 .coverage

requirements.txt

@@ -12,4 +12,5 @@ nexmo
 django-excel==0.0.9
 pyexcel-xls==0.5.0
 pyexcel==0.5.3
+pycurl==7.43.0
 

smash/smash/settings.py

@@ -71,7 +71,8 @@ TEMPLATES = [
 ]
 
 CRON_CLASSES = [
-    'web.views.kit.KitRequestEmailSendJob'
+    'web.views.kit.KitRequestEmailSendJob',
+    'web.redcap_connector.RedCapRefreshJob'
 ]
 
 # Password validation

smash/web/api_urls.py

@@ -15,7 +15,8 @@ Including another URLconf
 """
 from django.conf.urls import url
 
-from web.api_views import worker, location, subject, appointment_type, appointment, configuration, daily_planning
+from web.api_views import worker, location, subject, appointment_type, appointment, configuration, daily_planning, \
+    redcap
 
 urlpatterns = [
     # appointments
@@ -39,7 +40,6 @@ urlpatterns = [
     # locations
     url(r'^locations$', location.locations, name='web.api.locations'),
 
-
     # worker data
     url(r'^specializations$', worker.specializations, name='web.api.specializations'),
     url(r'^units$', worker.units, name='web.api.units'),
@@ -50,4 +50,10 @@ urlpatterns = [
     url(r'^availabilities/(?P<date>\d{4}-\d{2}-\d{2})/$', daily_planning.availabilities, name='web.api.availabilities'),
     url(r'^events_persist$', daily_planning.events_persist, name='web.api.events_persist'),
 
+    # worker data
+    url(r'^redcap/missing_subjects/(?P<missing_subject_id>\d+):ignore$', redcap.ignore_missing_subject,
+        name='web.api.redcap.ignore_missing_subject'),
+    url(r'^redcap/missing_subjects/(?P<missing_subject_id>\d+):unignore$', redcap.unignore_missing_subject,
+        name='web.api.redcap.unignore_missing_subject'),
+
 ]

smash/web/api_views/appointment.py

+import logging
 import traceback
 from datetime import datetime
 
@@ -13,6 +14,8 @@ from web.views.notifications import get_filter_locations, \
     get_today_midnight_date, \
     get_unfinished_appointments
 
+logger = logging.getLogger(__name__)
+
 
 @login_required
 def get_appointments(request, type, min_date, max_date):
@@ -76,7 +79,7 @@ def appointments(request, type):
             "data": data,
         })
     except:
-        traceback.print_exc()
+        logger.exception("Problem with getting appointments")
         return e500_error(request)
 
 

smash/web/api_views/redcap.py

+from django.contrib.auth.decorators import login_required
+from django.http import JsonResponse
+
+from web.models import MissingSubject
+
+
+@login_required
+def ignore_missing_subject(request, missing_subject_id):
+    missing_subjects = MissingSubject.objects.filter(id=missing_subject_id)
+    for missing_subject in missing_subjects:
+        missing_subject.ignore = True
+        missing_subject.save()
+    return JsonResponse({
+        "status": "ok"
+    })
+
+
+@login_required
+def unignore_missing_subject(request, missing_subject_id):
+    missing_subjects = MissingSubject.objects.filter(id=missing_subject_id)
+    for missing_subject in missing_subjects:
+        missing_subject.ignore = False
+        missing_subject.save()
+    return JsonResponse({
+        "status": "ok"
+    })

smash/web/api_views/subject.py

+import logging
+
 from django.contrib.auth.decorators import login_required
 from django.http import JsonResponse
 
@@ -7,6 +9,8 @@ from web.views import e500_error
 from web.views.notifications import get_subjects_with_no_visit, get_subjects_with_reminder
 from web.views.subject import SUBJECT_LIST_GENERIC, SUBJECT_LIST_NO_VISIT, SUBJECT_LIST_REQUIRE_CONTACT
 
+logger = logging.getLogger(__name__)
+
 
 @login_required
 def cities(request):
@@ -101,8 +105,12 @@ def get_subjects_filtered(subjects, filters):
         elif column == "":
             pass
         else:
-            print "UNKNOWN filter: "
-            print row
+            message = "UNKNOWN filter: "
+            if column is None:
+                message += "[None]"
+            else:
+                message += str(column)
+            logger.warn(message)
 
     return result
 

smash/web/migrations/0049_auto_20170912_1438.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-09-12 12:38
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('web', '0048_auto_20170911_1504'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='configurationitem',
+            name='name',
+            field=models.CharField(editable=False, max_length=255, verbose_name=b'Name'),
+        ),
+    ]

smash/web/migrations/0050_configurationitem_redcap_items.py

+from __future__ import unicode_literals
+
+from django.db import migrations
+
+from web.models.constants import REDCAP_TOKEN_CONFIGURATION_TYPE, \
+    REDCAP_BASE_URL_CONFIGURATION_TYPE
+
+
+def configuration_item_color_fields(apps, schema_editor):
+    # We can't import the ConfigurationItem model directly as it may be a newer
+    # version than this migration expects. We use the historical version.
+    ConfigurationItem = apps.get_model("web", "ConfigurationItem")
+    cancelled_appointment_color = ConfigurationItem.objects.create()
+    cancelled_appointment_color.type = REDCAP_TOKEN_CONFIGURATION_TYPE
+    cancelled_appointment_color.value = ""
+    cancelled_appointment_color.name = "API Token for RED Cap integration"
+    cancelled_appointment_color.save()
+
+    cancelled_appointment_color = ConfigurationItem.objects.create()
+    cancelled_appointment_color.type = REDCAP_BASE_URL_CONFIGURATION_TYPE
+    cancelled_appointment_color.value = ""
+    cancelled_appointment_color.name = "Base url of RED Cap (ie  'https://pd-redcap.uni.lu/redcap/')"
+    cancelled_appointment_color.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('web', '0049_auto_20170912_1438'),
+    ]
+
+    operations = [
+        migrations.RunPython(configuration_item_color_fields),
+    ]

smash/web/migrations/0051_missingsubject.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-09-12 14:54
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('web', '0050_configurationitem_redcap_items'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MissingSubject',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ignore', models.BooleanField(default=False, verbose_name=b'Ignore missing subject')),
+                ('redcap_id', models.CharField(blank=True, max_length=255, null=True, verbose_name=b'RED Cap id')),
+                ('redcap_url', models.CharField(blank=True, max_length=255, null=True, verbose_name=b'URL to RED Cap subject')),
+                ('subject', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='web.Subject', verbose_name=b'Subject')),
+            ],
+        ),
+    ]

smash/web/migrations/0052_auto_20170913_0943.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-09-13 07:43
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('web', '0051_missingsubject'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='InconsistentField',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, verbose_name=b'Field name')),
+                ('smash_value', models.CharField(max_length=255, verbose_name=b'Smash value')),
+                ('redcap_value', models.CharField(max_length=255, verbose_name=b'RED Cap value')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='InconsistentSubject',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ignore', models.BooleanField(default=False, verbose_name=b'Ignore missing subject')),
+                ('redcap_url', models.CharField(blank=True, max_length=255, null=True, verbose_name=b'URL to RED Cap subject')),
+                ('subject', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='web.Subject', verbose_name=b'Subject')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='inconsistentfield',
+            name='inconsistent_subject',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='web.InconsistentSubject', verbose_name=b'Invalid fields'),
+        ),
+    ]

smash/web/migrations/0053_auto_20170913_0948.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-09-13 07:48
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('web', '0052_auto_20170913_0943'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='subject',
+            name='nd_number',
+            field=models.CharField(blank=True, max_length=25, verbose_name=b'ND number'),
+        ),
+    ]

smash/web/migrations/0054_remove_inconsistentsubject_ignore.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-09-13 15:33
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('web', '0053_auto_20170913_0948'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='inconsistentsubject',
+            name='ignore',
+        ),
+    ]

smash/web/models/__init__.py

@@ -21,6 +21,8 @@ from language import Language
 from subject import Subject
 from contact_attempt import ContactAttempt
 from mail_template import MailTemplate
+from missing_subject import MissingSubject
+from inconsistent_subject import InconsistentSubject, InconsistentField
 
 
 def get_current_year():
@@ -28,4 +30,5 @@ def get_current_year():
 
 
 __all__ = [FlyingTeam, Appointment, AppointmentType, Availability, Holiday, Item, Language, Location, Room, Subject,
-           Visit, Worker, ContactAttempt, ConfigurationItem, MailTemplate, AppointmentTypeLink]
+           Visit, Worker, ContactAttempt, ConfigurationItem, MailTemplate, AppointmentTypeLink, MissingSubject,
+           InconsistentSubject, InconsistentField]

smash/web/models/configuration_item.py

@@ -12,7 +12,7 @@ class ConfigurationItem(models.Model):
                             verbose_name='Type',
                             editable=False
                             )
-    name = models.CharField(max_length=50,
+    name = models.CharField(max_length=255,
                             verbose_name='Name',
                             editable=False
                             )

smash/web/models/constants.py

@@ -71,3 +71,6 @@ WEEKDAY_CHOICES = (
     (SATURDAY_AS_DAY_OF_WEEK, 'SATURDAY'),
     (SUNDAY_AS_DAY_OF_WEEK, 'SUNDAY'),
 )
+
+REDCAP_TOKEN_CONFIGURATION_TYPE = "REDCAP_TOKEN_CONFIGURATION_TYPE"
+REDCAP_BASE_URL_CONFIGURATION_TYPE = "REDCAP_BASE_URL_CONFIGURATION_TYPE"

smash/web/models/inconsistent_subject.py

+# coding=utf-8
+import logging
+from django.db import models
+
+logger = logging.getLogger(__name__)
+
+
+class InconsistentField(models.Model):
+    name = models.CharField(max_length=255,
+                            verbose_name='Field name',
+                            null=False,
+                            blank=False
+                            )
+    smash_value = models.CharField(max_length=255,
+                                   verbose_name='Smash value'
+                                   )
+    redcap_value = models.CharField(max_length=255,
+                                    verbose_name='RED Cap value'
+                                    )
+    inconsistent_subject = models.ForeignKey("web.InconsistentSubject",
+                                             verbose_name='Invalid fields',
+                                             )
+
+    @staticmethod
+    def create(name, smash_value, redcap_value):
+        result = InconsistentField()
+        result.name = name
+        result.smash_value = smash_value
+        if result.smash_value is None:
+            result.smash_value = "[None]"
+        result.redcap_value = redcap_value
+        if result.redcap_value is None:
+            result.redcap_value = "[None]"
+        return result
+
+    def __str__(self):
+        return self.name + ": " + self.smash_value + "; RED Cap: " + self.redcap_value
+
+    def __unicode__(self):
+        return self.name + ": " + self.smash_value + "; RED Cap: " + self.redcap_value
+
+
+class InconsistentSubject(models.Model):
+    class Meta:
+        app_label = 'web'
+
+    subject = models.ForeignKey("web.Subject",
+                                verbose_name='Subject',
+                                null=True,
+                                blank=True
+                                )
+
+    redcap_url = models.CharField(max_length=255,
+                                  verbose_name='URL to RED Cap subject',
+                                  null=True,
+                                  blank=True
+                                  )
+
+    @staticmethod
+    def create(smash_subject, url=None, fields=None):
+        if fields is None:
+            fields = []
+        result = InconsistentSubject()
+        result.subject = smash_subject
+        result.redcap_url = url
+        result.fields = fields
+        return result
+
+    def __str__(self):
+        return "Subject: " + str(self.subject)
+
+    def __unicode__(self):
+        return "Subject: " + unicode(self.subject)

smash/web/models/missing_subject.py

+# coding=utf-8
+from django.db import models
+
+
+class MissingSubject(models.Model):
+    class Meta:
+        app_label = 'web'
+
+    ignore = models.BooleanField(
+        default=False,
+        verbose_name='Ignore missing subject'
+    )
+
+    subject = models.ForeignKey("web.Subject",
+                                verbose_name='Subject',
+                                null=True,
+                                blank=True
+                                )
+
+    redcap_id = models.CharField(max_length=255,
+                                 verbose_name='RED Cap id',
+                                 null=True,
+                                 blank=True
+                                 )
+    redcap_url = models.CharField(max_length=255,
+                                  verbose_name='URL to RED Cap subject',
+                                  null=True,
+                                  blank=True
+                                  )
+
+    @staticmethod
+    def create(red_cap_subject=None, smash_subject=None, url=None):
+        result = MissingSubject()
+        if red_cap_subject is not None:
+            result.redcap_id = red_cap_subject.nd_number
+        result.subject = smash_subject
+        result.redcap_url = url
+        return result
+
+    def __str__(self):
+        if self.subject is not None:
+            return "Subject: " + str(self.subject)
+        return "RED Cap subject: " + self.redcap_id
+
+    def __unicode__(self):
+        if self.subject is not None:
+            return "Subject: " + str(self.subject)
+        return "RED Cap subject: " + self.redcap_id

smash/web/models/subject.py

@@ -118,7 +118,7 @@ class Subject(models.Model):
                                         unique=True,
                                         verbose_name='Screening number', blank=False, null=False
                                         )
-    nd_number = models.CharField(max_length=6,
+    nd_number = models.CharField(max_length=25,
                                  blank=True,
                                  verbose_name='ND number'
                                  )

smash/web/redcap_connector.py

+# coding=utf-8
+import cStringIO
+import json
+import logging
+import pycurl
+
+import certifi
+from django_cron import CronJobBase, Schedule
+
+from web.models import ConfigurationItem, Subject, Language
+from web.models.constants import REDCAP_TOKEN_CONFIGURATION_TYPE, \
+    REDCAP_BASE_URL_CONFIGURATION_TYPE
+from web.models.inconsistent_subject import InconsistentField, InconsistentSubject
+from web.models.missing_subject import MissingSubject
+
+RED_CAP_LANGUAGE_4_FIELD = 'dm_language_4'
+
+RED_CAP_LANGUAGE_3_FIELD = 'dm_language_3'
+
+RED_CAP_LANGUAGE_2_FIELD = 'dm_language_2'
+
+RED_CAP_LANGUAGE_1_FIELD = 'dm_language_1'
+
+RED_CAP_MPOWER_ID_FIELD = 'dm_mpowerid'
+
+RED_CAP_DEAD_FIELD = 'dm_death'
+
+RED_CAP_SEX_FIELD = 'cdisc_dm_sex'
+
+RED_CAP_DATE_BORN_FIELD = 'cdisc_dm_brthdtc'
+
+RED_CAP_ND_NUMBER_FIELD = 'cdisc_dm_usubjd'
+
+logger = logging.getLogger(__name__)
+
+
+class RedcapSubject(object):
+    url = None
+    nd_number = None
+    date_born = None
+    sex = None
+    dead = None
+    languages = None
+    mpower_id = None
+
+    def __init__(self):
+        self.languages = []
+
+    def add_language(self, language):
+        if language is not None:
+            self.languages.append(language)
+
+
+class RedcapConnector(object):
+    def __init__(self):
+        self.token = None
+        self.base_url = None
+        items = ConfigurationItem.objects.filter(type=REDCAP_TOKEN_CONFIGURATION_TYPE)
+        if len(items) > 0:
+            if items[0].value:
+                self.token = items[0].value
+        items = ConfigurationItem.objects.filter(type=REDCAP_BASE_URL_CONFIGURATION_TYPE)
+        if len(items) > 0:
+            if items[0].value:
+                self.base_url = items[0].value
+        self.language_by_name = {}
+        languages = Language.objects.all()
+        for language in languages:
+            self.language_by_name[language.name.lower()] = language
+
+    def find_missing(self):
+        pid = self.get_project_id()
+        redcap_version = self.get_redcap_version()
+
+        red_cap_subjects = self.get_red_cap_subjects()
+        red_cap_subject_by_nd = {}
+        for subject in red_cap_subjects:
+            red_cap_subject_by_nd[subject.nd_number] = subject
+
+        smash_subjects = Subject.objects.exclude(nd_number='')
+        smash_subject_by_nd = {}
+        for subject in smash_subjects:
+            smash_subject_by_nd[subject.nd_number] = subject
+
+        result = []
+        for subject in red_cap_subjects:
+            if smash_subject_by_nd.get(subject.nd_number) is None:
+                url = self.create_redcap_link(pid, redcap_version, subject)
+                result.append(MissingSubject.create(red_cap_subject=subject, smash_subject=None, url=url))
+
+        for subject in smash_subjects:
+            if red_cap_subject_by_nd.get(subject.nd_number) is None:
+                result.append(MissingSubject.create(red_cap_subject=None, smash_subject=subject))
+
+        return result
+
+    @staticmethod
+    def add_missing(missing_subjects):
+        MissingSubject.objects.filter(ignore=False).delete()
+        ignored_missing_subjects = MissingSubject.objects.all()
+        ignored_redcap_by_nd_number = {}
+        ignored_smash_by_nd_number = {}
+        for missing_subject in ignored_missing_subjects:
+            if missing_subject.redcap_id is not None:
+                ignored_redcap_by_nd_number[missing_subject.redcap_id] = missing_subject
+            if missing_subject.subject is not None:
+                ignored_smash_by_nd_number[missing_subject.subject.nd_number] = missing_subject
+
+        for missing_subject in missing_subjects:
+            ignored = False
+            if missing_subject.redcap_id is not None and ignored_redcap_by_nd_number.get(
+                    missing_subject.redcap_id) is not None:
+                ignored = True
+            if missing_subject.subject is not None and ignored_smash_by_nd_number.get(
+                    missing_subject.subject.nd_number) is not None:
+                ignored = True
+            if not ignored:
+                MissingSubject.objects.create(subject=missing_subject.subject, redcap_id=missing_subject.redcap_id,
+                                              redcap_url=missing_subject.redcap_url)
+
+    @staticmethod
+    def add_inconsistent(inconsistent_subjects):
+        InconsistentField.objects.all().delete()
+        InconsistentSubject.objects.all().delete()
+
+        for inconsistent_subject in inconsistent_subjects:
+            subject = InconsistentSubject.objects.create(subject=inconsistent_subject.subject,
+                                                         redcap_url=inconsistent_subject.redcap_url)
+            for field in inconsistent_subject.fields:
+                InconsistentField.objects.create(
+                    name=field.name,
+                    smash_value=field.smash_value,
+                    redcap_value=field.redcap_value,
+                    inconsistent_subject=subject)
+
+    def refresh_missing(self):
+        missing = self.find_missing()
+        self.add_missing(missing)
+
+    def refresh_inconsistent(self):
+        inconsistent = self.find_inconsistent()
+        self.add_inconsistent(inconsistent)
+
+    def find_inconsistent(self):
+        pid = self.get_project_id()
+        redcap_version = self.get_redcap_version()
+
+        red_cap_subjects = self.get_red_cap_subjects()
+        red_cap_subject_by_nd = {}
+        for subject in red_cap_subjects:
+            red_cap_subject_by_nd[subject.nd_number] = subject
+
+        smash_subjects = Subject.objects.exclude(nd_number='')
+
+        result = []
+        for subject in smash_subjects:
+            red_cap_subject = red_cap_subject_by_nd.get(subject.nd_number)
+
+            if red_cap_subject is not None:
+                url = self.create_redcap_link(pid, redcap_version, subject)
+                subject = self.create_inconsistency_subject(red_cap_subject, subject, url)
+                if subject is not None:
+                    result.append(subject)
+
+        return result
+
+    @staticmethod
+    def create_inconsistency_subject(red_cap_subject, subject, url):
+        fields = []
+        if subject.sex != red_cap_subject.sex:
+            field = InconsistentField.create("sex", subject.sex, red_cap_subject.sex)
+            fields.append(field)
+        subject_date_born = ""
+        if subject.date_born is not None:
+            subject_date_born = subject.date_born.strftime('%Y-%m-%d')
+        redcap_subject_date_born = red_cap_subject.date_born
+        if redcap_subject_date_born is None:
+            redcap_subject_date_born = ""
+        if len(redcap_subject_date_born) > 10:
+            redcap_subject_date_born = redcap_subject_date_born[:10]
+        if subject_date_born != redcap_subject_date_born:
+            field = InconsistentField.create("date of birth", subject_date_born, redcap_subject_date_born)
+            fields.append(field)
+        if subject.dead != red_cap_subject.dead:
+            field = InconsistentField.create("dead", str(subject.dead), str(red_cap_subject.dead))
+            fields.append(field)
+        if subject.mpower_id != red_cap_subject.mpower_id:
+            field = InconsistentField.create("mpower id", subject.mpower_id, red_cap_subject.mpower_id)
+            fields.append(field)
+        missing_language = False
+        if len(red_cap_subject.languages) < 4:
+            for language in subject.languages.all():
+                if language not in red_cap_subject.languages:
+                    missing_language = True
+        for language in red_cap_subject.languages:
+            if language not in subject.languages.all():
+                missing_language = True
+        if missing_language:
+            subject_languages = ""
+            for language in subject.languages.all():
+                subject_languages += language.name + ", "
+            red_cap_subject_languages = ""
+            for language in red_cap_subject.languages:
+                red_cap_subject_languages += language.name + ", "
+            field = InconsistentField.create("languages", subject_languages, red_cap_subject_languages)
+            fields.append(field)
+        result = None
+        if len(fields) > 0:
+            result = InconsistentSubject.create(smash_subject=subject, url=url, fields=fields)
+        return result
+
+    def create_redcap_link(self, pid, redcap_version, subject):
+        return self.base_url + "/redcap_v" + redcap_version + "/DataEntry/index.php?pid=" + pid + "&id=" + \
+               subject.nd_number + "&page=demographics"
+
+    def get_red_cap_subjects(self):
+        query_data = {
+            'token': self.token,
+            'content': 'record',
+            'format': 'json',
+            'type': 'flat',
+            'fields[0]': RED_CAP_DATE_BORN_FIELD,
+            'fields[1]': RED_CAP_SEX_FIELD,
+            'fields[2]': RED_CAP_ND_NUMBER_FIELD,
+            'fields[3]': RED_CAP_DEAD_FIELD,
+            'fields[4]': RED_CAP_LANGUAGE_1_FIELD,
+            'fields[5]': RED_CAP_LANGUAGE_2_FIELD,
+            'fields[6]': RED_CAP_LANGUAGE_3_FIELD,
+            'fields[7]': RED_CAP_LANGUAGE_4_FIELD,
+            'fields[8]': RED_CAP_MPOWER_ID_FIELD,
+            'rawOrLabel': 'label',
+            'rawOrLabelHeaders': 'raw',
+            'exportCheckboxLabel': 'false',
+            'exportSurveyFields': 'false',
+            'exportDataAccessGroups': 'false',
+            'returnFormat': 'json'
+        }
+        data = self.execute_query(query_data)
+        result = []
+        for row in data:
+            redcap_subject = RedcapSubject()
+            redcap_subject.nd_number = row[RED_CAP_ND_NUMBER_FIELD]
+            redcap_subject.date_born = row[RED_CAP_DATE_BORN_FIELD]
+            redcap_subject.sex = row[RED_CAP_SEX_FIELD]
+            redcap_subject.dead = (row[RED_CAP_DEAD_FIELD].lower() == "yes")
+            redcap_subject.mpower_id = row[RED_CAP_MPOWER_ID_FIELD]
+            if row[RED_CAP_LANGUAGE_1_FIELD]:
+                redcap_subject.add_language(self.get_language(row[RED_CAP_LANGUAGE_1_FIELD]))
+            if row[RED_CAP_LANGUAGE_2_FIELD]:
+                redcap_subject.add_language(self.get_language(row[RED_CAP_LANGUAGE_2_FIELD]))
+            if row[RED_CAP_LANGUAGE_3_FIELD]:
+                redcap_subject.add_language(self.get_language(row[RED_CAP_LANGUAGE_3_FIELD]))
+            if row[RED_CAP_LANGUAGE_4_FIELD]:
+                redcap_subject.add_language(self.get_language(row[RED_CAP_LANGUAGE_4_FIELD]))
+
+            result.append(redcap_subject)
+        return result
+
+    def get_language(self, name):
+        language = self.language_by_name.get(name.lower())
+        if language is None:
+            logger.warn("Unknown language: " + name)
+        return language
+
+    def execute_query(self, query_data, is_json=True):
+        buf = cStringIO.StringIO()
+        curl_connection = pycurl.Curl()
+        curl_connection.setopt(pycurl.CAINFO, certifi.where())
+        curl_connection.setopt(curl_connection.URL, self.base_url + "/api/")
+        curl_connection.setopt(curl_connection.HTTPPOST, query_data.items())
+        curl_connection.setopt(curl_connection.WRITEFUNCTION, buf.write)
+        curl_connection.perform()
+        curl_connection.close()
+        if is_json:
+            data = json.loads(buf.getvalue())
+        else:
+            data = buf.getvalue()
+        buf.close()
+        return data
+
+    def get_project_id(self):
+        query_data = {
+            'token': self.token,
+            'content': 'project',
+            'format': 'json',
+            'returnFormat': 'json'
+        }
+        data = self.execute_query(query_data)
+        return data['project_id']
+
+    def get_redcap_version(self):
+        query_data = {
+            'token': self.token,
+            'content': 'version'
+        }
+        data = self.execute_query(query_data, is_json=False)
+        return data
+
+    def is_valid(self):
+        if not self.token:
+            return False
+        if not self.base_url:
+            return False
+
+        return True
+
+
+class RedCapRefreshJob(CronJobBase):
+    RUN_EVERY_MINUTES = 60
+    schedule = Schedule(run_every_mins=RUN_EVERY_MINUTES)
+    code = 'web.red_cap_hourly_refresh'  # a unique code
+
+    def do(self):
+        connector = RedcapConnector()
+        if connector.is_valid():
+            logger.info("Refreshing redcap data")
+            connector.refresh_inconsistent()
+            connector.refresh_missing()
+            logger.info("Redcap data refreshed")
+            return "ok"
+        else:
+            logger.info("Redcap connector is down")
+            return "connector down"