From b1d1b52eddff2a854597d79429404012714f6b38 Mon Sep 17 00:00:00 2001
From: Piotr Gawron <piotr.gawron@uni.lu>
Date: Wed, 13 Sep 2017 17:37:26 +0200
Subject: [PATCH] refereshing missing redcap data in database
---
.gitignore | 1 +
.../0054_remove_inconsistentsubject_ignore.py | 19 ++++++
smash/web/models/inconsistent_subject.py | 5 --
smash/web/redcap_connector.py | 61 ++++++++++++++++---
smash/web/tests/test_RedcapConnector.py | 56 +++++++++++++++++
5 files changed, 130 insertions(+), 12 deletions(-)
create mode 100644 smash/web/migrations/0054_remove_inconsistentsubject_ignore.py
diff --git a/.gitignore b/.gitignore
index 1fca780f..7230e12f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ appointment-import/tmp.sql
out
.idea
smash/smash.log
+smash/smash/smash.log
.coverage
diff --git a/smash/web/migrations/0054_remove_inconsistentsubject_ignore.py b/smash/web/migrations/0054_remove_inconsistentsubject_ignore.py
new file mode 100644
index 00000000..e0e4de83
--- /dev/null
+++ b/smash/web/migrations/0054_remove_inconsistentsubject_ignore.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2017-09-13 15:33
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('web', '0053_auto_20170913_0948'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='inconsistentsubject',
+ name='ignore',
+ ),
+ ]
diff --git a/smash/web/models/inconsistent_subject.py b/smash/web/models/inconsistent_subject.py
index 590388fd..b9298c51 100644
--- a/smash/web/models/inconsistent_subject.py
+++ b/smash/web/models/inconsistent_subject.py
@@ -44,11 +44,6 @@ class InconsistentSubject(models.Model):
class Meta:
app_label = 'web'
- ignore = models.BooleanField(
- default=False,
- verbose_name='Ignore missing subject'
- )
-
subject = models.ForeignKey("web.Subject",
verbose_name='Subject',
null=True,
diff --git a/smash/web/redcap_connector.py b/smash/web/redcap_connector.py
index 586bd701..959c56aa 100644
--- a/smash/web/redcap_connector.py
+++ b/smash/web/redcap_connector.py
@@ -1,16 +1,16 @@
# coding=utf-8
-import logging
-import pycurl, cStringIO
-import certifi
+import cStringIO
import json
+import logging
+import pycurl
-from web.models.missing_subject import MissingSubject
-from web.models.inconsistent_subject import InconsistentField, InconsistentSubject
+import certifi
from web.models import ConfigurationItem, Subject, Language
-
from web.models.constants import REDCAP_TOKEN_CONFIGURATION_TYPE, \
- REDCAP_BASE_URL_CONFIGURATION_TYPE, SEX_CHOICES_MALE, SEX_CHOICES_FEMALE
+ REDCAP_BASE_URL_CONFIGURATION_TYPE
+from web.models.inconsistent_subject import InconsistentField, InconsistentSubject
+from web.models.missing_subject import MissingSubject
RED_CAP_LANGUAGE_4_FIELD = 'dm_language_4'
@@ -93,6 +93,53 @@ class RedcapConnector(object):
return result
+ @staticmethod
+ def add_missing(missing_subjects):
+ MissingSubject.objects.filter(ignore=False).delete()
+ ignored_missing_subjects = MissingSubject.objects.all()
+ ignored_redcap_by_nd_number = {}
+ ignored_smash_by_nd_number = {}
+ for missing_subject in ignored_missing_subjects:
+ if missing_subject.redcap_id is not None:
+ ignored_redcap_by_nd_number[missing_subject.redcap_id] = missing_subject
+ if missing_subject.subject is not None:
+ ignored_smash_by_nd_number[missing_subject.subject.nd_number] = missing_subject
+
+ for missing_subject in missing_subjects:
+ ignored = False
+ if missing_subject.redcap_id is not None and ignored_redcap_by_nd_number.get(
+ missing_subject.redcap_id) is not None:
+ ignored = True
+ if missing_subject.subject is not None and ignored_smash_by_nd_number.get(
+ missing_subject.subject.nd_number) is not None:
+ ignored = True
+ if not ignored:
+ MissingSubject.objects.create(subject=missing_subject.subject, redcap_id=missing_subject.redcap_id,
+ redcap_url=missing_subject.redcap_url)
+
+ @staticmethod
+ def add_inconsistent(inconsistent_subjects):
+ InconsistentField.objects.all().delete()
+ InconsistentSubject.objects.all().delete()
+
+ for inconsistent_subject in inconsistent_subjects:
+ subject = InconsistentSubject.objects.create(subject=inconsistent_subject.subject,
+ redcap_url=inconsistent_subject.redcap_url)
+ for field in inconsistent_subject.fields:
+ InconsistentField.objects.create(
+ name=field.name,
+ smash_value=field.smash_value,
+ redcap_value=field.redcap_value,
+ inconsistent_subject=subject)
+
+ def refresh_missing(self):
+ missing = self.find_missing()
+ self.add_missing(missing)
+
+ def refresh_inconsistent(self):
+ missing = self.find_inconsistent()
+ self.add_inconsistent(missing)
+
def find_inconsistent(self):
pid = self.get_project_id()
redcap_version = self.get_redcap_version()
diff --git a/smash/web/tests/test_RedcapConnector.py b/smash/web/tests/test_RedcapConnector.py
index db5b5726..61ec9179 100644
--- a/smash/web/tests/test_RedcapConnector.py
+++ b/smash/web/tests/test_RedcapConnector.py
@@ -7,6 +7,8 @@ from django.test import TestCase
from functions import create_subject
from web.models import ConfigurationItem, Language
from web.models.constants import REDCAP_TOKEN_CONFIGURATION_TYPE, REDCAP_BASE_URL_CONFIGURATION_TYPE
+from web.models.inconsistent_subject import InconsistentSubject
+from web.models.missing_subject import MissingSubject
from web.redcap_connector import RedcapConnector, RedcapSubject
from web.views.notifications import get_today_midnight_date
@@ -147,3 +149,57 @@ class TestRedcapConnector(TestCase):
self.assertIsNotNone(redcap_connection.get_language("xx"))
self.assertIsNone(redcap_connection.get_language("yy"))
+
+ def test_refresh_missing(self):
+ self.prepare_test_redcap_connection()
+
+ redcap_connection = RedcapConnector()
+ redcap_connection.refresh_missing()
+ count = MissingSubject.objects.count()
+ self.assertTrue(count > 0)
+
+ for missing in MissingSubject.objects.all():
+ missing.ignore = True
+ missing.save()
+
+ redcap_connection.refresh_missing()
+ count2 = MissingSubject.objects.count()
+
+ self.assertEqual(count, count2)
+
+ def test_refresh_missing_remove_old(self):
+ missing = MissingSubject.objects.create()
+ missing.ignore = False
+ missing.save()
+ self.prepare_test_redcap_connection()
+
+ self.assertEqual(MissingSubject.objects.filter(id=missing.id).count(), 1)
+
+ redcap_connection = RedcapConnector()
+ redcap_connection.refresh_missing()
+ self.assertTrue(MissingSubject.objects.count() > 0)
+ self.assertEqual(MissingSubject.objects.filter(id=missing.id).count(), 0)
+
+ def test_refresh_missing_don_t_remove_manually_ignored(self):
+ missing = MissingSubject.objects.create()
+ missing.ignore = True
+ missing.save()
+ self.prepare_test_redcap_connection()
+
+ self.assertEqual(MissingSubject.objects.filter(id=missing.id).count(), 1)
+
+ redcap_connection = RedcapConnector()
+ redcap_connection.refresh_missing()
+ self.assertTrue(MissingSubject.objects.count() > 0)
+ self.assertEqual(MissingSubject.objects.filter(id=missing.id).count(), 1)
+
+ def test_refresh_inconsistent_data(self):
+ self.prepare_test_redcap_connection()
+ subject = create_subject()
+ # noinspection SpellCheckingInspection
+ subject.nd_number = 'NDtest_external'
+ subject.save()
+
+ redcap_connection = RedcapConnector()
+ redcap_connection.refresh_inconsistent()
+ self.assertTrue(InconsistentSubject.objects.count() > 0)
--
GitLab