From ffeba1717b15c16f06e2858b176b3ab05d13f76f Mon Sep 17 00:00:00 2001
From: Piotr Gawron <piotr.gawron@uni.lu>
Date: Thu, 23 Feb 2017 18:33:56 +0100
Subject: [PATCH] simple parser for redcap added
---
.../appointment/parse/AppointmentEntry.java | 21 ++
.../smash/appointment/parse/RedcapParser.java | 204 ++++++++++++++++++
.../java/smash/appointment/parse/Subject.java | 67 ++++++
.../smash/appointment/parse/SubjectDao.java | 9 +
.../appointment/parse/CellParserTest.java | 11 +-
.../appointment/parse/RedcapParserTest.java | 75 +++++++
.../smash/appointment/parse/TestBase.java | 14 +-
.../parse/XlsxCalendarProcessorTest.java | 14 +-
appointment-import/testFiles/redcap_imp.txt | 7 +
9 files changed, 402 insertions(+), 20 deletions(-)
create mode 100644 appointment-import/src/main/java/smash/appointment/parse/RedcapParser.java
create mode 100644 appointment-import/src/test/java/smash/appointment/parse/RedcapParserTest.java
create mode 100644 appointment-import/testFiles/redcap_imp.txt
diff --git a/appointment-import/src/main/java/smash/appointment/parse/AppointmentEntry.java b/appointment-import/src/main/java/smash/appointment/parse/AppointmentEntry.java
index 8db01548..2268c902 100644
--- a/appointment-import/src/main/java/smash/appointment/parse/AppointmentEntry.java
+++ b/appointment-import/src/main/java/smash/appointment/parse/AppointmentEntry.java
@@ -1,8 +1,12 @@
package smash.appointment.parse;
+import java.util.ArrayList;
+import java.util.List;
+
public class AppointmentEntry {
private String day;
private String time;
+ private String duration;
private Subject subject;
private AppointmentType type;
private String source;
@@ -96,4 +100,21 @@ public class AppointmentEntry {
public String toString() {
return day + " " + time + " " + subject + " " + type + "\t\t[source: " + source + "]";
}
+
+ /**
+ * @return the duration
+ * @see #duration
+ */
+ public String getDuration() {
+ return duration;
+ }
+
+ /**
+ * @param duration
+ * the duration to set
+ * @see #duration
+ */
+ public void setDuration(String duration) {
+ this.duration = duration;
+ }
}
diff --git a/appointment-import/src/main/java/smash/appointment/parse/RedcapParser.java b/appointment-import/src/main/java/smash/appointment/parse/RedcapParser.java
new file mode 100644
index 00000000..2e8cbc2f
--- /dev/null
+++ b/appointment-import/src/main/java/smash/appointment/parse/RedcapParser.java
@@ -0,0 +1,204 @@
+package smash.appointment.parse;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+public class RedcapParser {
+ Logger logger = Logger.getLogger(RedcapParser.class);
+
+ private SubjectDao subjectDao;
+
+ public List<AppointmentEntry> parse(String filename) throws FileNotFoundException, IOException {
+ List<AppointmentEntry> result = new ArrayList<>();
+ try (BufferedReader br = new BufferedReader(new FileReader(filename))) {
+ int lineNumber = 0;
+ String line;
+ while ((line = br.readLine()) != null) {
+ lineNumber++;
+ if (lineNumber == 1) {
+ continue;
+ }
+ String tmp[] = line.split("\t", -1);
+
+ String ndNumber = tmp[0];
+ Subject subject = subjectDao.getByNdNumber(ndNumber);
+ if (subject == null) {
+ logger.warn("Cannot find subject with id: " + ndNumber);
+ } else {
+ subject.addLanguage(tmp[4]);
+ subject.addLanguage(tmp[5]);
+ subject.addLanguage(tmp[6]);
+ subject.addLanguage(tmp[7]);
+ subject.setSex(tmp[3]);
+ subject.setBirthDate(tmp[2]);
+
+ List<AppointmentEntry> subjectAppointments = new ArrayList<>();
+ if (!tmp[9].isEmpty()) {
+ subjectAppointments.add(createEntryLevelA(tmp[9], tmp[10], subject));
+ }
+
+ if (!tmp[13].isEmpty()) {
+ subjectAppointments.add(createEntryLevelB(tmp[13], tmp[14], subject));
+ }
+
+ if (!tmp[17].isEmpty()) {
+ subjectAppointments.add(createEntryLevelBG(tmp[17], tmp[18], subject));
+ }
+
+ if (!tmp[21].isEmpty()) {
+ subjectAppointments.add(createEntryLevelBV(tmp[21], tmp[22], subject));
+ }
+
+ if (!tmp[41].isEmpty()) {
+ subjectAppointments.add(createEntryLevelSB(tmp[41], subject));
+ }
+
+ if (!tmp[45].isEmpty()) {
+ subjectAppointments.add(createEntryLevelMPower(tmp[45], subject));
+ }
+
+ mergeAppointments(subjectAppointments);
+
+ result.addAll(subjectAppointments);
+ }
+
+ }
+ }
+ return result;
+
+ }
+
+ private void mergeAppointments(List<AppointmentEntry> subjectAppointments) {
+ logger.warn("Not implemented");
+ }
+
+ private AppointmentEntry createEntryLevelMPower(String from, Subject subject) {
+ AppointmentEntry entry = new AppointmentEntry();
+ String date = getDate(from);
+ entry.setDay(date);
+ entry.setSubject(subject);
+ entry.setSource("Imported from RedCap");
+ entry.setType(AppointmentType.LEVEL_B_M_POWER);
+ return entry;
+ }
+
+ private AppointmentEntry createEntryLevelSB(String from, Subject subject) {
+ AppointmentEntry entry = new AppointmentEntry();
+ String date = getDate(from);
+ entry.setDay(date);
+ entry.setSubject(subject);
+ entry.setSource("Imported from RedCap");
+ entry.setType(AppointmentType.LEVEL_SB);
+ return entry;
+ }
+
+ private AppointmentEntry createEntryLevelBV(String from, String to, Subject subject) {
+ AppointmentEntry entry = new AppointmentEntry();
+ String date = getDate(from);
+ String date2 = getDate(to);
+ if (!date.equals(date2)){
+ logger.warn("Different start and end date for level BV appointment. Subject: "+subject.getNdNumber());
+ }
+ entry.setDay(date);
+ entry.setSubject(subject);
+ entry.setSource("Imported from RedCap");
+ entry.setType(AppointmentType.LEVEL_BV);
+ return entry;
+ }
+
+ private AppointmentEntry createEntryLevelBG(String from, String to, Subject subject) {
+ AppointmentEntry entry = new AppointmentEntry();
+ String date = getDate(from);
+ String date2 = getDate(to);
+ if (!date.equals(date2)){
+ logger.warn("Different start and end date for level BG appointment. Subject: "+subject.getNdNumber());
+ }
+ entry.setDay(date);
+ entry.setSubject(subject);
+ entry.setSource("Imported from RedCap");
+ entry.setType(AppointmentType.LEVEL_BG);
+ return entry;
+ }
+
+ private AppointmentEntry createEntryLevelB(String from, String to, Subject subject) {
+ AppointmentEntry entry = new AppointmentEntry();
+ String date = getDate(from);
+ String date2 = getDate(to);
+ if (!date.equals(date2)){
+ logger.warn("Different start and end date for level B appointment. Subject: "+subject.getNdNumber());
+ }
+ entry.setDay(date);
+ entry.setSubject(subject);
+ entry.setSource("Imported from RedCap");
+ entry.setType(AppointmentType.LEVEL_B);
+ return entry;
+ }
+
+ private AppointmentEntry createEntryLevelA(String from, String to, Subject subject) {
+ AppointmentEntry entry = new AppointmentEntry();
+ String date = getDate(from);
+ String date2 = getDate(to);
+ if (!date.equals(date2)){
+ logger.warn("Different start and end date for level A appointment. Subject: "+subject.getNdNumber()+"; "+date+", "+date2);
+ }
+ String time = getTime(from);
+ String duration = getDuration(from, to);
+ entry.setDay(date);
+ entry.setDuration(duration);
+ entry.setSubject(subject);
+ entry.setSource("Imported from RedCap");
+ entry.setTime(time);
+ entry.setType(AppointmentType.LEVEL_A);
+ return entry;
+ }
+
+ private String getDuration(String string, String string2) {
+ if (string2.isEmpty()) {
+ return null;
+ }
+ String time1 = getTime(string);
+ String time2 = getTime(string2);
+
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ LocalDateTime dateTime1 = LocalDateTime.parse("2014-11-25 " + time1 + ":00", formatter);
+ LocalDateTime dateTime2 = LocalDateTime.parse("2014-11-25 " + time2 + ":00", formatter);
+
+ long diffInMinutes = java.time.Duration.between(dateTime1, dateTime2).toMinutes();
+
+ return diffInMinutes + "";
+ }
+
+ private String getTime(String string) {
+ return string.split(" ")[1];
+ }
+
+ private String getDate(String string) {
+ return string.split(" ")[0];
+ }
+
+ /**
+ * @return the subjectDao
+ * @see #subjectDao
+ */
+ public SubjectDao getSubjectDao() {
+ return subjectDao;
+ }
+
+ /**
+ * @param subjectDao
+ * the subjectDao to set
+ * @see #subjectDao
+ */
+ public void setSubjectDao(SubjectDao subjectDao) {
+ this.subjectDao = subjectDao;
+ }
+}
diff --git a/appointment-import/src/main/java/smash/appointment/parse/Subject.java b/appointment-import/src/main/java/smash/appointment/parse/Subject.java
index ee5cc02d..92c2ab9f 100644
--- a/appointment-import/src/main/java/smash/appointment/parse/Subject.java
+++ b/appointment-import/src/main/java/smash/appointment/parse/Subject.java
@@ -1,10 +1,21 @@
package smash.appointment.parse;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
public class Subject {
+ Logger logger = Logger.getLogger(Subject .class);
+
private String name;
private String surname;
private String ndNumber;
private String screeningNumber;
+ private String sex;
+ private String birthDate;
+
+ private List<String> languages = new ArrayList<>();
public Subject(String name, String surname, String ndNumber, String screeningNumber) {
this.name = name;
@@ -86,4 +97,60 @@ public class Subject {
return this.getName() + " " + this.getSurname() + " (" + this.getNdNumber() + "; " + this.getScreeningNumber() + ")";
}
+ public void addLanguage(String string) {
+ if (!string.isEmpty()) {
+ this.languages.add(string);
+ }
+ }
+
+ /**
+ * @return the sex
+ * @see #sex
+ */
+ public String getSex() {
+ return sex;
+ }
+
+ /**
+ * @param sex
+ * the sex to set
+ * @see #sex
+ */
+ public void setSex(String sex) {
+ this.sex = sex;
+ }
+
+ /**
+ * @return the birthDate
+ * @see #birthDate
+ */
+ public String getBirthDate() {
+ return birthDate;
+ }
+
+ /**
+ * @param birthDate
+ * the birthDate to set
+ * @see #birthDate
+ */
+ public void setBirthDate(String birthDate) {
+ this.birthDate = birthDate;
+ }
+
+ /**
+ * @return the languages
+ * @see #languages
+ */
+ public List<String> getLanguages() {
+ return languages;
+ }
+
+ /**
+ * @param languages the languages to set
+ * @see #languages
+ */
+ public void setLanguages(List<String> languages) {
+ this.languages = languages;
+ }
+
}
diff --git a/appointment-import/src/main/java/smash/appointment/parse/SubjectDao.java b/appointment-import/src/main/java/smash/appointment/parse/SubjectDao.java
index c852c84c..682133ca 100644
--- a/appointment-import/src/main/java/smash/appointment/parse/SubjectDao.java
+++ b/appointment-import/src/main/java/smash/appointment/parse/SubjectDao.java
@@ -40,4 +40,13 @@ public class SubjectDao {
this.subjects = subjects;
}
+ public Subject getByNdNumber(String ndNumber) {
+ for (Subject s : subjects) {
+ if (ndNumber.equals(s.getNdNumber())) {
+ return s;
+ }
+ }
+ return null;
+ }
+
}
diff --git a/appointment-import/src/test/java/smash/appointment/parse/CellParserTest.java b/appointment-import/src/test/java/smash/appointment/parse/CellParserTest.java
index 5c96c720..9178b38a 100644
--- a/appointment-import/src/test/java/smash/appointment/parse/CellParserTest.java
+++ b/appointment-import/src/test/java/smash/appointment/parse/CellParserTest.java
@@ -6,20 +6,23 @@ import static org.junit.Assert.assertFalse;
import java.util.ArrayList;
import java.util.List;
+import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;
public class CellParserTest extends TestBase {
+ Logger logger = Logger.getLogger(CellParserTest.class);
+
List<CellParseTestCase> testCases;
- CellParser parser;
+ CellParser parser;
@Before
public void setUp() {
super.setUp();
-
+
parser = new CellParser();
parser.setSubjectDao(subjectDao);
-
+
testCases = new ArrayList<CellParseTestCase>();
testCases.add(new CellParseTestCase("Piotr Gawron level A FU V3", piotrGawron, null, AppointmentType.LEVEL_A));
@@ -32,9 +35,9 @@ public class CellParserTest extends TestBase {
testCases.add(new CellParseTestCase("ND0004 level BV (c) + SB ©", cateKowalsky, null, AppointmentType.LEVEL_BV_SB));
testCases.add(new CellParseTestCase("Cate Kowalsky level BV + BG + SB + m-Power", cateKowalsky, null, AppointmentType.LEVEL_BV_BG_SB_MPOWER));
testCases.add(new CellParseTestCase("sb name level A", null, null, AppointmentType.LEVEL_A));
+ testCases.add(new CellParseTestCase("Andrew Dude level A FU V3", andrewDude, null, AppointmentType.LEVEL_A));
}
-
@Test
public void testExtractTime() {
for (CellParseTestCase testCase : testCases) {
diff --git a/appointment-import/src/test/java/smash/appointment/parse/RedcapParserTest.java b/appointment-import/src/test/java/smash/appointment/parse/RedcapParserTest.java
new file mode 100644
index 00000000..33a72db1
--- /dev/null
+++ b/appointment-import/src/test/java/smash/appointment/parse/RedcapParserTest.java
@@ -0,0 +1,75 @@
+package smash.appointment.parse;
+
+import static org.junit.Assert.*;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+
+public class RedcapParserTest extends TestBase{
+ Logger logger = Logger.getLogger(RedcapParserTest.class);
+ RedcapParser parser = new RedcapParser();
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Before
+ public void setUp() {
+ super.setUp();
+ parser.setSubjectDao(subjectDao);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testParseFile() throws FileNotFoundException, IOException {
+ List<AppointmentEntry> entries = parser.parse("testFiles/redcap_imp.txt");
+ assertTrue(entries.size() > 0);
+ assertEquals(3, subjectDao.getByNdNumber("ND0001").getLanguages().size());
+
+ int levelACount = 0;
+ int levelBCount = 0;
+ int levelBVCount = 0;
+ int levelBGCount = 0;
+ int levelSBCount = 0;
+ int levelMPowerCount = 0;
+ for (AppointmentEntry appointmentEntry : entries) {
+ logger.debug(appointmentEntry);
+ if (appointmentEntry.getType().equals(AppointmentType.LEVEL_A)) {
+ levelACount++;
+ }
+ if (appointmentEntry.getType().equals(AppointmentType.LEVEL_B)) {
+ levelBCount++;
+ }
+ if (appointmentEntry.getType().equals(AppointmentType.LEVEL_BG)) {
+ levelBGCount++;
+ }
+ if (appointmentEntry.getType().equals(AppointmentType.LEVEL_BV)) {
+ levelBVCount++;
+ }
+ if (appointmentEntry.getType().equals(AppointmentType.LEVEL_SB)) {
+ levelSBCount++;
+ }
+ if (appointmentEntry.getType().equals(AppointmentType.LEVEL_B_M_POWER)) {
+ levelMPowerCount++;
+ }
+ }
+ assertTrue(levelACount > 0);
+ assertTrue(levelBCount > 0);
+ assertTrue(levelBVCount > 0);
+ assertTrue(levelBGCount > 0);
+ assertTrue(levelSBCount > 0);
+ assertTrue(levelMPowerCount > 0);
+
+ }
+
+}
diff --git a/appointment-import/src/test/java/smash/appointment/parse/TestBase.java b/appointment-import/src/test/java/smash/appointment/parse/TestBase.java
index e38cc5c3..72704a8a 100644
--- a/appointment-import/src/test/java/smash/appointment/parse/TestBase.java
+++ b/appointment-import/src/test/java/smash/appointment/parse/TestBase.java
@@ -1,27 +1,25 @@
package smash.appointment.parse;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Before;
-
public class TestBase {
SubjectDao subjectDao;
Subject piotrGawron = new Subject("Piotr", "Gawron", "ND0001", "1");
Subject janKowalskiNowak = new Subject("Jan", "Kowalski-Nowak", "ND0002", "2");
Subject johnDoe = new Subject("John", "Doe", "ND0003", "3");
- Subject cateKowalsky = new Subject("Cate", "Kowalsky", "ND0004", "4");
+ Subject cateKowalsky = new Subject("Cate", "Kowalsky", "ND0004", "4");
+ Subject andrewDude = new Subject(" Andrew ", " Dude ", "ND0005", "5");
+ Subject unknownDude = new Subject(" Unknownnnnnnnn ", " Dude ", "ND0006", "6");
public void setUp() {
subjectDao = new SubjectDao();
- subjectDao.addSubject(new Subject("Piotr Marcin", "Gawron", "ND0005", "5"));
+ subjectDao.addSubject(new Subject("Piotr Marcin", "Gawron", "ND1005", "1005"));
subjectDao.addSubject(piotrGawron);
subjectDao.addSubject(janKowalskiNowak);
subjectDao.addSubject(johnDoe);
subjectDao.addSubject(cateKowalsky);
+ subjectDao.addSubject(andrewDude);
+ subjectDao.addSubject(unknownDude);
}
-
}
diff --git a/appointment-import/src/test/java/smash/appointment/parse/XlsxCalendarProcessorTest.java b/appointment-import/src/test/java/smash/appointment/parse/XlsxCalendarProcessorTest.java
index 8c07b601..7625586d 100644
--- a/appointment-import/src/test/java/smash/appointment/parse/XlsxCalendarProcessorTest.java
+++ b/appointment-import/src/test/java/smash/appointment/parse/XlsxCalendarProcessorTest.java
@@ -1,5 +1,6 @@
package smash.appointment.parse;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.FileNotFoundException;
@@ -11,17 +12,17 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
-public class XlsxCalendarProcessorTest extends TestBase{
- Logger logger = Logger.getLogger(XlsxCalendarProcessorTest .class);
-
- XlsxCalendarProcessor processor = new XlsxCalendarProcessor();
+public class XlsxCalendarProcessorTest extends TestBase {
+ Logger logger = Logger.getLogger(XlsxCalendarProcessorTest.class);
+
+ XlsxCalendarProcessor processor = new XlsxCalendarProcessor();
@AfterClass
public static void tearDownAfterClass() throws Exception {
}
@Before
- public void setUp() {
+ public void setUp() {
super.setUp();
processor.setSubjectDao(subjectDao);
}
@@ -34,9 +35,6 @@ public class XlsxCalendarProcessorTest extends TestBase{
public void testReadExcel() throws Exception {
List<AppointmentEntry> entries = processor.processExcel("testFiles/calendarExample.xlsx");
assertTrue(entries.size() > 0);
-// for (AppointmentEntry appointmentEntry : entries) {
-// logger.debug(appointmentEntry);
-// }
}
}
diff --git a/appointment-import/testFiles/redcap_imp.txt b/appointment-import/testFiles/redcap_imp.txt
new file mode 100644
index 00000000..34ea9b17
--- /dev/null
+++ b/appointment-import/testFiles/redcap_imp.txt
@@ -0,0 +1,7 @@
+Subject ID Event Name Date/Time of Birth Sex Language: most fluent Language: second most fluent Language: third most fluent Language: fourth most fluent Basic assessment (Level A) Basic assessment: start date Basic assessment: end date Basic assessment: number of visits Detailed neuropsychological assessment (Level B) Detailed neuropsychological assessment: start date Detailed neuropsychological assessment: end date Detailed neuropsychological assessment: number of visits Detailed motor assessment (Level B) Detailed motor assessment: start date Detailed motor assessment: end date Detailed motor assessment: number of visits Detailed sensory assessment (Level B) Detailed sensory assessment: start date Detailed sensory assessment: end date Detailed sensory assessment: number of visits PSP (Level B) PSP: start date PSP: end date PSP: number of visits Blood Blood: date Urine Urine: date CSF CSF: date Imaging Imaging: date Stool Stool: collection acceptance by patient Stool: collection date proposed Stool: date collected Skin biopsy Skin biopsy: date Saliva Saliva: date mPower mPower: date
+ND0001 Visit 1 1955-10-24 10:44 M Polish German French Yes 2015-01-04 10:14 2015-03-04 14:18 1 Yes 2015-03-04 Yes 2015-03-04 Yes 2016-10-20 Yes 2015-03-04 2015-03-11
+ND0001 Visit 2 Yes 2016-12-12 09:34 2017-01-24 10:46 2
+ND0002 Visit 1 1956-05-22 14:53 M Polish Yes 2015-03-04 14:56 2015-03-04 17:43 1 Yes 2015-07-29 2015-07-29 1 No No No Yes 2015-05-13 Yes 2015-05-13 No No No No Yes 2015-05-13
+ND0002 Visit 2 Yes 2016-01-18 10:00 2016-01-18 13:10 1 Yes 2016-05-26 2016-05-26 1 Yes 2016-01-18 Yes 2016-01-18 Yes Yes 2016-05-26 2016-05-26 Yes 2016-05-26 Yes 2016-01-18
+ND0003 Visit 1 1957-07-15 09:21 F Polish German Rusian English Yes 2015-03-11 09:25 2015-03-11 14:58 1 Yes 2015-11-12 2015-11-12 1 Yes 2015-03-11 Yes 2015-03-11 Yes 2015-03-11
+ND0003 Visit 2 Yes 2016-04-27 11:31 2016-05-11 13:30 2 Yes 2016-06-16 2016-06-16 1 Yes 2016-08-18 2016-08-18 1 Yes 2016-04-27 Yes 2016-04-27 Yes Yes 2016-04-27 2016-04-28 Yes 2016-07-28 Yes 2016-04-27
--
GitLab