Merge branch 'duplicates-removal' into 'master'

old import code

See merge request !52

appointment-import/src/main/java/smash/appointment/parse/Main.java

 package smash.appointment.parse;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Calendar;
 import java.util.List;
@@ -29,14 +30,17 @@ public class Main {
 		Option lihControls = Option.builder().required().argName("file").hasArg().desc("LIH controls").longOpt("lih-controls").build();
 		Option lihMappingControls = Option.builder().required().argName("file").hasArg().desc("LIH controls mapping").longOpt("lih-mapping").build();
 		Option redCap = Option.builder().required().argName("file").hasArg().desc("RedCap appointments").longOpt("red-cap").build();
+		Option redCapCalendar = Option.builder().required().argName("file").hasArg().desc("RedCap calendar sql").longOpt("red-cap-calendar").build();
 		Option duplicates = Option.builder().required().argName("file").hasArg().desc("duplicates").longOpt("duplicates").build();
 		options.addOption(agenda);
+
 		options.addOption(subjects);
 		options.addOption(controls);
 		options.addOption(flyingTeam);
 		options.addOption(lihControls);
 		options.addOption(lihMappingControls);
 		options.addOption(redCap);
+		options.addOption(redCapCalendar);
 		options.addOption(duplicates);
 
 		CommandLineParser parser = new DefaultParser();
@@ -83,6 +87,9 @@ public class Main {
 			String redCapFile = line.getOptionValue("red-cap");
 			appointmentDao.addAppointments(processRedCapAppointments(redCapFile));
 
+			String redCapCalendarFile = line.getOptionValue("red-cap-calendar");
+			appointmentDao.addAppointments(processRedCapCalendar(redCapCalendarFile));
+
 			System.out.println("delete from web_visit_appointment_types;");
 			System.out.println("delete from web_appointment_appointment_types;");
 			System.out.println("delete from web_subject_languages;");
@@ -110,6 +117,14 @@ public class Main {
 		}
 	}
 
+	private List<AppointmentEntry> processRedCapCalendar(String redCapCalendarFile) throws FileNotFoundException, IOException {
+		RedcapCalendarParser redcapCalendarParser = new RedcapCalendarParser();
+		redcapCalendarParser.setSubjectDao(subjectDao);
+		Calendar date = Calendar.getInstance();
+		date.set(Calendar.HOUR, 0);
+		return redcapCalendarParser.parse(redCapCalendarFile, date);
+	}
+
 	private void removeDuplicates(String duplicatesFile) throws IOException {
 		logger.info("PROCESSING FILE: " + duplicatesFile);
 		DuplicateRemoveParser parser = new DuplicateRemoveParser();

appointment-import/src/main/java/smash/appointment/parse/RedcapCalendarParser.java

@@ -81,7 +81,7 @@ public class RedcapCalendarParser {
 		if (subject == null) {
 			subject = findSubject(query);
 		}
-		if (subject != null && !subject.getToBeSeenAt().equalsIgnoreCase("LIH")) {
+		if (subject != null && !subject.getToBeSeenAt().toLowerCase().startsWith("l")) {
 			return null;
 		}
 		result.setLocation("LIH");
@@ -93,7 +93,12 @@ public class RedcapCalendarParser {
 		if (result.getTypes().contains(AppointmentType.OTHER)) {
 			logger.warn("Cannot find types for: " + query);
 		}
-		return result;
+		if (result.getTypes().contains(AppointmentType.LEVEL_ASAMP)) {
+			return result;
+		} else {
+			logger.debug("Skipping LIH appointment: " + query);
+			return null;
+		}
 	}
 
 	CellParser cellParser = new CellParser();
@@ -107,6 +112,14 @@ public class RedcapCalendarParser {
 				query = query.substring(index);
 				if (query.startsWith("_lev a_")) {
 					collection = AppointmentTypeCollection.LEVEL_A;
+				} else {
+					index = query.indexOf("_", 1);
+					if (index >= 0) {
+						query = query.substring(index);
+						if (query.startsWith("_lev a_")) {
+							collection = AppointmentTypeCollection.LEVEL_A;
+						}
+					}
 				}
 			}
 			if (collection == null) {

appointment-import/src/main/java/smash/appointment/parse/RedcapMissingSqlExporter.java

+package smash.appointment.parse;
+
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+
+public class RedcapMissingSqlExporter {
+	static SubjectDao subjectDao = new SubjectDao();
+	
+	public static void main(String[] args) throws Exception {
+		List<Subject> subjects = new ArrayList<>();
+		for (int count =0 ;count<10000;count++) {
+			String nd = count+"";
+			while (nd.length()<4) {
+				nd = "0"+nd;
+			}
+			Subject subject = new Subject("name "+nd, "surname"+nd, "ND"+nd, count+"");
+			
+			String screening = nd;
+			if (screening.startsWith("0")) {
+				screening = screening.substring(1);
+			}
+			subject.setScreeningNumber("P-"+screening);
+			subject.setToBeSeenAt("PRC");
+			if (count>=3000) {
+				screening=screening.substring(1);
+				subject.setScreeningNumber("L-"+screening);
+				subject.setToBeSeenAt("LIH");
+			}
+			if (count>=7000) {
+				subject.setScreeningNumber("F-"+nd);
+				subject.setToBeSeenAt("FLYING TEAM");
+			}
+			subjects.add(subject);
+		}
+		subjectDao.setSubjects(subjects);
+		
+		RedcapCalendarParser parser = new RedcapCalendarParser();
+		parser.setSubjectDao(subjectDao);
+		Calendar date = Calendar.getInstance();
+		date.set(2017, 2, 21, 1, 0, 0);
+		List<AppointmentEntry> entries = parser.parse("E:/redcap_events_calendar-20032017.sql", date);
+		
+		VisitSqlExporter sqlExporter = new VisitSqlExporter();
+		for (AppointmentEntry entry: entries) {
+			Visit visit = new Visit(entry.getSubject());
+			visit.addAppointment(entry);
+			visit.setStartDate(entry.getDay());
+			System.out.println(sqlExporter.toSql(visit));
+		}
+	}
+}

appointment-import/src/main/java/smash/appointment/parse/RedcapParser.java

@@ -47,35 +47,32 @@ public class RedcapParser {
 							subject.setBirthDate(tmp[2]);
 						}
 
-						// for now only PRC and flying team subjects
-						if (subject.getToBeSeenAt() == null || subject.getToBeSeenAt().startsWith("P") || subject.getToBeSeenAt().startsWith("F")) {
-							List<AppointmentEntry> subjectAppointments = new ArrayList<>();
-							if (!tmp[9].isEmpty()) {
-								subjectAppointments.add(createEntryLevelA(tmp[9], tmp[10], subject));
-							}
-
-							if (!tmp[13].isEmpty()) {
-								subjectAppointments.add(createEntryLevelB(tmp[13], tmp[14], subject));
-							}
-
-							if (!tmp[17].isEmpty()) {
-								subjectAppointments.add(createEntryLevelBG(tmp[17], tmp[18], subject));
-							}
-
-							if (!tmp[21].isEmpty()) {
-								subjectAppointments.add(createEntryLevelBV(tmp[21], tmp[22], subject));
-							}
-
-							if (!tmp[41].isEmpty()) {
-								subjectAppointments.add(createEntryLevelSB(tmp[41], subject));
-							}
-
-							if (!tmp[45].isEmpty()) {
-								subjectAppointments.add(createEntryLevelMPower(tmp[45], subject));
-							}
-
-							result.addAll(subjectAppointments);
+						List<AppointmentEntry> subjectAppointments = new ArrayList<>();
+						if (!tmp[9].isEmpty()) {
+							subjectAppointments.add(createEntryLevelA(tmp[9], tmp[10], subject));
 						}
+
+						if (!tmp[13].isEmpty()) {
+							subjectAppointments.add(createEntryLevelB(tmp[13], tmp[14], subject));
+						}
+
+						if (!tmp[17].isEmpty()) {
+							subjectAppointments.add(createEntryLevelBG(tmp[17], tmp[18], subject));
+						}
+
+						if (!tmp[21].isEmpty()) {
+							subjectAppointments.add(createEntryLevelBV(tmp[21], tmp[22], subject));
+						}
+
+						if (!tmp[41].isEmpty()) {
+							subjectAppointments.add(createEntryLevelSB(tmp[41], subject));
+						}
+
+						if (!tmp[45].isEmpty()) {
+							subjectAppointments.add(createEntryLevelMPower(tmp[45], subject));
+						}
+
+						result.addAll(subjectAppointments);
 					}
 				}
 

appointment-import/src/main/java/smash/appointment/parse/SubjectDao.java

@@ -32,8 +32,10 @@ public class SubjectDao {
 
 	Subject getByScreeningNumber(String screeningNumber) {
 		for (Subject s : subjects) {
-			if (screeningNumber.equals(s.getScreeningNumber())) {
-				return s;
+			for (String number: s.getScreeningNumber().split(";")) {
+				if (screeningNumber.equals(number.trim())) {
+					return s;
+				}				
 			}
 		}
 		return null;

appointment-import/src/main/java/smash/appointment/parse/VisitSqlExporter.java

@@ -16,7 +16,7 @@ public class VisitSqlExporter extends SqlExporter {
 		result.append("is_finished)");
 
 		result.append("values (");
-		result.append("(SELECT id from web_subject where screening_number = "+getStringVal(visit.getSubject().getScreeningNumber()) + "),");
+		result.append("(SELECT id from web_subject where screening_number like "+getStringVal('%'+visit.getSubject().getScreeningNumber()+'%') + "),");
 		result.append(getStringVal(visit.getStartDate()) + ",");
 		result.append(getStringVal(visit.getEndDate()) + ",");
 		result.append("false,");