Merge pull request #265 from nkindlon/master

1st check-in: Converted groupby to PFM

src/bedtools.cpp

@@ -54,7 +54,7 @@ int fastafrombed_main(int argc, char* argv[]);//
 int flank_main(int argc, char* argv[]); //
 int genomecoverage_main(int argc, char* argv[]);//
 int getoverlap_main(int argc, char* argv[]);//
-int groupby_main(int argc, char* argv[]);//
+void groupby_help();
 void intersect_help();
 void map_help();
 void jaccard_help(); //
@@ -146,7 +146,6 @@ int main(int argc, char *argv[])
     else if (subCmd == "igv")         return bedtoigv_main(argc-1, argv+1);
     else if (subCmd == "links")       return links_main(argc-1, argv+1);
     else if (subCmd == "makewindows") return windowmaker_main(argc-1, argv+1);
-    else if (subCmd == "groupby")     return groupby_main(argc-1, argv+1);
     else if (subCmd == "expand")      return expand_main(argc-1, argv+1);
     else if (subCmd == "neksb1")       return nek_sandbox1_main(argc-1, argv+1);
     else if (subCmd == "regresstest")  return regress_test_main(argc, argv); //this command does need all the orig args.
@@ -308,6 +307,9 @@ void showHelp(const QuickString &subCmd) {
 		coverage_help();
 	} else if (subCmd == "complement") {
 		complement_help();
+	} else if (subCmd == "groupby") {
+		groupby_help();
 	}
 
+
 }

src/groupBy/Makefile

 UTILITIES_DIR = ../utils/
 OBJ_DIR = ../../obj/
 BIN_DIR = ../../bin/
-
+TOOL_DIR = ../../src/
 
 # -------------------
 # define our includes
 # -------------------
-INCLUDES = -I$(UTILITIES_DIR)/tabFile/ \
-           -I$(UTILITIES_DIR)/lineFileUtilities/ \
-           -I$(UTILITIES_DIR)/gzstream/ \
-           -I$(UTILITIES_DIR)/fileType/ \
-           -I$(UTILITIES_DIR)/VectorOps/ \
-           -I$(UTILITIES_DIR)/version/
+INCLUDES = -I$(UTILITIES_DIR)/Contexts/ \
+			-I$(UTILITIES_DIR)/general/ \
+			-I$(UTILITIES_DIR)/fileType/ \
+			-I$(UTILITIES_DIR)/gzstream/ \
+           -I$(UTILITIES_DIR)/GenomeFile/ \
+           -I$(UTILITIES_DIR)/BamTools/include \
+           -I$(UTILITIES_DIR)/BamTools/src \
+           -I$(UTILITIES_DIR)/BlockedIntervals \
+           -I$(UTILITIES_DIR)/BamTools-Ancillary \
+           -I$(UTILITIES_DIR)/FileRecordTools/ \
+           -I$(UTILITIES_DIR)/FileRecordTools/FileReaders/ \
+           -I$(UTILITIES_DIR)/FileRecordTools/Records/ \
+			-I$(UTILITIES_DIR)/KeyListOps/ \
+           -I$(UTILITIES_DIR)/RecordOutputMgr/ \
+           -I$(UTILITIES_DIR)/NewChromsweep \
+           -I$(UTILITIES_DIR)/BinTree \
+           -I$(UTILITIES_DIR)/version/ \
+           -I$(UTILITIES_DIR)/ToolBase/ \
+           -I$(TOOL_DIR)/intersectFile/
+            
 
 # ----------------------------------
 # define our source and object files
 # ----------------------------------
-SOURCES= groupBy.cpp $(UTILITIES_DIR)/lineFileUtilities/lineFileUtilities.h
-OBJECTS= groupBy.o
+SOURCES= groupByHelp.cpp groupBy.cpp groupBy.h
+OBJECTS= groupBy.o groupByHelp.o
 BUILT_OBJECTS= $(patsubst %,$(OBJ_DIR)/%,$(OBJECTS))
+PROGRAM= groupBy
 
 all: $(BUILT_OBJECTS)
 
 .PHONY: all
 
-
 $(BUILT_OBJECTS): $(SOURCES)
 	@echo "  * compiling" $(*F).cpp
-	@$(CXX) -c -o $@ $(*F).cpp $(LDFLAGS) $(CXXFLAGS) $(INCLUDES)
+	@$(CXX) -c -o $@ $(*F).cpp $(LDFLAGS) $(CXXFLAGS) $(DFLAGS) $(INCLUDES)
 	
 clean:
 	@echo "Cleaning up."
-	@rm -f $(OBJ_DIR)/groupBy.o
+	@rm -f $(OBJ_DIR)/groupBy.o $(OBJ_DIR)/groupBy.o
 
 .PHONY: clean

src/groupBy/groupBy.h

+/* groupBy.h
+ *
+ *  Created on: Apr 22, 2015
+ *      Author: nek3d
+ */
+
+#ifndef GROUPBY_H_
+#define GROUPBY_H_
+
+#include "ToolBase.h"
+#include "ContextGroupBy.h"
+
+class GroupBy : public ToolBase {
+
+public:
+    GroupBy(ContextGroupBy *context);
+    ~GroupBy();
+	virtual bool init(); // after construction
+	virtual bool findNext(RecordKeyVector &hits);
+	virtual void processHits(RecordOutputMgr *outputMgr, RecordKeyVector &hits);
+	virtual void cleanupHits(RecordKeyVector &hits);
+	//do any last things needed to wrap up.
+	virtual bool finalizeCalculations() { return true;}
+	virtual void  giveFinalReport(RecordOutputMgr *outputMgr) {}
+
+protected:
+	virtual ContextGroupBy *upCast(ContextBase *context) { return static_cast<ContextGroupBy *>(context); }
+
+	vector<int> _groupCols;
+	vector<QuickString> _prevFields;
+	FileRecordMgr *_queryFRM;
+	const Record *_prevRecord;
+	const Record *getNextRecord();
+	bool canGroup(const Record *);
+	void assignPrevFields();
+};
+
+
+
+
+#endif /* GROUPBY_H_ */

src/groupBy/groupByHelp.cpp

+#include "CommonHelp.h"
+
+void groupby_help(void) {
+
+    cerr << "\nTool:    bedtools groupby " << endl;
+    cerr << "Version: " << VERSION << "\n";
+    cerr << "Summary: Summarizes a dataset column based upon" << endl;
+    cerr << "\t common column groupings. Akin to the SQL \"group by\" command." << endl << endl;
+
+    cerr << "Usage:\t " << "bedtools groupby" << " -g [group_column(s)] -c [op_column(s)] -o [ops] " << endl;
+    cerr << "\t "     << "cat [FILE] | " << "bedtools groupby" << " -g [group_column(s)] -c [op_column(s)] -o [ops] " << endl << endl;
+
+    cerr << "Options: " << endl;
+    cerr << "\t-i\t\t"        << "Input file. Assumes \"stdin\" if omitted." << endl << endl;
+
+    cerr << "\t-g -grp\t\t"      << "Specify the columns (1-based) for the grouping." << endl;
+    cerr                         << "\t\t\tThe columns must be comma separated." << endl;
+    cerr                         << "\t\t\t- Default: 1,2,3" << endl << endl;
+
+    cerr << "\t-c -opCols\t"     << "Specify the column (1-based) that should be summarized." << endl;
+    cerr                         << "\t\t\t- Required." << endl << endl;
+
+    cerr << "\t-o -ops\t\t"      << "Specify the operation that should be applied to opCol." << endl;
+    cerr                         << "\t\t\tValid operations:" << endl;
+    cerr                         << "\t\t\t    sum, count, count_distinct, min, max," << endl;
+    cerr                         << "\t\t\t    mean, median, mode, antimode," << endl;
+    cerr                         << "\t\t\t    stdev, sstdev (sample standard dev.)," << endl;
+    cerr                         << "\t\t\t    collapse (i.e., print a comma separated list (duplicates allowed)), " << endl;
+    cerr                         << "\t\t\t    distinct (i.e., print a comma separated list (NO duplicates allowed)), " << endl;
+    cerr                         << "\t\t\t    distinct_sort_num (as distinct, but sorted numerically, ascending), " << endl;
+    cerr                         << "\t\t\t    distinct_sort_num_desc (as distinct, but sorted numerically, descending), " << endl;
+    cerr                         << "\t\t\t    concat   (i.e., merge values into a single, non-delimited string), " << endl;
+    cerr                         << "\t\t\t    freqdesc (i.e., print desc. list of values:freq)" << endl;
+    cerr                         << "\t\t\t    freqasc (i.e., print asc. list of values:freq)" << endl;
+    cerr                         << "\t\t\t    first (i.e., print first value)" << endl;
+    cerr                         << "\t\t\t    last (i.e., print last value)" << endl;
+
+    cerr                         << "\t\t\t- Default: sum" << endl << endl;
+
+    cerr						<< "\t\tIf there is only column, but multiple operations, all operations will be" << endl;
+    cerr						<< "\t\tapplied on that column. Likewise, if there is only one operation, but" << endl;
+    cerr						<< "\t\tmultiple columns, that operation will be applied to all columns." << endl;
+    cerr						<< "\t\tOtherwise, the number of columns must match the the number of operations," << endl;
+    cerr						<< "\t\tand will be applied in respective order." << endl;
+    cerr						<< "\t\tE.g., \"-c 5,4,6 -o sum,mean,count\" will give the sum of column 5," << endl;
+    cerr						<< "\t\tthe mean of column 4, and the count of column 6." << endl;
+    cerr						<< "\t\tThe order of output columns will match the ordering given in the command." << endl << endl<<endl;
+
+    cerr << "\t-full\t\t"   << "Print all columns from input file.  The first line in the group is used." << endl;
+    cerr            << "\t\t\tDefault: print only grouped columns." << endl << endl;
+
+    cerr << "\t-inheader\t" << "Input file has a header line - the first line will be ignored." << endl << endl ;
+
+    cerr << "\t-outheader\t"    << "Print header line in the output, detailing the column names. " << endl;
+    cerr            << "\t\t\tIf the input file has headers (-inheader), the output file" << endl;
+    cerr            << "\t\t\twill use the input's column names." << endl;
+    cerr            << "\t\t\tIf the input file has no headers, the output file" << endl;
+    cerr            << "\t\t\twill use \"col_1\", \"col_2\", etc. as the column names." << endl << endl;
+
+    cerr << "\t-header\t\t" << "same as '-inheader -outheader'" << endl << endl;
+
+    cerr << "\t-ignorecase\t"   << "Group values regardless of upper/lower case." << endl << endl;
+
+    cerr << "\t-prec\t"   << "Sets the decimal precision for output (Default: 5)" << endl << endl;
+    cerr << "\t-delim\t"                 << "Specify a custom delimiter for the collapse operations." << endl;
+    cerr                                 << "\t\t- Example: -delim \"|\"" << endl;
+    cerr                                 << "\t\t- Default: \",\"." << endl << endl;
+
+    cerr << "Examples: " << endl;
+    cerr << "\t$ cat ex1.out" << endl;
+    cerr << "\tchr1 10  20  A   chr1    15  25  B.1 1000    ATAT" << endl;
+    cerr << "\tchr1 10  20  A   chr1    25  35  B.2 10000   CGCG" << endl << endl;
+    cerr << "\t$ groupBy -i ex1.out -g 1,2,3,4 -c 9 -o sum" << endl;
+    cerr << "\tchr1 10  20  A   11000" << endl << endl;
+    cerr << "\t$ groupBy -i ex1.out -grp 1,2,3,4 -opCols 9,9 -ops sum,max" << endl;
+    cerr << "\tchr1 10  20  A   11000   10000" << endl << endl;
+    cerr << "\t$ groupBy -i ex1.out -g 1,2,3,4 -c 8,9 -o collapse,mean" << endl;
+    cerr << "\tchr1 10  20  A   B.1,B.2,    5500" << endl << endl;
+    cerr << "\t$ cat ex1.out | groupBy -g 1,2,3,4 -c 8,9 -o collapse,mean" << endl;
+    cerr << "\tchr1 10  20  A   B.1,B.2,    5500" << endl << endl;
+    cerr << "\t$ cat ex1.out | groupBy -g 1,2,3,4 -c 10 -o concat" << endl;
+    cerr << "\tchr1 10  20  A   ATATCGCG" << endl << endl;
+
+    cerr << "Notes: " << endl;
+    cerr << "\t(1)  The input file/stream should be sorted/grouped by the -grp. columns" << endl;
+    cerr << "\t(2)  If -i is unspecified, input is assumed to come from stdin." << endl << endl;
+
+
+    // end the program here
+    exit(1);
+
+}

src/utils/Contexts/ContextBase.cpp

@@ -62,6 +62,8 @@ ContextBase::ContextBase()
   _splitBlockInfo(NULL),
   _allFilesHaveChrInChromNames(UNTESTED),
   _allFileHaveLeadingZeroInChromNames(UNTESTED),
+  _noEnforceCoordSort(false),
+  _inheader(false),
   _nameConventionWarningTripped(false)
 
 {
@@ -74,6 +76,12 @@ ContextBase::ContextBase()
 	_programNames["jaccard"] = JACCARD;
 	_programNames["spacing"] = SPACING;
 	_programNames["fisher"] = FISHER;
+	_programNames["sample"] = SAMPLE;
+	_programNames["coverage"] = COVERAGE;
+	_programNames["complement"] = COMPLEMENT;
+	_programNames["groupby"] = GROUP_BY;
+
+
 
 	if (hasColumnOpsMethods()) {
 		_keyListOps = new KeyListOps();
@@ -241,6 +249,12 @@ bool ContextBase::isValidState()
 	if (!determineOutputType()) {
 		return false;
 	}
+	if (_program != GROUP_BY && _files[0]->getRecordType() == FileRecordTypeChecker::NO_POS_PLUS_RECORD_TYPE) {
+		_errorMsg = "ERROR: file ";
+		_errorMsg.append(_files[0]->getFileName());
+		_errorMsg.append(" has non positional records, which are only valid for the groupBy tool.");
+		return false;
+	}
 	if (getObeySplits()) {
 		_splitBlockInfo = new BlockMgr(_overlapFraction, _reciprocal);
 	}
@@ -314,9 +328,12 @@ bool ContextBase::openFiles() {
 		frm->setFullBamFlags(_useFullBamTags);
 		frm->setIsSorted(_sortedInput);
 		frm->setIoBufSize(_ioBufSize);
-		if (!frm->open()) {
+		if (!frm->open(_inheader)) {
 			return false;
 		}
+		if (_noEnforceCoordSort) {
+			frm->setNoEnforceCoodSort(true);
+		}
 		_files[i] = frm;
 	}
 	_allFilesOpened = true;

src/utils/Contexts/ContextBase.h

@@ -38,7 +38,7 @@ public:
 	typedef enum {UNSPECIFIED_PROGRAM, INTERSECT, WINDOW, CLOSEST, COVERAGE, MAP, GENOMECOV, MERGE, CLUSTER,
 		COMPLEMENT, SUBTRACT, SLOP, FLANK, SORT, RANDOM, SAMPLE, SHUFFLE, ANNOTATE, MULTIINTER, UNIONBEDG, PAIRTOBED,
 		PAIRTOPAIR,BAMTOBED, BEDTOBAM, BEDTOFASTQ, BEDPETOBAM, BED12TOBED6, GETFASTA, MASKFASTA, NUC,
-		MULTICOV, TAG, JACCARD, OVERLAP, IGV, LINKS,MAKEWINDOWS, GROUPBY, EXPAND, SPACING, FISHER} PROGRAM_TYPE;
+		MULTICOV, TAG, JACCARD, OVERLAP, IGV, LINKS,MAKEWINDOWS, GROUPBY, EXPAND, SPACING, FISHER, GROUP_BY} PROGRAM_TYPE;
 
 	PROGRAM_TYPE getProgram() const { return _program; }
 	FileRecordMgr *getFile(int fileIdx) { return _files[fileIdx]; }
@@ -230,7 +230,7 @@ protected:
 	bool isUsed(int i) const { return _argsProcessed[i]; }
 	virtual bool parseCmdArgs(int argc, char **argv, int skipFirstArgs);
 	bool cmdArgsValid();
-	bool openFiles();
+	virtual bool openFiles();
 	virtual FileRecordMgr *getNewFRM(const QuickString &filename, int fileIdx);
 
 	//set cmd line params and counter, i, as members so code
@@ -254,6 +254,9 @@ protected:
 
     testType _allFilesHaveChrInChromNames;
     testType _allFileHaveLeadingZeroInChromNames;
+    bool _noEnforceCoordSort;
+	bool _inheader;
+
 
     virtual bool handle_bed();
 	virtual bool handle_fbam();
@@ -282,6 +285,7 @@ protected:
     testType fileHasChrInChromNames(int fileIdx);
     testType fileHasLeadingZeroInChromNames(int fileIdx);
 
+    void setNoEnforceCoordSort(bool val) { _noEnforceCoordSort = val; }
     //Warning messages.
    bool _nameConventionWarningTripped;
    QuickString _nameConventionWarningMsg;

src/utils/Contexts/ContextClosest.cpp

@@ -31,17 +31,6 @@ ContextClosest::~ContextClosest(){
 }
 
 bool ContextClosest::parseCmdArgs(int argc, char **argv, int skipFirstArgs){
-	_argc = argc;
-	_argv = argv;
-	_skipFirstArgs = skipFirstArgs;
-	if (_argc < 2) {
-		setShowHelp(true);
-		return false;
-	}
-
-	setProgram(_programNames[argv[0]]);
-
-	_argsProcessed.resize(_argc - _skipFirstArgs, false);
 
 	for (_i=_skipFirstArgs; _i < argc; _i++) {
 		if (isUsed(_i - _skipFirstArgs)) {

src/utils/Contexts/ContextGroupBy.cpp

+/*
+ * ContextGroupBy.cpp
+ *
+ *  Created on: Mar 26, 2014
+ *      Author: nek3d
+ */
+
+
+#include "ContextGroupBy.h"
+
+ContextGroupBy::ContextGroupBy()
+: _printFullCols(false),
+  _ignoreCase(false)
+{
+	setSortedInput(true);
+	_noEnforceCoordSort = true;
+	setColumnOpsMethods(true);
+	setExplicitBedOutput(true);
+
+	//For columnOps, groupBy has default operation sum but no default column,
+	///so we have to clear that.
+	//
+	_keyListOps->setColumns("");
+
+}
+
+ContextGroupBy::~ContextGroupBy()
+{
+
+}
+
+
+bool ContextGroupBy::parseCmdArgs(int argc, char **argv, int skipFirstArgs)
+{
+	for (_i=_skipFirstArgs; _i < argc; _i++) {
+		if (isUsed(_i - _skipFirstArgs)) {
+			continue;
+		}
+		else if (strcmp(_argv[_i], "-g") == 0) {
+			if (!handle_g()) return false;
+		}
+		else if (strcmp(_argv[_i], "-inheader") == 0) {
+			if (!handle_inheader()) return false;
+		}
+		else if (strcmp(_argv[_i], "-outheader") == 0) {
+			if (!handle_outheader()) return false;
+		}
+		else if (strcmp(_argv[_i], "-header") == 0) {
+			if (!handle_header()) return false;
+		}
+		else if (strcmp(_argv[_i], "-full") == 0) {
+			if (!handle_full()) return false;
+		}
+		else if (strcmp(_argv[_i], "-ignorecase") == 0) {
+			if (!handle_ignorecase()) return false;
+		}
+	}
+	return ContextBase::parseCmdArgs(argc, argv, _skipFirstArgs);
+}
+
+bool ContextGroupBy::isValidState()
+{
+	// The user was required to have entered one or more columns
+	if (_keyListOps->getColumns().empty()) {
+		_errorMsg = "***** ERROR: -opCols parameter requires a value.";
+		return false;
+	}
+
+	//default to stdin
+	if (getNumInputFiles() == 0) {
+		addInputFile("-");
+	}
+	//default grouping is cols 1,2,3
+	if (_groupStr.empty()) _groupStr = "1,2,3";
+
+	return ContextBase::isValidState();
+}
+
+
+bool ContextGroupBy::handle_g()
+{
+	if (_argc <= _i+1) {
+		_errorMsg = "\n***** ERROR: -g option given, but columns to group not specified. *****";
+		return false;
+	}
+	_groupStr = _argv[_i+1];
+	markUsed(_i - _skipFirstArgs);
+	_i++;
+	markUsed(_i - _skipFirstArgs);
+	return true;
+}
+
+bool ContextGroupBy::handle_inheader()
+{
+	_inheader = true;
+	markUsed(_i - _skipFirstArgs);
+	return true;
+}
+
+bool ContextGroupBy::handle_outheader() {
+	setPrintHeader(true);
+	markUsed(_i - _skipFirstArgs);
+	return true;
+}
+
+bool ContextGroupBy::handle_header() {
+	_inheader = true;
+	setPrintHeader(true);
+	markUsed(_i - _skipFirstArgs);
+	return true;
+}
+
+bool ContextGroupBy::handle_full() {
+	_printFullCols = true;
+	markUsed(_i - _skipFirstArgs);
+	return true;
+}
+
+bool ContextGroupBy::handle_ignorecase() {
+	_ignoreCase = true;
+	markUsed(_i - _skipFirstArgs);
+	return true;
+}
+
+const QuickString &ContextGroupBy::getDefaultHeader() {
+	//groupBy does not support multiple databases.
+	FileRecordMgr *frm = _files[0];
+	int numFields = frm->getNumFields();
+	_defaultHeader.clear();
+	for (int i=1; i <= numFields; i++) {
+		_defaultHeader.append("col_");
+		_defaultHeader.append(i);
+		_defaultHeader.append('\t');
+	}
+	//change last tab into newline
+	_defaultHeader[_defaultHeader.size()-1] = '\n';
+	return _defaultHeader;
+}

src/utils/Contexts/ContextGroupBy.h

+/*
+ * ContextGroupBy.h
+ *
+ *  Created on: Mar 26, 2014
+ *      Author: nek3d
+ */
+
+#ifndef CONTEXTGROUPBY_H_
+#define CONTEXTGROUPBY_H_
+
+#include "ContextBase.h"
+
+class ContextGroupBy: public ContextBase {
+public:
+	ContextGroupBy();
+	~ContextGroupBy();
+	virtual bool parseCmdArgs(int argc, char **argv, int skipFirstArgs);
+	virtual bool isValidState();
+
+	bool printFullCols() const { return _printFullCols; }
+	const QuickString &getGroupCols() const { return _groupStr; }
+	bool ignoreCase() const { return _ignoreCase; }
+	const QuickString &getDefaultHeader();
+
+protected:
+	QuickString _groupStr;
+	QuickString _defaultHeader;
+	bool _printFullCols;
+	bool _ignoreCase;
+
+	bool handle_g();
+	bool handle_inheader();
+	bool handle_outheader();
+	bool handle_header();
+	bool handle_full();
+	bool handle_ignorecase();
+};
+
+
+#endif /* CONTEXTGROUPBY_H_ */

src/utils/Contexts/Makefile

@@ -21,9 +21,10 @@ INCLUDES =  -I$(UTILITIES_DIR)/general/ \
 # ----------------------------------
 SOURCES= ContextBase.cpp ContextBase.h ContextIntersect.cpp ContextIntersect.h ContextFisher.cpp ContextFisher.h ContextMap.cpp \
 	ContextMap.h ContextSample.cpp ContextSpacing.cpp ContextSample.h ContextSpacing.h ContextMerge.h ContextMerge.cpp ContextJaccard.h ContextJaccard.cpp \
-	ContextClosest.cpp ContextClosest.h ContextSubtract.cpp ContextSubtract.h ContextCoverage.cpp ContextCoverage.h ContextComplement.cpp ContextComplement.h
+	ContextClosest.cpp ContextClosest.h ContextSubtract.cpp ContextSubtract.h ContextCoverage.cpp ContextCoverage.h ContextComplement.cpp ContextComplement.h \
+	ContextGroupBy.cpp ContextGroupBy.cpp
 OBJECTS= ContextBase.o ContextIntersect.o ContextFisher.o ContextMap.o ContextSample.o ContextSpacing.o ContextMerge.o ContextJaccard.o ContextClosest.o \
-	ContextSubtract.o ContextCoverage.o ContextComplement.o
+	ContextSubtract.o ContextCoverage.o ContextComplement.o ContextGroupBy.o
 _EXT_OBJECTS=ParseTools.o QuickString.o
 EXT_OBJECTS=$(patsubst %,$(OBJ_DIR)/%,$(_EXT_OBJECTS))
 BUILT_OBJECTS= $(patsubst %,$(OBJ_DIR)/%,$(OBJECTS))
@@ -47,7 +48,8 @@ clean:
 	       $(OBJ_DIR)/ContextMerge.o \
 	       $(OBJ_DIR)/ContextJaccard.o \
 	       $(OBJ_DIR)/ContextClosest.o \
-			$(OBJ_DIR)/ContextSubtract.o \
-			$(OBJ_DIR)/ContextCoverage.o \
-			$(OBJ_DIR)/ContextComplement.o 
+		   $(OBJ_DIR)/ContextSubtract.o \
+		   $(OBJ_DIR)/ContextCoverage.o \
+		   $(OBJ_DIR)/ContextComplement.o \
+		   $(OBJ_DIR)/ContextGroupBy.o
 .PHONY: clean

src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.h

@@ -20,7 +20,7 @@ public:
 
 	bool init();
 
-	const FileRecordTypeChecker & getTypeChecker() const { return _typeChecker; }
+	FileRecordTypeChecker & getTypeChecker() { return _typeChecker; }
 
 	bool eof() const { return _eof; }
 	bool getLine(QuickString &line);

src/utils/FileRecordTools/FileReaders/SingleLineDelimTextFileReader.cpp

@@ -8,7 +8,8 @@ SingleLineDelimTextFileReader::SingleLineDelimTextFileReader(int numFields, char
   _delimChar(delimChar),
   _fullHeaderFound(false),
   _currDataPos(0),
-  _lineNum(0)
+  _lineNum(0),
+  _inheader(false)
 {
 	_delimPositions = new int[numFields +1];
 }
@@ -39,9 +40,9 @@ bool SingleLineDelimTextFileReader::readEntry()
 	//scan the whole header in one call.
 	bool wasHeader = false;
 	while (detectAndHandleHeader()) { //header line
+		_lineNum++;
 		if (!_bufStreamMgr->getLine(_sLine)) {
 			return false;
-			_lineNum++;
 		}
 	}
 	//after the first time we find a header, any other header line
@@ -108,7 +109,7 @@ bool SingleLineDelimTextFileReader::detectAndHandleHeader()
 	//passing a non-const QuickString to isHeaderLine, but
 	//this const ref is a workaround.
 	const QuickString &sLine2 = _sLine;
-	if (!isHeaderLine(sLine2)) {
+	if (!isHeaderLine(sLine2) && (!(_inheader && _lineNum==1))) {
 		return false;
 	}
 	if (!_fullHeaderFound) {

src/utils/FileRecordTools/FileReaders/SingleLineDelimTextFileReader.h

@@ -28,7 +28,7 @@ public:
 	virtual void appendField(int fieldNum, QuickString &str) const;
 	virtual const QuickString &getHeader() const { return _header; }
 	virtual bool hasHeader() const { return _fullHeaderFound; }
-
+	virtual void setInHeader(bool val) { _inheader = val; }
 
 protected:
 	int _numFields;
@@ -41,6 +41,7 @@ protected:
 	QuickString _currChromStr;
 	QuickString _tempChrPosStr;
 	int _lineNum;
+	bool _inheader;
 	bool detectAndHandleHeader();
 	bool findDelimiters();
 

src/utils/FileRecordTools/FileRecordMgr.cpp

@@ -25,8 +25,9 @@ FileRecordMgr::FileRecordMgr(const QuickString &filename)
   _bamReader(NULL),
   _hasGenomeFile(false),
   _genomeFile(NULL),
-  _ioBufSize(0)
-{
+  _ioBufSize(0),
+  _noEnforceCoordSort(false)
+ {
 }