diff --git a/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.cpp b/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.cpp index 55ec8865e0745c90f7d80a5b93b48b07d0f4fc74..78590a7eb5c7475c93c5377c76645d26c8a11936 100644 --- a/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.cpp +++ b/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.cpp @@ -54,7 +54,7 @@ bool BufferedStreamMgr::init() if (_inputStreamMgr->isGzipped()) { _useBufSize = GZIP_LINE_BUF_SIZE; } else { - _useBufSize = MAIN_BUF_READ_SIZE; + _useBufSize = 67108863; //64Mb -1 } size_t trueBufSize = max(_useBufSize, (int)_currScanBuffer.size()); diff --git a/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.h b/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.h index 424766c5b3864c9f1e60a193a953610196ddc84b..ca9d2545d89488b353a9fc8a5edcabd30f1f5db5 100644 --- a/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.h +++ b/src/utils/FileRecordTools/FileReaders/BufferedStreamMgr.h @@ -23,7 +23,6 @@ public: bool init(); const FileRecordTypeChecker & getTypeChecker() const { return _typeChecker; } -// istream *getStream() { return _inputStreamMgr->getFinalStream(); } bool eof() const { return _eof; } bool getLine(QuickString &line); @@ -44,12 +43,10 @@ private: bool _streamFinished; QuickString _currScanBuffer; //The minus ones in these constants are for leaving room for a null terminator after reading into buffers. - static const int MAIN_BUF_READ_SIZE = 67108863; //64 Mb minus 1 - static const int TYPE_CHECK_READ_SIZE = 4095; // 4K +// static const int MAIN_BUF_READ_SIZE = 2047; //64 Mb minus 1 static const int GZIP_LINE_BUF_SIZE = 8191; // 8K bool readFileChunk(); bool getTypeData(); -// void resetStream(); }; diff --git a/src/utils/FileRecordTools/RecordOutputMgr.h b/src/utils/FileRecordTools/RecordOutputMgr.h index 1bb8bbfcadb4f84e98a9a2039a63f3e33d618bd1..cb1bbdfc7982bf6dfb0aeea5e3d95cf7e5576ba9 100644 --- a/src/utils/FileRecordTools/RecordOutputMgr.h +++ b/src/utils/FileRecordTools/RecordOutputMgr.h @@ -52,7 +52,7 @@ private: void reportOverlapDetail(const Record *keyRecord, const Record *hitRecord); void reportOverlapSummary(RecordKeyList &keyList); - static const unsigned int MAX_OUTBUF_SIZE = 16384; //16 K + static const unsigned int MAX_OUTBUF_SIZE = 16386; //16 K bool needsFlush() const { return _outBuf.size() >= MAX_OUTBUF_SIZE *.9; } void flush(); }; diff --git a/src/utils/fileType/FileRecordTypeChecker.h b/src/utils/fileType/FileRecordTypeChecker.h index 4eb3663bd1c9b5d06688e4e98028621e7813ac51..6d598acea456c8a2e5852a97ca8c3dacd0d72b4e 100644 --- a/src/utils/fileType/FileRecordTypeChecker.h +++ b/src/utils/fileType/FileRecordTypeChecker.h @@ -85,7 +85,6 @@ private: vector<QuickString> _lines; vector<QuickString> _currLineElems; int _firstValidDataLineIdx; - static const int SCAN_BUFFER_SIZE = 8192; //8 KB buffer int _numBytesInBuffer; //this will hold the length of the buffer after the scan. int _numFields;