Commit fba95089 authored by nkindlon's avatar nkindlon
Browse files

Changed input stream buffer to only 1Kb to improve performance for small files.

parent 074a146c
......@@ -54,7 +54,7 @@ bool BufferedStreamMgr::init()
if (_inputStreamMgr->isGzipped()) {
_useBufSize = GZIP_LINE_BUF_SIZE;
} else {
_useBufSize = MAIN_BUF_READ_SIZE;
_useBufSize = 67108863; //64Mb -1
}
size_t trueBufSize = max(_useBufSize, (int)_currScanBuffer.size());
......
......@@ -23,7 +23,6 @@ public:
bool init();
const FileRecordTypeChecker & getTypeChecker() const { return _typeChecker; }
// istream *getStream() { return _inputStreamMgr->getFinalStream(); }
bool eof() const { return _eof; }
bool getLine(QuickString &line);
......@@ -44,12 +43,10 @@ private:
bool _streamFinished;
QuickString _currScanBuffer;
//The minus ones in these constants are for leaving room for a null terminator after reading into buffers.
static const int MAIN_BUF_READ_SIZE = 67108863; //64 Mb minus 1
static const int TYPE_CHECK_READ_SIZE = 4095; // 4K
// static const int MAIN_BUF_READ_SIZE = 2047; //64 Mb minus 1
static const int GZIP_LINE_BUF_SIZE = 8191; // 8K
bool readFileChunk();
bool getTypeData();
// void resetStream();
};
......
......@@ -52,7 +52,7 @@ private:
void reportOverlapDetail(const Record *keyRecord, const Record *hitRecord);
void reportOverlapSummary(RecordKeyList &keyList);
static const unsigned int MAX_OUTBUF_SIZE = 16384; //16 K
static const unsigned int MAX_OUTBUF_SIZE = 16386; //16 K
bool needsFlush() const { return _outBuf.size() >= MAX_OUTBUF_SIZE *.9; }
void flush();
};
......
......@@ -85,7 +85,6 @@ private:
vector<QuickString> _lines;
vector<QuickString> _currLineElems;
int _firstValidDataLineIdx;
static const int SCAN_BUFFER_SIZE = 8192; //8 KB buffer
int _numBytesInBuffer; //this will hold the length of the buffer after the scan.
int _numFields;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment