Performance improvements

Author: aacuevas

Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.cpp

@@ -41,7 +41,7 @@
 #endif
 
 #ifndef TIMESTAMP_CHUNK_SIZE
-#define TIMESTAMP_CHUNK_SIZE 4
+#define TIMESTAMP_CHUNK_SIZE 16
 #endif
 
 #define MAX_TRANSFORM_SIZE 512
@@ -101,7 +101,7 @@ int HDF5FileBase::open(bool newfile, int nChans)
 		FileAccPropList props = FileAccPropList::DEFAULT;
 		if (nChans > 0)
 		{
-			props.setCache(0, 809, 8 * 2 * CHUNK_XSIZE * nChans, 1);
+			props.setCache(0, 1667, 2 * 8 * 2 * CHUNK_XSIZE * nChans, 1);
 			//std::cout << "opening HDF5 " << getFileName() << " with nchans: " << nChans << std::endl;
 		}
 

Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.cpp

@@ -23,7 +23,8 @@
 
 #include "HDF5Recording.h"
 #define MAX_BUFFER_SIZE 40960
-#define CHANNEL_TIMESTAMP_PREALLOC_SIZE 16
+#define CHANNEL_TIMESTAMP_PREALLOC_SIZE 128
+#define CHANNEL_TIMESTAMP_MIN_WRITE	32
 #define TIMESTAMP_EACH_NSAMPLES 1024
 
 HDF5Recording::HDF5Recording() : processorIndex(-1), hasAcquired(false), bufferSize(MAX_BUFFER_SIZE)
@@ -187,15 +188,6 @@ void HDF5Recording::closeFiles()
 	channelLeftOverSamples.clear();
 }
 
-void HDF5Recording::startChannelBlock()
-{
-	int nCh = channelTimestampArray.size();
-	for (int i = 0; i < nCh; ++i)
-	{
-		channelTimestampArray[i]->clearQuick();
-	}
-}
-
 void HDF5Recording::writeData(int writeChannel, int realChannel, const float* buffer, int size)
 {
 	if (size > bufferSize) //Shouldn't happen, and if it happens it'll be slow, but better this than crashing. Will be reset on reset.
@@ -232,17 +224,18 @@ void HDF5Recording::writeData(int writeChannel, int realChannel, const float* bu
 	channelLeftOverSamples.set(writeChannel, (size + sampleOffset) % TIMESTAMP_EACH_NSAMPLES);
 }
 
-void HDF5Recording::endChannelBlock()
+void HDF5Recording::endChannelBlock(bool lastBlock)
 {
 	int nCh = channelTimestampArray.size();
 	for (int ch = 0; ch < nCh; ++ch)
 	{
 		int tsSize = channelTimestampArray[ch]->size();
-		if (tsSize > 0)
+		if ((tsSize > 0) && ((tsSize > CHANNEL_TIMESTAMP_MIN_WRITE) || lastBlock))
 		{
 			int realChan = getRealChannel(ch);
 			int index = processorMap[getChannel(realChan)->recordIndex];
 			fileArray[index]->writeTimestamps(channelTimestampArray[ch]->getRawDataPointer(), tsSize, ch);
+			channelTimestampArray[ch]->clearQuick();
 		}
 	}
 }

Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.h

@@ -43,8 +43,7 @@ class HDF5Recording : public RecordEngine
 	void registerProcessor(const GenericProcessor* processor) override;
 	void resetChannels() override;
 	void startAcquisition() override;
-	void startChannelBlock() override;
-	void endChannelBlock() override;
+	void endChannelBlock(bool lastBlock) override;
 
     static RecordEngineManager* getEngineManager();
 private:

Source/Processors/RecordNode/RecordEngine.cpp

@@ -44,9 +44,9 @@ void RecordEngine::registerProcessor(const GenericProcessor* processor) {}
 
 void RecordEngine::addChannel(int index, const Channel* chan) {}
 
-void RecordEngine::startChannelBlock() {}
+void RecordEngine::startChannelBlock(bool lastBlock) {}
 
-void RecordEngine::endChannelBlock() {}
+void RecordEngine::endChannelBlock(bool lastBlock) {}
 
 Channel* RecordEngine::getChannel(int index) const
 {

Source/Processors/RecordNode/RecordEngine.h

@@ -101,7 +101,7 @@ class PLUGIN_API RecordEngine
 
 	/** Called by the record thread before it starts writing the channels to disk
 	*/
-	virtual void startChannelBlock();
+	virtual void startChannelBlock(bool lastBlock);
 
     /** Write continuous data for a channel. The raw buffer pointer is passed for speed, 
 		care must be taken to only read the specified number of bytes.
@@ -110,7 +110,7 @@ class PLUGIN_API RecordEngine
 
 	/** Called by the record thread after it has written a channel block
 	*/
-	virtual void endChannelBlock();
+	virtual void endChannelBlock(bool lastBlock);
 
     /** Write a single event to disk.
     */

Source/Processors/RecordNode/RecordThread.cpp

@@ -110,13 +110,13 @@ void RecordThread::run()
 	m_receivedFirstBlock = false;
 }
 
-void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples, int maxEvents, int maxSpikes)
+void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples, int maxEvents, int maxSpikes, bool lastBlock)
 {
 	Array<int64> timestamps;
 	Array<CircularBufferIndexes> idx;
 	m_dataQueue->startRead(idx, timestamps, maxSamples);
 	EVERY_ENGINE->updateTimestamps(timestamps);
-	EVERY_ENGINE->startChannelBlock();
+	EVERY_ENGINE->startChannelBlock(lastBlock);
 	for (int chan = 0; chan < m_numChannels; ++chan)
 	{
 		if (idx[chan].size1 > 0)
@@ -131,7 +131,7 @@ void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples
 		}
 	}
 	m_dataQueue->stopRead();
-	EVERY_ENGINE->endChannelBlock();
+	EVERY_ENGINE->endChannelBlock(lastBlock);
 
 	std::vector<EventMessagePtr> events;
 	int nEvents = m_eventQueue->getEvents(events, maxEvents);

Source/Processors/RecordNode/RecordThread.h

@@ -52,7 +52,7 @@ class RecordThread : public Thread
 	void forceCloseFiles();
 
 private:
-	void writeData(const AudioSampleBuffer& buffer, int maxSamples, int maxEvents, int maxSpikes);
+	void writeData(const AudioSampleBuffer& buffer, int maxSamples, int maxEvents, int maxSpikes, bool lastBlock = false);
 
 	const OwnedArray<RecordEngine>& m_engineArray;
 	Array<int> m_channelArray;