Save timestamps in kwd files (WIP)

Author: aacuevas

Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.cpp

@@ -40,6 +40,10 @@
 #define SPIKE_CHUNK_YSIZE 40
 #endif
 
+#ifndef TIMESTAMP_CHUNK_SIZE
+#define TIMESTAMP_CHUNK_SIZE 4
+#endif
+
 #define MAX_TRANSFORM_SIZE 512
 
 #define MAX_STR_SIZE 256
@@ -675,13 +679,21 @@ void KWDFile::startNewRecording(int recordingNumber, int nChannels, HDF5Recordin
     CHECK_ERROR(createGroup(recordPath+"/application_data"));
    // CHECK_ERROR(setAttributeArray(F32,info->bitVolts.getRawDataPointer(),info->bitVolts.size(),recordPath+"/application_data",String("channel_bit_volts")));
 	bitVoltsSet = createDataSet(F32, info->bitVolts.size(), 0, recordPath + "/application_data/channel_bit_volts");
-	bitVoltsSet->writeDataBlock(info->bitVolts.size(), F32, info->bitVolts.getRawDataPointer());
+	if (bitVoltsSet.get())
+		bitVoltsSet->writeDataBlock(info->bitVolts.size(), F32, info->bitVolts.getRawDataPointer());
+	else
+		std::cerr << "Error creating bitvolts data set" << std::endl;
 
     CHECK_ERROR(setAttribute(U8,&mSample,recordPath+"/application_data",String("is_multiSampleRate_data")));
     CHECK_ERROR(setAttributeArray(F32,info->channelSampleRates.getRawDataPointer(),info->channelSampleRates.size(),recordPath+"/application_data",String("channel_sample_rates")));
     recdata = createDataSet(I16,0,nChannels,CHUNK_XSIZE,recordPath+"/data");
     if (!recdata.get())
         std::cerr << "Error creating data set" << std::endl;
+
+	tsData = createDataSet(I64, 0, nChannels, TIMESTAMP_CHUNK_SIZE, recordPath + "/application_data/timestamps");
+	if (!tsData.get())
+		std::cerr << "Error creating timestamps data set" << std::endl;
+
     curChan = nChannels;
 }
 
@@ -694,6 +706,7 @@ void KWDFile::stopRecording()
     CHECK_ERROR(setAttributeArray(U32,samples.getRawDataPointer(),samples.size(),path,"valid_samples"));
     //ScopedPointer does the deletion and destructors the closings
     recdata = nullptr;
+	tsData = nullptr;
 }
 
 int KWDFile::createFileStructure()
@@ -728,6 +741,14 @@ void KWDFile::writeRowData(int16* data, int nSamples, int channel)
 	}
 }
 
+void KWDFile::writeTimestamps(int64* ts, int nTs, int channel)
+{
+	if (channel >= 0 && channel < nChannels)
+	{
+		CHECK_ERROR(tsData->writeDataRow(channel, nTs, I64, ts));
+	}
+}
+
 //KWE File
 
 KWEFile::KWEFile(String basename) : HDF5FileBase()

Source/Plugins/KWIKFormat/RecordEngine/HDF5FileFormat.h

@@ -128,6 +128,7 @@ class KWDFile : public HDF5FileBase
     void writeBlockData(int16* data, int nSamples);
     void writeRowData(int16* data, int nSamples);
 	void writeRowData(int16* data, int nSamples, int channel);
+	void writeTimestamps(int64* ts, int nTs, int channel);
     String getFileName();
 
 protected:
@@ -140,6 +141,7 @@ class KWDFile : public HDF5FileBase
     String filename;
     bool multiSample;
     ScopedPointer<HDF5RecordingData> recdata;
+	ScopedPointer<HDF5RecordingData> tsData;
 
     JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(KWDFile);
 };

Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.cpp

@@ -23,6 +23,8 @@
 
 #include "HDF5Recording.h"
 #define MAX_BUFFER_SIZE 10000
+#define CHANNEL_TIMESTAMP_PREALLOC_SIZE 16
+#define TIMESTAMP_EACH_NSAMPLES 1024
 
 HDF5Recording::HDF5Recording() : processorIndex(-1), hasAcquired(false)
 {
@@ -37,7 +39,7 @@ HDF5Recording::~HDF5Recording()
     delete intBuffer;
 }
 
-String HDF5Recording::getEngineID()
+String HDF5Recording::getEngineID() const
 {
     return "KWIK";
 }
@@ -73,6 +75,8 @@ void HDF5Recording::resetChannels()
     processorMap.clear();
     infoArray.clear();
 	recordedChanToKWDChan.clear();
+	channelLeftOverSamples.clear();
+	channelTimestampArray.clear();
     if (spikesFile)
         spikesFile->resetChannels();
 }
@@ -129,31 +133,11 @@ void HDF5Recording::openFiles(File rootFolder, int experimentNumber, int recordi
 		int procPos = processorRecPos[index];
 		recordedChanToKWDChan.add(procPos);
 		processorRecPos.set(index, procPos+1);
+		channelTimestampArray.add(new Array<int64>);
+		channelTimestampArray.getLast()->ensureStorageAllocated(CHANNEL_TIMESTAMP_PREALLOC_SIZE);
+		channelLeftOverSamples.add(0);
 	} 
-#if 0
-    for (int i = 0; i < processorMap.size(); i++)
-    {
-        int index = processorMap[i];
-        if (getChannel(i)->getRecordState())
-        {
-			if (!fileArray[index]->isOpen())
-            {
-                fileArray[index]->initFile(getChannel(i)->nodeId,basepath);
-                if (hasAcquired)
-                    infoArray[index]->start_time = (*timestamps)[getChannel(i)->sourceNodeId]; //the timestamps of the first channel
-                else
-                    infoArray[index]->start_time = 0;
-            }
-			channelsPerProcessor.set(index, channelsPerProcessor[index] + 1);
-            bitVoltsArray[index]->add(getChannel(i)->bitVolts);
-            sampleRatesArray[index]->add(getChannel(i)->sampleRate);
-            if (getChannel(i)->sampleRate != infoArray[index]->sample_rate)
-            {
-                infoArray[index]->multiSample = true;
-            }
-        }
-    }
-#endif
+
     for (int i = 0; i < fileArray.size(); i++)
     {
 		if ((!fileArray[i]->isOpen()) && (fileArray[i]->isReadyToOpen()))
@@ -196,6 +180,18 @@ void HDF5Recording::closeFiles()
         }
 		channelsPerProcessor.set(i, 0);
     }
+	recordedChanToKWDChan.clear();
+	channelTimestampArray.clear();
+	channelLeftOverSamples.clear();
+}
+
+void HDF5Recording::startChannelBlock()
+{
+	int nCh = channelTimestampArray.size();
+	for (int i = 0; i < nCh; ++i)
+	{
+		channelTimestampArray[i]->clearQuick();
+	}
 }
 
 void HDF5Recording::writeData(int writeChannel, int realChannel, const float* buffer, int size)
@@ -208,6 +204,48 @@ void HDF5Recording::writeData(int writeChannel, int realChannel, const float* bu
 	fileArray[index]->writeRowData(intBuffer, size, recordedChanToKWDChan[writeChannel]);
 //	int64 t2 = Time::getHighResolutionTicks();
 //	std::cout << "record time: " << float(t2 - t1) / float(Time::getHighResolutionTicksPerSecond()) << std::endl;
+	int64 sampleOffset = channelLeftOverSamples[writeChannel];
+	if (writeChannel == 0)
+		std::cout << "Write " << size << " off " << sampleOffset << " ts " << getTimestamp(realChannel) << " - ";
+	if (sampleOffset + size >= TIMESTAMP_EACH_NSAMPLES)
+	{
+		int64 currentTimestamp = getTimestamp(realChannel);
+		if (sampleOffset > 0)
+		{
+			currentTimestamp = getTimestamp(realChannel) + TIMESTAMP_EACH_NSAMPLES - sampleOffset;
+		}
+
+		for (int samp = 0; samp < size; samp += TIMESTAMP_EACH_NSAMPLES)
+		{
+			if (writeChannel == 0)
+				std::cout << "w: " << currentTimestamp << " ";
+			channelTimestampArray[writeChannel]->add(currentTimestamp);
+			currentTimestamp += TIMESTAMP_EACH_NSAMPLES;
+		}
+
+		channelLeftOverSamples.set(writeChannel, (size + sampleOffset) % TIMESTAMP_EACH_NSAMPLES);
+	}
+	else
+	{
+		channelLeftOverSamples.set(writeChannel, sampleOffset + size);
+	}
+	if (writeChannel == 0)
+		std::cout << std::endl;
+}
+
+void HDF5Recording::endChannelBlock()
+{
+	int nCh = channelTimestampArray.size();
+	for (int ch = 0; ch < nCh; ++ch)
+	{
+		int tsSize = channelTimestampArray[ch]->size();
+		if (tsSize > 0)
+		{
+			int realChan = getRealChannel(ch);
+			int index = processorMap[getChannel(realChan)->recordIndex];
+			fileArray[index]->writeTimestamps(channelTimestampArray[ch]->getRawDataPointer(), tsSize, ch);
+		}
+	}
 }
 
 void HDF5Recording::writeEvent(int eventType, const MidiMessage& event, int64 timestamp)

Source/Plugins/KWIKFormat/RecordEngine/HDF5Recording.h

@@ -32,7 +32,7 @@ class HDF5Recording : public RecordEngine
 public:
     HDF5Recording();
     ~HDF5Recording();
-    String getEngineID();
+    String getEngineID() const override;
     void openFiles(File rootFolder, int experimentNumber, int recordingNumber) override;
 	void closeFiles() override;
 	void writeData(int writeChannel, int realChannel, const float* buffer, int size) override;
@@ -43,6 +43,8 @@ class HDF5Recording : public RecordEngine
 	void registerProcessor(const GenericProcessor* processor) override;
 	void resetChannels() override;
 	void startAcquisition() override;
+	void startChannelBlock() override;
+	void endChannelBlock() override;
 
     static RecordEngineManager* getEngineManager();
 private:
@@ -54,6 +56,8 @@ class HDF5Recording : public RecordEngine
 	Array<int> recordedChanToKWDChan;
     OwnedArray<Array<float>> bitVoltsArray;
     OwnedArray<Array<float>> sampleRatesArray;
+	OwnedArray<Array<int64>> channelTimestampArray;
+	Array<int> channelLeftOverSamples;
     OwnedArray<KWDFile> fileArray;
     OwnedArray<HDF5RecordingInfo> infoArray;
     ScopedPointer<KWEFile> eventFile;

Source/Processors/RecordNode/OriginalRecording.cpp

@@ -62,7 +62,7 @@ OriginalRecording::~OriginalRecording()
     delete recordMarker;*/
 }
 
-String OriginalRecording::getEngineID()
+String OriginalRecording::getEngineID() const
 {
     return "OPENEPHYS";
 }

Source/Processors/RecordNode/OriginalRecording.h

@@ -45,7 +45,7 @@ class OriginalRecording : public RecordEngine
     ~OriginalRecording();
 
     void setParameter(EngineParameter& parameter);
-    String getEngineID();
+    String getEngineID() const override;
     void openFiles(File rootFolder, int experimentNumber, int recordingNumber) override;
 	void closeFiles() override;
 	void writeData(int writeChannel, int realChannel, const float* buffer, int size) override;

Source/Processors/RecordNode/RecordEngine.h

@@ -59,7 +59,7 @@ class PLUGIN_API RecordEngine
 public:
     RecordEngine();
     virtual ~RecordEngine();
-    virtual String getEngineID() =0;
+    virtual String getEngineID() const =0;
 
     /** All the public methods (except registerManager) are called by RecordNode or RecordingThread:
     When acquisition starts (in the specified order):