Set HDF5 chuck cache size dependent on the number of recorded channels

Author: aacuevas

Builds/VisualStudio2013/open-ephys.vcxproj.filters

@@ -64,9 +64,6 @@
     <Filter Include="open-ephys\Source\Processors\EventDetector">
       <UniqueIdentifier>{FDD5B79A-1AA4-7745-BB6E-E09A6DFC328E}</UniqueIdentifier>
     </Filter>
-    <Filter Include="open-ephys\Source\Processors\EventNode">
-      <UniqueIdentifier>{3BDCC6CD-7B1F-ED1C-7442-F2B0CD1A86C8}</UniqueIdentifier>
-    </Filter>
     <Filter Include="open-ephys\Source\Processors\FileReader">
       <UniqueIdentifier>{59F490B2-F5D9-601B-01AE-368632E3C9D7}</UniqueIdentifier>
     </Filter>
@@ -439,6 +436,9 @@
     <Filter Include="Juce Library Code">
       <UniqueIdentifier>{8B4D1BAA-6DB4-CAEC-A0FA-271F354D5C61}</UniqueIdentifier>
     </Filter>
+    <Filter Include="open-ephys\Source\Processors\EventDetector\EventNode">
+      <UniqueIdentifier>{3BDCC6CD-7B1F-ED1C-7442-F2B0CD1A86C8}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\Source\CoreServices.cpp">
@@ -583,10 +583,10 @@
       <Filter>open-ephys\Source\Processors\EventDetector</Filter>
     </ClCompile>
     <ClCompile Include="..\..\Source\Processors\EventNode\EventNode.cpp">
-      <Filter>open-ephys\Source\Processors\EventNode</Filter>
+      <Filter>open-ephys\Source\Processors\EventDetector\EventNode</Filter>
     </ClCompile>
     <ClCompile Include="..\..\Source\Processors\EventNode\EventNodeEditor.cpp">
-      <Filter>open-ephys\Source\Processors\EventNode</Filter>
+      <Filter>open-ephys\Source\Processors\EventDetector\EventNode</Filter>
     </ClCompile>
     <ClCompile Include="..\..\Source\Processors\FileReader\KwikFileSource.cpp">
       <Filter>open-ephys\Source\Processors\FileReader</Filter>
@@ -2124,10 +2124,10 @@
       <Filter>open-ephys\Source\Processors\EventDetector</Filter>
     </ClInclude>
     <ClInclude Include="..\..\Source\Processors\EventNode\EventNode.h">
-      <Filter>open-ephys\Source\Processors\EventNode</Filter>
+      <Filter>open-ephys\Source\Processors\EventDetector\EventNode</Filter>
     </ClInclude>
     <ClInclude Include="..\..\Source\Processors\EventNode\EventNodeEditor.h">
-      <Filter>open-ephys\Source\Processors\EventNode</Filter>
+      <Filter>open-ephys\Source\Processors\EventDetector\EventNode</Filter>
     </ClInclude>
     <ClInclude Include="..\..\Source\Processors\FileReader\KwikFileSource.h">
       <Filter>open-ephys\Source\Processors\FileReader</Filter>

Source/Processors/RecordNode/HDF5FileFormat.cpp

@@ -66,17 +66,27 @@ bool HDF5FileBase::isOpen() const
     return opened;
 }
 
+bool HDF5FileBase::isReadyToOpen() const
+{
+	return readyToOpen;
+}
+
 int HDF5FileBase::open()
+{
+	return open(-1);
+}
+
+int HDF5FileBase::open(int nChans)
 {
     if (!readyToOpen) return -1;
     if (File(getFileName()).existsAsFile())
-        return open(false);
+        return open(false, nChans);
     else
-        return open(true);
+        return open(true, nChans);
 
 }
 
-int HDF5FileBase::open(bool newfile)
+int HDF5FileBase::open(bool newfile, int nChans)
 {
     int accFlags,ret=0;
 
@@ -85,7 +95,11 @@ int HDF5FileBase::open(bool newfile)
     try
     {
 		FileAccPropList props = FileAccPropList::DEFAULT;
-		props.setCache(0, 401, 4 * 2 * 35 * 16 * 320, 1);
+		if (nChans > 0)
+		{
+			props.setCache(0, 809, 8 * 2 * 640 * nChans, 1);
+			std::cout << "opening HDF5 " << getFileName() << " with nchans: " << nChans << std::endl;
+		}
 
         if (newfile) accFlags = H5F_ACC_TRUNC;
         else accFlags = H5F_ACC_RDWR;

Source/Processors/RecordNode/HDF5FileFormat.h

@@ -53,9 +53,11 @@ class HDF5FileBase
     virtual ~HDF5FileBase();
 
     int open();
+	int open(int nChans);
     void close();
     virtual String getFileName() = 0;
     bool isOpen() const;
+	bool isReadyToOpen() const;
     typedef enum DataTypes { U8, U16, U32, U64, I8, I16, I32, I64, F32, STR} DataTypes;
 
     static H5::DataType getNativeType(DataTypes type);
@@ -83,7 +85,7 @@ class HDF5FileBase
 private:
     //create an extendable dataset
     HDF5RecordingData* createDataSet(DataTypes type, int dimension, int* size, int* chunking, String path);
-    int open(bool newfile);
+    int open(bool newfile, int nChans);
     ScopedPointer<H5::H5File> file;
     bool opened;
 

Source/Processors/RecordNode/HDF5Recording.cpp

@@ -57,13 +57,15 @@ void HDF5Recording::registerProcessor(GenericProcessor* proc)
     fileArray.add(new KWDFile());
     bitVoltsArray.add(new Array<float>);
     sampleRatesArray.add(new Array<float>);
+	channelsPerProcessor.add(0);
     processorIndex++;
 }
 
 void HDF5Recording::resetChannels()
 {
     processorIndex = -1;
     fileArray.clear();
+	channelsPerProcessor.clear();
     bitVoltsArray.clear();
     sampleRatesArray.clear();
     processorMap.clear();
@@ -106,15 +108,15 @@ void HDF5Recording::openFiles(File rootFolder, int experimentNumber, int recordi
         int index = processorMap[i];
         if (getChannel(i)->getRecordState())
         {
-            if (!fileArray[index]->isOpen())
+			if (!fileArray[index]->isOpen())
             {
                 fileArray[index]->initFile(getChannel(i)->nodeId,basepath);
-                fileArray[index]->open();
                 if (hasAcquired)
                     infoArray[index]->start_time = (*timestamps)[getChannel(i)->sourceNodeId]; //the timestamps of the first channel
                 else
                     infoArray[index]->start_time = 0;
             }
+			channelsPerProcessor.set(index, channelsPerProcessor[index] + 1);
             bitVoltsArray[index]->add(getChannel(i)->bitVolts);
             sampleRatesArray[index]->add(getChannel(i)->sampleRate);
             if (getChannel(i)->sampleRate != infoArray[index]->sample_rate)
@@ -125,6 +127,10 @@ void HDF5Recording::openFiles(File rootFolder, int experimentNumber, int recordi
     }
     for (int i = 0; i < fileArray.size(); i++)
     {
+		if ((!fileArray[i]->isOpen()) && (fileArray[i]->isReadyToOpen()))
+		{
+			fileArray[i]->open(channelsPerProcessor[i]);
+		}
         if (fileArray[i]->isOpen())
         {
             File f(fileArray[i]->getFileName());
@@ -158,6 +164,7 @@ void HDF5Recording::closeFiles()
             fileArray[i]->close();
             bitVoltsArray[i]->clear();
         }
+		channelsPerProcessor.set(i, 0);
     }
 }
 

Source/Processors/RecordNode/HDF5Recording.h

@@ -51,6 +51,7 @@ class HDF5Recording : public RecordEngine
     int processorIndex;
 
     Array<int> processorMap;
+	Array<int> channelsPerProcessor;
     OwnedArray<Array<float>> bitVoltsArray;
     OwnedArray<Array<float>> sampleRatesArray;
     OwnedArray<KWDFile> fileArray;