Skip to content

Commit 5435220

Browse files
aacuevasaacuevas
committed
Clean up folder structure in binary format
1 parent 4552bd5 commit 5435220

4 files changed

Lines changed: 158 additions & 173 deletions

File tree

‎Source/Plugins/BinaryWriter/BinaryRecording.cpp‎

Lines changed: 74 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,21 @@ String BinaryRecording::getEngineID() const
4444
return "RAWBINARY";
4545
}
4646

47+
String BinaryRecording::getProcessorString(const InfoObjectCommon* channelInfo)
48+
{
49+
String fName = (channelInfo->getCurrentNodeName().replaceCharacter(' ', '_') + "-" + String(channelInfo->getCurrentNodeID()));
50+
if (channelInfo->getCurrentNodeID() == channelInfo->getSourceNodeID()) //it is the channel source
51+
{
52+
fName += "." + String(channelInfo->getSubProcessorIdx());
53+
}
54+
else
55+
{
56+
fName += "_" + String(channelInfo->getSourceNodeID()) + "." + String(channelInfo->getSubProcessorIdx());
57+
}
58+
fName += File::separatorString;
59+
return fName;
60+
}
61+
4762
void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recordingNumber)
4863
{
4964
String basepath = rootFolder.getFullPathName() + rootFolder.separatorString + "experiment" + String(experimentNumber)
@@ -76,7 +91,7 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
7691
int nInfoArrays = indexedDataChannels.size();
7792
bool found = false;
7893
DynamicObject::Ptr jsonChan = new DynamicObject();
79-
jsonChan->setProperty("name", channelInfo->getName());
94+
jsonChan->setProperty("channel_name", channelInfo->getName());
8095
jsonChan->setProperty("description", channelInfo->getDescription());
8196
jsonChan->setProperty("identifier", channelInfo->getIdentifier());
8297
jsonChan->setProperty("history", channelInfo->getHistoricString());
@@ -100,13 +115,13 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
100115
}
101116
if (!found)
102117
{
103-
String datFileName(channelInfo->getCurrentNodeName() + "_(" + String(channelInfo->getCurrentNodeID()) + ")" + File::separatorString + channelInfo->getSourceName() + "_(" + String(sourceId) + "." + String(sourceSubIdx) + ")");
104-
continuousFileNames.add(contPath + datFileName + ".dat");
118+
String datPath = getProcessorString(channelInfo);
119+
continuousFileNames.add(contPath + datPath + "continuous.dat");
105120

106121
Array<NpyType> tstypes;
107122
tstypes.add(NpyType("Timestamp", BaseType::INT64, 1));
108123

109-
ScopedPointer<NpyFile> tFile = new NpyFile(contPath + datFileName + "_timestamps.npy", tstypes);
124+
ScopedPointer<NpyFile> tFile = new NpyFile(contPath + datPath + "timestamps.npy", tstypes);
110125
m_dataTimestampFiles.add(tFile.release());
111126

112127
m_fileIndexes.set(recordedChan, nInfoArrays);
@@ -118,7 +133,7 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
118133
jsonChanArray.add(var(jsonChan));
119134
jsonChannels.add(var(jsonChanArray));
120135
DynamicObject::Ptr jsonFile = new DynamicObject();
121-
jsonFile->setProperty("name", datFileName);
136+
jsonFile->setProperty("folder_name", datPath.replace(File::separatorString, "/")); //to make it more system agnostic, replace separator with only one slash
122137
jsonFile->setProperty("sample_rate", channelInfo->getSampleRate());
123138
jsonFile->setProperty("source_processor_name", channelInfo->getSourceName());
124139
jsonFile->setProperty("source_processor_id", channelInfo->getSourceNodeID());
@@ -154,81 +169,56 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
154169

155170
int nEvents = getNumRecordedEvents();
156171
String eventPath(basepath + "events" + File::separatorString);
157-
int binCount = 0, ttlCount = 0, textCount = 0;
158172
Array<var> jsonEventFiles;
159173

160174
for (int ev = 0; ev < nEvents; ev++)
161175
{
162176
const EventChannel* chan = getEventChannel(ev);
163-
String eventName;
164-
Array<NpyType> types;
165-
String typeName;
177+
String eventName = getProcessorString(chan);
178+
NpyType type;
179+
String dataFileName;
166180

167181
switch (chan->getChannelType())
168182
{
169183
case EventChannel::TEXT:
170-
textCount++;
171-
eventName += "TEXT" + String(textCount);
172-
types.add(NpyType("message", BaseType::CHAR, chan->getLength()));
173-
typeName = "text_message";
184+
eventName += "TEXT_group";
185+
type = NpyType(BaseType::CHAR, chan->getLength());
186+
dataFileName = "text";
174187
break;
175188
case EventChannel::TTL:
176-
ttlCount++;
177-
eventName += "TTL" + String(ttlCount);
178-
types.add(NpyType("TTL_Channel", BaseType::INT16, 1));
179-
typeName = "ttl";
189+
eventName += "TTL";
190+
type = NpyType(BaseType::INT16, 1);
191+
dataFileName = "channel_states";
180192
break;
181193
default:
182-
binCount++;
183-
eventName += "BIN" + String(ttlCount);
184-
types.add(NpyType("Data", chan->getEquivalentMetaDataType(), chan->getLength()));
185-
typeName = jsonTypeValue(chan->getEquivalentMetaDataType());
194+
eventName += "BINARY_group";
195+
type = NpyType(chan->getEquivalentMetaDataType(), chan->getLength());
196+
dataFileName = "data_array";
186197
break;
187198
}
188-
eventName += "_" + chan->getSourceName() + "(" + String(chan->getSourceNodeID()) + "." + String(chan->getSubProcessorIdx()) + ")";
189-
String fName = eventPath + eventName;
199+
eventName += "_" + String(chan->getSourceIndex() + 1) + File::separatorString;
190200
ScopedPointer<EventRecording> rec = new EventRecording();
191-
Array<NpyType> tsType;
192-
tsType.add(NpyType("Timestamp", BaseType::INT64, 1));
193-
//TTL channels behave a bit different
194-
if (chan->getChannelType() == EventChannel::TTL)
195-
{
196-
if (m_TTLMode == TTLMode::JointWord)
197-
{
198-
types.add(NpyType("TTL_Word", BaseType::UINT8, chan->getDataSize()));
199-
}
200-
else if (m_TTLMode == TTLMode::SeparateWord)
201-
{
202-
Array<NpyType> wordType;
203-
wordType.add(NpyType("TTL_Word", BaseType::UINT8, chan->getDataSize()));
204-
rec->extraFile = new NpyFile(fName + "_TTLWord.npy", wordType);
205-
}
206-
//since the main TTL file already contins channel numbers, it would be redundant to store them on the timestamp file
207-
}
208-
else
201+
202+
rec->mainFile = new NpyFile(eventPath + eventName + dataFileName + ".npy", type);
203+
rec->timestampFile = new NpyFile(eventPath + eventName + "timestamps.npy", NpyType(BaseType::INT64, 1));
204+
rec->channelFile = new NpyFile(eventPath + eventName + "channels.npy", NpyType(BaseType::UINT16, 1));
205+
if (chan->getChannelType() == EventChannel::TTL && m_saveTTLWords)
209206
{
210-
if (m_eventMode == EventMode::SeparateChannel)
211-
{
212-
Array<NpyType> chanType;
213-
chanType.add(NpyType("Channel", BaseType::UINT16, 1));
214-
rec->channelFile = new NpyFile(fName + "_channel.npy", chanType);
215-
}
216-
else
217-
tsType.add(NpyType("Channel", BaseType::UINT16, 1));
207+
rec->extraFile = new NpyFile(eventPath + eventName + "full_words.npy", NpyType(BaseType::UINT8, chan->getDataSize()));
218208
}
219-
rec->mainFile = new NpyFile(fName + ".npy", types);
220-
rec->timestampFile = new NpyFile(fName + "_timestamps.npy", tsType);
209+
221210
DynamicObject::Ptr jsonChannel = new DynamicObject();
222-
jsonChannel->setProperty("name", chan->getName());
211+
jsonChannel->setProperty("folder_name", eventName.replace(File::separatorString, "/"));
212+
jsonChannel->setProperty("channel_name", chan->getName());
223213
jsonChannel->setProperty("description", chan->getDescription());
224214
jsonChannel->setProperty("identifier", chan->getIdentifier());
225215
jsonChannel->setProperty("sample_rate", chan->getSampleRate());
226-
jsonChannel->setProperty("type", typeName);
216+
jsonChannel->setProperty("type", jsonTypeValue(type.getType()));
227217
jsonChannel->setProperty("num_channels", (int)chan->getNumChannels());
228218
jsonChannel->setProperty("source_processor", chan->getSourceName());
229219
createChannelMetaData(chan, jsonChannel);
230220

231-
rec->metaDataFile = createEventMetadataFile(chan, fName + "_metadata.npy", jsonChannel);
221+
rec->metaDataFile = createEventMetadataFile(chan, eventPath + eventName + "metadata.npy", jsonChannel);
232222
m_eventFiles.add(rec.release());
233223
jsonEventFiles.add(var(jsonChannel));
234224
}
@@ -241,12 +231,13 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
241231
String spikePath(basepath + "spikes" + File::separatorString);
242232
Array<var> jsonSpikeFiles;
243233
Array<var> jsonSpikeChannels;
234+
std::map<uint32, int> groupMap;
244235
for (int sp = 0; sp < nSpikes; sp++)
245236
{
246237
const SpikeChannel* ch = getSpikeChannel(sp);
247238
DynamicObject::Ptr jsonChannel = new DynamicObject();
248239
unsigned int numSpikeChannels = ch->getNumChannels();
249-
jsonChannel->setProperty("name", ch->getName());
240+
jsonChannel->setProperty("channel_name", ch->getName());
250241
jsonChannel->setProperty("description", ch->getDescription());
251242
jsonChannel->setProperty("identifier", ch->getIdentifier());
252243
Array<var> jsonChannelInfo;
@@ -288,51 +279,31 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
288279
m_spikeChannelIndexes.set(sp, 0);
289280
indexedChannels.add(1);
290281
ScopedPointer<EventRecording> rec = new EventRecording();
291-
Array<NpyType> spTypes;
292-
for (int c = 0; c < ch->getNumChannels(); c++)
293-
{
294-
spTypes.add(NpyType("channel" + String(c + 1), BaseType::INT16, ch->getTotalSamples()));
295-
}
296-
String spikeName("spike_group_" + String(fileIndex + 1));
297-
String fName(spikePath + spikeName);
298-
rec->mainFile = new NpyFile(fName + ".npy", spTypes);
282+
283+
uint32 procID = GenericProcessor::getProcessorFullId(ch->getSourceNodeID(), ch->getSubProcessorIdx());
284+
int groupIndex = ++groupMap[procID];
285+
286+
String spikeName = getProcessorString(ch) + "spike_group_" + String(groupIndex) + File::separatorString;
287+
288+
rec->mainFile = new NpyFile(spikePath + spikeName + "spike_waveforms.npy", NpyType(BaseType::INT16, ch->getTotalSamples()), ch->getNumChannels());
289+
rec->timestampFile = new NpyFile(spikePath + spikeName + "spike_times.npy", NpyType(BaseType::INT64, 1));
290+
rec->channelFile = new NpyFile(spikePath + spikeName + "spike_electrode_indices.npy", NpyType(BaseType::UINT16, 1));
291+
rec->extraFile = new NpyFile(spikePath + spikeName + "spike_clusters.npy", NpyType(BaseType::UINT16, 1));
299292
Array<NpyType> tsTypes;
300-
tsTypes.add(NpyType("timestamp", BaseType::INT64, 1));
301-
if (m_spikeMode == SpikeMode::AllInOne)
302-
{
303-
tsTypes.add(NpyType("electrode_index", BaseType::UINT16, 1));
304-
tsTypes.add(NpyType("sorted_id", BaseType::UINT16, 1));
305-
}
306-
else
307-
{
308-
Array<NpyType> indexType;
309-
indexType.add(NpyType("electrode_index", BaseType::UINT16, 1));
310-
if (m_spikeMode == SpikeMode::AllSeparated)
311-
{
312-
Array<NpyType> sortedType;
313-
sortedType.add(NpyType("sorted_id", BaseType::UINT16, 1));
314-
rec->extraFile = new NpyFile(fName + "_sortedID.npy", sortedType);
315-
}
316-
else
317-
{
318-
indexType.add(NpyType("sorted_id", BaseType::UINT16, 1));
319-
}
320-
rec->channelFile = new NpyFile(fName + "indexes.npy", indexType);
321-
}
322-
rec->timestampFile = new NpyFile(fName + "_timestamps.npy", tsTypes);
293+
323294
Array<var> jsonChanArray;
324295
jsonChanArray.add(var(jsonChannel));
325296
jsonSpikeChannels.add(var(jsonChanArray));
326297
DynamicObject::Ptr jsonFile = new DynamicObject();
327298

328-
jsonFile->setProperty("name", spikeName);
299+
jsonFile->setProperty("folder_name", spikeName.replace(File::separatorString,"/"));
329300
jsonFile->setProperty("sample_rate", ch->getSampleRate());
330301
jsonFile->setProperty("source_processor", ch->getSourceName());
331302
jsonFile->setProperty("num_channels", (int)numSpikeChannels);
332303
jsonFile->setProperty("pre_peak_samples", (int)ch->getPrePeakSamples());
333304
jsonFile->setProperty("post_peak_samples", (int)ch->getPostPeakSamples());
334305

335-
rec->metaDataFile = createEventMetadataFile(ch, fName + "_metadata.npy", jsonFile);
306+
rec->metaDataFile = createEventMetadataFile(ch, spikePath + spikeName + "metadata.npy", jsonFile);
336307
m_spikeFiles.add(rec.release());
337308
jsonSpikeFiles.add(var(jsonFile));
338309
}
@@ -552,38 +523,23 @@ void BinaryRecording::writeEvent(int eventIndex, const MidiMessage& event)
552523
const EventChannel* info = getEventChannel(eventIndex);
553524
int64 ts = ev->getTimestamp();
554525
rec->timestampFile->writeData(&ts, sizeof(int64));
526+
527+
uint16 chan = ev->getChannel();
528+
rec->channelFile->writeData(&chan, sizeof(uint16));
529+
555530
if (ev->getEventType() == EventChannel::TTL)
556531
{
557532
TTLEvent* ttl = static_cast<TTLEvent*>(ev.get());
558533
int16 data = ttl->getChannel() * (ttl->getState() ? 1 : -1);
559534
rec->mainFile->writeData(&data, sizeof(int16));
560-
NpyFile* wordFile = nullptr;
561-
if (m_TTLMode == TTLMode::JointWord)
562-
{
563-
wordFile = rec->mainFile;
564-
}
565-
else if (m_TTLMode == TTLMode::SeparateWord)
566-
{
567-
wordFile = rec->extraFile;
568-
}
569-
if (wordFile)
570-
wordFile->writeData(ttl->getTTLWordPointer(), info->getDataSize());
535+
if (rec->extraFile)
536+
rec->extraFile->writeData(ttl->getTTLWordPointer(), info->getDataSize());
571537
}
572538
else
573539
{
574540
rec->mainFile->writeData(ev->getRawDataPointer(), info->getDataSize());
575-
NpyFile* chanFile = nullptr;
576-
if (m_eventMode == EventMode::SeparateChannel)
577-
{
578-
chanFile = rec->channelFile;
579-
}
580-
else
581-
{
582-
chanFile = rec->timestampFile;
583-
}
584-
uint16 chan = ev->getChannel();
585-
chanFile->writeData(&chan, sizeof(uint16));
586541
}
542+
587543
writeEventMetaData(ev.get(), rec->metaDataFile);
588544
increaseEventCounts(rec);
589545
}
@@ -617,29 +573,15 @@ void BinaryRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike)
617573
FloatVectorOperations::copyWithMultiply(m_scaledBuffer.getData(), spike->getDataPointer(), multFactor, totalSamples);
618574
AudioDataConverters::convertFloatToInt16LE(m_scaledBuffer.getData(), m_intBuffer.getData(), totalSamples);
619575
rec->mainFile->writeData(m_intBuffer.getData(), totalSamples*sizeof(int16));
576+
620577
int64 ts = spike->getTimestamp();
621578
rec->timestampFile->writeData(&ts, sizeof(int64));
622-
NpyFile* indexFile;
623-
NpyFile* sortedFile;
624-
if (m_spikeMode == SpikeMode::AllInOne)
625-
{
626-
indexFile = rec->timestampFile;
627-
sortedFile = rec->timestampFile;
628-
}
629-
else if (m_spikeMode == SpikeMode::SeparateTimestamps)
630-
{
631-
indexFile = rec->channelFile;
632-
sortedFile = rec->channelFile;
633-
}
634-
else
635-
{
636-
indexFile = rec->channelFile;
637-
sortedFile = rec->extraFile;
638-
}
639-
indexFile->writeData(&spikeChannel, sizeof(uint16));
579+
580+
rec->channelFile->writeData(&spikeChannel, sizeof(uint16));
640581

641582
uint16 sortedID = spike->getSortedID();
642-
sortedFile->writeData(&sortedID, sizeof(uint16));
583+
rec->extraFile->writeData(&sortedID, sizeof(uint16));
584+
643585
increaseEventCounts(rec);
644586
}
645587

@@ -656,20 +598,15 @@ RecordEngineManager* BinaryRecording::getEngineManager()
656598
{
657599
RecordEngineManager* man = new RecordEngineManager("RAWBINARY", "Binary", &(engineFactory<BinaryRecording>));
658600
EngineParameter* param;
659-
param = new EngineParameter(EngineParameter::MULTI, 0, "Spike TS/chan/sortedID File Mode|All in one|Separate timestamps|All Separated", 0);
660-
man->addParameter(param);
661-
param = new EngineParameter(EngineParameter::MULTI, 1, "TTL Event word file|In main file|Separated|Do not save ttl word", 0);
662-
man->addParameter(param);
663-
param = new EngineParameter(EngineParameter::MULTI, 2, "Other event channel file|With timestamp|Separate", 0);
601+
param = new EngineParameter(EngineParameter::BOOL, 0, "Record TTL full words", true);
664602
man->addParameter(param);
603+
665604
return man;
666605
}
667606

668607
void BinaryRecording::setParameter(EngineParameter& parameter)
669608
{
670-
multiParameter(0, reinterpret_cast<int&>(m_spikeMode));
671-
multiParameter(1, reinterpret_cast<int&>(m_TTLMode));
672-
multiParameter(2, reinterpret_cast<int&>(m_eventMode));
609+
boolParameter(0, m_saveTTLWords);
673610
}
674611

675612
String BinaryRecording::jsonTypeValue(BaseType type)

‎Source/Plugins/BinaryWriter/BinaryRecording.h‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -61,37 +61,16 @@ namespace BinaryRecordingEngine
6161
ScopedPointer<NpyFile> extraFile;
6262
};
6363

64-
enum SpikeMode
65-
{
66-
AllInOne = 0,
67-
SeparateTimestamps = 1,
68-
AllSeparated = 2
69-
};
70-
71-
enum TTLMode
72-
{
73-
JointWord = 0,
74-
SeparateWord = 1,
75-
NoWord = 2
76-
};
77-
78-
enum EventMode
79-
{
80-
JointChannel = 0,
81-
SeparateChannel = 1
82-
};
8364

8465
NpyFile* createEventMetadataFile(const MetaDataEventObject* channel, String fileName, DynamicObject* jsonObject);
8566
void createChannelMetaData(const MetaDataInfoObject* channel, DynamicObject* jsonObject);
8667
void writeEventMetaData(const MetaDataEvent* event, NpyFile* file);
8768
void increaseEventCounts(EventRecording* rec);
8869
static String jsonTypeValue(BaseType type);
70+
static String getProcessorString(const InfoObjectCommon* channelInfo);
8971

90-
SpikeMode m_spikeMode;
91-
TTLMode m_TTLMode;
92-
EventMode m_eventMode;
93-
94-
72+
bool m_saveTTLWords{ true };
73+
9574
HeapBlock<float> m_scaledBuffer;
9675
HeapBlock<int16> m_intBuffer;
9776
HeapBlock<int64> m_tsBuffer;

0 commit comments

Comments
 (0)