Skip to content

Commit d90abe7

Browse files
kmichaelfoxkmichaelfox
committed
pull upstream and merge
2 parents a68d15c + 3b91905 commit d90abe7

19 files changed

Lines changed: 287 additions & 317 deletions

‎Source/Plugins/BinaryWriter/BinaryRecording.cpp‎

Lines changed: 74 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,21 @@ String BinaryRecording::getEngineID() const
4444
return "RAWBINARY";
4545
}
4646

47+
String BinaryRecording::getProcessorString(const InfoObjectCommon* channelInfo)
48+
{
49+
String fName = (channelInfo->getCurrentNodeName().replaceCharacter(' ', '_') + "-" + String(channelInfo->getCurrentNodeID()));
50+
if (channelInfo->getCurrentNodeID() == channelInfo->getSourceNodeID()) //it is the channel source
51+
{
52+
fName += "." + String(channelInfo->getSubProcessorIdx());
53+
}
54+
else
55+
{
56+
fName += "_" + String(channelInfo->getSourceNodeID()) + "." + String(channelInfo->getSubProcessorIdx());
57+
}
58+
fName += File::separatorString;
59+
return fName;
60+
}
61+
4762
void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recordingNumber)
4863
{
4964
String basepath = rootFolder.getFullPathName() + rootFolder.separatorString + "experiment" + String(experimentNumber)
@@ -76,7 +91,7 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
7691
int nInfoArrays = indexedDataChannels.size();
7792
bool found = false;
7893
DynamicObject::Ptr jsonChan = new DynamicObject();
79-
jsonChan->setProperty("name", channelInfo->getName());
94+
jsonChan->setProperty("channel_name", channelInfo->getName());
8095
jsonChan->setProperty("description", channelInfo->getDescription());
8196
jsonChan->setProperty("identifier", channelInfo->getIdentifier());
8297
jsonChan->setProperty("history", channelInfo->getHistoricString());
@@ -100,13 +115,10 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
100115
}
101116
if (!found)
102117
{
103-
String datFileName(channelInfo->getCurrentNodeName() + "_(" + String(channelInfo->getCurrentNodeID()) + ")" + File::separatorString + channelInfo->getSourceName() + "_(" + String(sourceId) + "." + String(sourceSubIdx) + ")");
104-
continuousFileNames.add(contPath + datFileName + ".dat");
118+
String datPath = getProcessorString(channelInfo);
119+
continuousFileNames.add(contPath + datPath + "continuous.dat");
105120

106-
Array<NpyType> tstypes;
107-
tstypes.add(NpyType("Timestamp", BaseType::INT64, 1));
108-
109-
ScopedPointer<NpyFile> tFile = new NpyFile(contPath + datFileName + "_timestamps.npy", tstypes);
121+
ScopedPointer<NpyFile> tFile = new NpyFile(contPath + datPath + "timestamps.npy", NpyType(BaseType::INT64,1));
110122
m_dataTimestampFiles.add(tFile.release());
111123

112124
m_fileIndexes.set(recordedChan, nInfoArrays);
@@ -118,7 +130,7 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
118130
jsonChanArray.add(var(jsonChan));
119131
jsonChannels.add(var(jsonChanArray));
120132
DynamicObject::Ptr jsonFile = new DynamicObject();
121-
jsonFile->setProperty("name", datFileName);
133+
jsonFile->setProperty("folder_name", datPath.replace(File::separatorString, "/")); //to make it more system agnostic, replace separator with only one slash
122134
jsonFile->setProperty("sample_rate", channelInfo->getSampleRate());
123135
jsonFile->setProperty("source_processor_name", channelInfo->getSourceName());
124136
jsonFile->setProperty("source_processor_id", channelInfo->getSourceNodeID());
@@ -154,81 +166,56 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
154166

155167
int nEvents = getNumRecordedEvents();
156168
String eventPath(basepath + "events" + File::separatorString);
157-
int binCount = 0, ttlCount = 0, textCount = 0;
158169
Array<var> jsonEventFiles;
159170

160171
for (int ev = 0; ev < nEvents; ev++)
161172
{
162173
const EventChannel* chan = getEventChannel(ev);
163-
String eventName;
164-
Array<NpyType> types;
165-
String typeName;
174+
String eventName = getProcessorString(chan);
175+
NpyType type;
176+
String dataFileName;
166177

167178
switch (chan->getChannelType())
168179
{
169180
case EventChannel::TEXT:
170-
textCount++;
171-
eventName += "TEXT" + String(textCount);
172-
types.add(NpyType("message", BaseType::CHAR, chan->getLength()));
173-
typeName = "text_message";
181+
eventName += "TEXT_group";
182+
type = NpyType(BaseType::CHAR, chan->getLength());
183+
dataFileName = "text";
174184
break;
175185
case EventChannel::TTL:
176-
ttlCount++;
177-
eventName += "TTL" + String(ttlCount);
178-
types.add(NpyType("TTL_Channel", BaseType::INT16, 1));
179-
typeName = "ttl";
186+
eventName += "TTL";
187+
type = NpyType(BaseType::INT16, 1);
188+
dataFileName = "channel_states";
180189
break;
181190
default:
182-
binCount++;
183-
eventName += "BIN" + String(ttlCount);
184-
types.add(NpyType("Data", chan->getEquivalentMetaDataType(), chan->getLength()));
185-
typeName = jsonTypeValue(chan->getEquivalentMetaDataType());
191+
eventName += "BINARY_group";
192+
type = NpyType(chan->getEquivalentMetaDataType(), chan->getLength());
193+
dataFileName = "data_array";
186194
break;
187195
}
188-
eventName += "_" + chan->getSourceName() + "(" + String(chan->getSourceNodeID()) + "." + String(chan->getSubProcessorIdx()) + ")";
189-
String fName = eventPath + eventName;
196+
eventName += "_" + String(chan->getSourceIndex() + 1) + File::separatorString;
190197
ScopedPointer<EventRecording> rec = new EventRecording();
191-
Array<NpyType> tsType;
192-
tsType.add(NpyType("Timestamp", BaseType::INT64, 1));
193-
//TTL channels behave a bit different
194-
if (chan->getChannelType() == EventChannel::TTL)
195-
{
196-
if (m_TTLMode == TTLMode::JointWord)
197-
{
198-
types.add(NpyType("TTL_Word", BaseType::UINT8, chan->getDataSize()));
199-
}
200-
else if (m_TTLMode == TTLMode::SeparateWord)
201-
{
202-
Array<NpyType> wordType;
203-
wordType.add(NpyType("TTL_Word", BaseType::UINT8, chan->getDataSize()));
204-
rec->extraFile = new NpyFile(fName + "_TTLWord.npy", wordType);
205-
}
206-
//since the main TTL file already contins channel numbers, it would be redundant to store them on the timestamp file
207-
}
208-
else
198+
199+
rec->mainFile = new NpyFile(eventPath + eventName + dataFileName + ".npy", type);
200+
rec->timestampFile = new NpyFile(eventPath + eventName + "timestamps.npy", NpyType(BaseType::INT64, 1));
201+
rec->channelFile = new NpyFile(eventPath + eventName + "channels.npy", NpyType(BaseType::UINT16, 1));
202+
if (chan->getChannelType() == EventChannel::TTL && m_saveTTLWords)
209203
{
210-
if (m_eventMode == EventMode::SeparateChannel)
211-
{
212-
Array<NpyType> chanType;
213-
chanType.add(NpyType("Channel", BaseType::UINT16, 1));
214-
rec->channelFile = new NpyFile(fName + "_channel.npy", chanType);
215-
}
216-
else
217-
tsType.add(NpyType("Channel", BaseType::UINT16, 1));
204+
rec->extraFile = new NpyFile(eventPath + eventName + "full_words.npy", NpyType(BaseType::UINT8, chan->getDataSize()));
218205
}
219-
rec->mainFile = new NpyFile(fName + ".npy", types);
220-
rec->timestampFile = new NpyFile(fName + "_timestamps.npy", tsType);
206+
221207
DynamicObject::Ptr jsonChannel = new DynamicObject();
222-
jsonChannel->setProperty("name", chan->getName());
208+
jsonChannel->setProperty("folder_name", eventName.replace(File::separatorString, "/"));
209+
jsonChannel->setProperty("channel_name", chan->getName());
223210
jsonChannel->setProperty("description", chan->getDescription());
224211
jsonChannel->setProperty("identifier", chan->getIdentifier());
225212
jsonChannel->setProperty("sample_rate", chan->getSampleRate());
226-
jsonChannel->setProperty("type", typeName);
213+
jsonChannel->setProperty("type", jsonTypeValue(type.getType()));
227214
jsonChannel->setProperty("num_channels", (int)chan->getNumChannels());
228215
jsonChannel->setProperty("source_processor", chan->getSourceName());
229216
createChannelMetaData(chan, jsonChannel);
230217

231-
rec->metaDataFile = createEventMetadataFile(chan, fName + "_metadata.npy", jsonChannel);
218+
rec->metaDataFile = createEventMetadataFile(chan, eventPath + eventName + "metadata.npy", jsonChannel);
232219
m_eventFiles.add(rec.release());
233220
jsonEventFiles.add(var(jsonChannel));
234221
}
@@ -241,12 +228,13 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
241228
String spikePath(basepath + "spikes" + File::separatorString);
242229
Array<var> jsonSpikeFiles;
243230
Array<var> jsonSpikeChannels;
231+
std::map<uint32, int> groupMap;
244232
for (int sp = 0; sp < nSpikes; sp++)
245233
{
246234
const SpikeChannel* ch = getSpikeChannel(sp);
247235
DynamicObject::Ptr jsonChannel = new DynamicObject();
248236
unsigned int numSpikeChannels = ch->getNumChannels();
249-
jsonChannel->setProperty("name", ch->getName());
237+
jsonChannel->setProperty("channel_name", ch->getName());
250238
jsonChannel->setProperty("description", ch->getDescription());
251239
jsonChannel->setProperty("identifier", ch->getIdentifier());
252240
Array<var> jsonChannelInfo;
@@ -288,51 +276,31 @@ void BinaryRecording::openFiles(File rootFolder, int experimentNumber, int recor
288276
m_spikeChannelIndexes.set(sp, 0);
289277
indexedChannels.add(1);
290278
ScopedPointer<EventRecording> rec = new EventRecording();
291-
Array<NpyType> spTypes;
292-
for (int c = 0; c < ch->getNumChannels(); c++)
293-
{
294-
spTypes.add(NpyType("channel" + String(c + 1), BaseType::INT16, ch->getTotalSamples()));
295-
}
296-
String spikeName("spike_group_" + String(fileIndex + 1));
297-
String fName(spikePath + spikeName);
298-
rec->mainFile = new NpyFile(fName + ".npy", spTypes);
279+
280+
uint32 procID = GenericProcessor::getProcessorFullId(ch->getSourceNodeID(), ch->getSubProcessorIdx());
281+
int groupIndex = ++groupMap[procID];
282+
283+
String spikeName = getProcessorString(ch) + "spike_group_" + String(groupIndex) + File::separatorString;
284+
285+
rec->mainFile = new NpyFile(spikePath + spikeName + "spike_waveforms.npy", NpyType(BaseType::INT16, ch->getTotalSamples()), ch->getNumChannels());
286+
rec->timestampFile = new NpyFile(spikePath + spikeName + "spike_times.npy", NpyType(BaseType::INT64, 1));
287+
rec->channelFile = new NpyFile(spikePath + spikeName + "spike_electrode_indices.npy", NpyType(BaseType::UINT16, 1));
288+
rec->extraFile = new NpyFile(spikePath + spikeName + "spike_clusters.npy", NpyType(BaseType::UINT16, 1));
299289
Array<NpyType> tsTypes;
300-
tsTypes.add(NpyType("timestamp", BaseType::INT64, 1));
301-
if (m_spikeMode == SpikeMode::AllInOne)
302-
{
303-
tsTypes.add(NpyType("electrode_index", BaseType::UINT16, 1));
304-
tsTypes.add(NpyType("sorted_id", BaseType::UINT16, 1));
305-
}
306-
else
307-
{
308-
Array<NpyType> indexType;
309-
indexType.add(NpyType("electrode_index", BaseType::UINT16, 1));
310-
if (m_spikeMode == SpikeMode::AllSeparated)
311-
{
312-
Array<NpyType> sortedType;
313-
sortedType.add(NpyType("sorted_id", BaseType::UINT16, 1));
314-
rec->extraFile = new NpyFile(fName + "_sortedID.npy", sortedType);
315-
}
316-
else
317-
{
318-
indexType.add(NpyType("sorted_id", BaseType::UINT16, 1));
319-
}
320-
rec->channelFile = new NpyFile(fName + "indexes.npy", indexType);
321-
}
322-
rec->timestampFile = new NpyFile(fName + "_timestamps.npy", tsTypes);
290+
323291
Array<var> jsonChanArray;
324292
jsonChanArray.add(var(jsonChannel));
325293
jsonSpikeChannels.add(var(jsonChanArray));
326294
DynamicObject::Ptr jsonFile = new DynamicObject();
327295

328-
jsonFile->setProperty("name", spikeName);
296+
jsonFile->setProperty("folder_name", spikeName.replace(File::separatorString,"/"));
329297
jsonFile->setProperty("sample_rate", ch->getSampleRate());
330298
jsonFile->setProperty("source_processor", ch->getSourceName());
331299
jsonFile->setProperty("num_channels", (int)numSpikeChannels);
332300
jsonFile->setProperty("pre_peak_samples", (int)ch->getPrePeakSamples());
333301
jsonFile->setProperty("post_peak_samples", (int)ch->getPostPeakSamples());
334302

335-
rec->metaDataFile = createEventMetadataFile(ch, fName + "_metadata.npy", jsonFile);
303+
rec->metaDataFile = createEventMetadataFile(ch, spikePath + spikeName + "metadata.npy", jsonFile);
336304
m_spikeFiles.add(rec.release());
337305
jsonSpikeFiles.add(var(jsonFile));
338306
}
@@ -552,38 +520,23 @@ void BinaryRecording::writeEvent(int eventIndex, const MidiMessage& event)
552520
const EventChannel* info = getEventChannel(eventIndex);
553521
int64 ts = ev->getTimestamp();
554522
rec->timestampFile->writeData(&ts, sizeof(int64));
523+
524+
uint16 chan = ev->getChannel();
525+
rec->channelFile->writeData(&chan, sizeof(uint16));
526+
555527
if (ev->getEventType() == EventChannel::TTL)
556528
{
557529
TTLEvent* ttl = static_cast<TTLEvent*>(ev.get());
558530
int16 data = ttl->getChannel() * (ttl->getState() ? 1 : -1);
559531
rec->mainFile->writeData(&data, sizeof(int16));
560-
NpyFile* wordFile = nullptr;
561-
if (m_TTLMode == TTLMode::JointWord)
562-
{
563-
wordFile = rec->mainFile;
564-
}
565-
else if (m_TTLMode == TTLMode::SeparateWord)
566-
{
567-
wordFile = rec->extraFile;
568-
}
569-
if (wordFile)
570-
wordFile->writeData(ttl->getTTLWordPointer(), info->getDataSize());
532+
if (rec->extraFile)
533+
rec->extraFile->writeData(ttl->getTTLWordPointer(), info->getDataSize());
571534
}
572535
else
573536
{
574537
rec->mainFile->writeData(ev->getRawDataPointer(), info->getDataSize());
575-
NpyFile* chanFile = nullptr;
576-
if (m_eventMode == EventMode::SeparateChannel)
577-
{
578-
chanFile = rec->channelFile;
579-
}
580-
else
581-
{
582-
chanFile = rec->timestampFile;
583-
}
584-
uint16 chan = ev->getChannel();
585-
chanFile->writeData(&chan, sizeof(uint16));
586538
}
539+
587540
writeEventMetaData(ev.get(), rec->metaDataFile);
588541
increaseEventCounts(rec);
589542
}
@@ -617,29 +570,15 @@ void BinaryRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike)
617570
FloatVectorOperations::copyWithMultiply(m_scaledBuffer.getData(), spike->getDataPointer(), multFactor, totalSamples);
618571
AudioDataConverters::convertFloatToInt16LE(m_scaledBuffer.getData(), m_intBuffer.getData(), totalSamples);
619572
rec->mainFile->writeData(m_intBuffer.getData(), totalSamples*sizeof(int16));
573+
620574
int64 ts = spike->getTimestamp();
621575
rec->timestampFile->writeData(&ts, sizeof(int64));
622-
NpyFile* indexFile;
623-
NpyFile* sortedFile;
624-
if (m_spikeMode == SpikeMode::AllInOne)
625-
{
626-
indexFile = rec->timestampFile;
627-
sortedFile = rec->timestampFile;
628-
}
629-
else if (m_spikeMode == SpikeMode::SeparateTimestamps)
630-
{
631-
indexFile = rec->channelFile;
632-
sortedFile = rec->channelFile;
633-
}
634-
else
635-
{
636-
indexFile = rec->channelFile;
637-
sortedFile = rec->extraFile;
638-
}
639-
indexFile->writeData(&spikeChannel, sizeof(uint16));
576+
577+
rec->channelFile->writeData(&spikeChannel, sizeof(uint16));
640578

641579
uint16 sortedID = spike->getSortedID();
642-
sortedFile->writeData(&sortedID, sizeof(uint16));
580+
rec->extraFile->writeData(&sortedID, sizeof(uint16));
581+
643582
increaseEventCounts(rec);
644583
}
645584

@@ -656,20 +595,15 @@ RecordEngineManager* BinaryRecording::getEngineManager()
656595
{
657596
RecordEngineManager* man = new RecordEngineManager("RAWBINARY", "Binary", &(engineFactory<BinaryRecording>));
658597
EngineParameter* param;
659-
param = new EngineParameter(EngineParameter::MULTI, 0, "Spike TS/chan/sortedID File Mode|All in one|Separate timestamps|All Separated", 0);
660-
man->addParameter(param);
661-
param = new EngineParameter(EngineParameter::MULTI, 1, "TTL Event word file|In main file|Separated|Do not save ttl word", 0);
662-
man->addParameter(param);
663-
param = new EngineParameter(EngineParameter::MULTI, 2, "Other event channel file|With timestamp|Separate", 0);
598+
param = new EngineParameter(EngineParameter::BOOL, 0, "Record TTL full words", true);
664599
man->addParameter(param);
600+
665601
return man;
666602
}
667603

668604
void BinaryRecording::setParameter(EngineParameter& parameter)
669605
{
670-
multiParameter(0, reinterpret_cast<int&>(m_spikeMode));
671-
multiParameter(1, reinterpret_cast<int&>(m_TTLMode));
672-
multiParameter(2, reinterpret_cast<int&>(m_eventMode));
606+
boolParameter(0, m_saveTTLWords);
673607
}
674608

675609
String BinaryRecording::jsonTypeValue(BaseType type)

‎Source/Plugins/BinaryWriter/BinaryRecording.h‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -61,37 +61,16 @@ namespace BinaryRecordingEngine
6161
ScopedPointer<NpyFile> extraFile;
6262
};
6363

64-
enum SpikeMode
65-
{
66-
AllInOne = 0,
67-
SeparateTimestamps = 1,
68-
AllSeparated = 2
69-
};
70-
71-
enum TTLMode
72-
{
73-
JointWord = 0,
74-
SeparateWord = 1,
75-
NoWord = 2
76-
};
77-
78-
enum EventMode
79-
{
80-
JointChannel = 0,
81-
SeparateChannel = 1
82-
};
8364

8465
NpyFile* createEventMetadataFile(const MetaDataEventObject* channel, String fileName, DynamicObject* jsonObject);
8566
void createChannelMetaData(const MetaDataInfoObject* channel, DynamicObject* jsonObject);
8667
void writeEventMetaData(const MetaDataEvent* event, NpyFile* file);
8768
void increaseEventCounts(EventRecording* rec);
8869
static String jsonTypeValue(BaseType type);
70+
static String getProcessorString(const InfoObjectCommon* channelInfo);
8971

90-
SpikeMode m_spikeMode;
91-
TTLMode m_TTLMode;
92-
EventMode m_eventMode;
93-
94-
72+
bool m_saveTTLWords{ true };
73+
9574
HeapBlock<float> m_scaledBuffer;
9675
HeapBlock<int16> m_intBuffer;
9776
HeapBlock<int64> m_tsBuffer;

0 commit comments

Comments
 (0)