Skip to content

Commit d68963e

Browse files
aacuevasaacuevas
authored
Merge pull request #231 from mspacek/binarywriter
BinaryWriter: fix data loss in large .npy files, tweaks, clean up
2 parents f9f7a9a + 6dbac10 commit d68963e

8 files changed

Lines changed: 998 additions & 987 deletions

File tree

‎Source/Plugins/BinaryWriter/BinaryRecording.cpp‎

Lines changed: 540 additions & 530 deletions
Large diffs are not rendered by default.

‎Source/Plugins/BinaryWriter/BinaryRecording.h‎

Lines changed: 66 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -30,72 +30,72 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
3030
namespace BinaryRecordingEngine
3131
{
3232

33-
class BinaryRecording : public RecordEngine
34-
{
35-
public:
36-
BinaryRecording();
37-
~BinaryRecording();
38-
39-
String getEngineID() const override;
40-
void openFiles(File rootFolder, int experimentNumber, int recordingNumber) override;
41-
void closeFiles() override;
42-
void writeData(int writeChannel, int realChannel, const float* buffer, int size) override;
43-
void writeEvent(int eventIndex, const MidiMessage& event) override;
44-
void resetChannels() override;
45-
void addSpikeElectrode(int index, const SpikeChannel* elec) override;
46-
void writeSpike(int electrodeIndex, const SpikeEvent* spike) override;
47-
void writeTimestampSyncText(uint16 sourceID, uint16 sourceIdx, int64 timestamp, float, String text) override;
48-
void setParameter(EngineParameter& parameter) override;
49-
50-
static RecordEngineManager* getEngineManager();
51-
52-
private:
53-
54-
class EventRecording
55-
{
56-
public:
57-
ScopedPointer<NpyFile> mainFile;
58-
ScopedPointer<NpyFile> timestampFile;
59-
ScopedPointer<NpyFile> metaDataFile;
60-
ScopedPointer<NpyFile> channelFile;
61-
ScopedPointer<NpyFile> extraFile;
62-
};
63-
64-
65-
NpyFile* createEventMetadataFile(const MetaDataEventObject* channel, String fileName, DynamicObject* jsonObject);
66-
void createChannelMetaData(const MetaDataInfoObject* channel, DynamicObject* jsonObject);
67-
void writeEventMetaData(const MetaDataEvent* event, NpyFile* file);
68-
void increaseEventCounts(EventRecording* rec);
69-
static String jsonTypeValue(BaseType type);
70-
static String getProcessorString(const InfoObjectCommon* channelInfo);
71-
72-
bool m_saveTTLWords{ true };
73-
74-
HeapBlock<float> m_scaledBuffer;
75-
HeapBlock<int16> m_intBuffer;
76-
HeapBlock<int64> m_tsBuffer;
77-
int m_bufferSize;
78-
79-
OwnedArray<SequentialBlockFile> m_DataFiles;
80-
Array<unsigned int> m_channelIndexes;
81-
Array<unsigned int> m_fileIndexes;
82-
OwnedArray<EventRecording> m_eventFiles;
83-
OwnedArray<EventRecording> m_spikeFiles;
84-
OwnedArray<NpyFile> m_dataTimestampFiles;
85-
ScopedPointer<FileOutputStream> m_syncTextFile;
86-
87-
Array<unsigned int> m_spikeFileIndexes;
88-
Array<uint16> m_spikeChannelIndexes;
89-
90-
int m_recordingNum;
91-
Array<int64> m_startTS;
92-
93-
94-
//Compile-time constants
95-
const int samplesPerBlock{ 4096 };
96-
97-
};
33+
class BinaryRecording : public RecordEngine
34+
{
35+
public:
36+
BinaryRecording();
37+
~BinaryRecording();
38+
39+
String getEngineID() const override;
40+
void openFiles(File rootFolder, int experimentNumber, int recordingNumber) override;
41+
void closeFiles() override;
42+
void writeData(int writeChannel, int realChannel, const float* buffer, int size) override;
43+
void writeEvent(int eventIndex, const MidiMessage& event) override;
44+
void resetChannels() override;
45+
void addSpikeElectrode(int index, const SpikeChannel* elec) override;
46+
void writeSpike(int electrodeIndex, const SpikeEvent* spike) override;
47+
void writeTimestampSyncText(uint16 sourceID, uint16 sourceIdx, int64 timestamp, float, String text) override;
48+
void setParameter(EngineParameter& parameter) override;
49+
50+
static RecordEngineManager* getEngineManager();
51+
52+
private:
53+
54+
class EventRecording
55+
{
56+
public:
57+
ScopedPointer<NpyFile> mainFile;
58+
ScopedPointer<NpyFile> timestampFile;
59+
ScopedPointer<NpyFile> metaDataFile;
60+
ScopedPointer<NpyFile> channelFile;
61+
ScopedPointer<NpyFile> extraFile;
62+
};
63+
64+
65+
NpyFile* createEventMetadataFile(const MetaDataEventObject* channel, String fileName, DynamicObject* jsonObject);
66+
void createChannelMetaData(const MetaDataInfoObject* channel, DynamicObject* jsonObject);
67+
void writeEventMetaData(const MetaDataEvent* event, NpyFile* file);
68+
void increaseEventCounts(EventRecording* rec);
69+
static String jsonTypeValue(BaseType type);
70+
static String getProcessorString(const InfoObjectCommon* channelInfo);
71+
72+
bool m_saveTTLWords{ true };
73+
74+
HeapBlock<float> m_scaledBuffer;
75+
HeapBlock<int16> m_intBuffer;
76+
HeapBlock<int64> m_tsBuffer;
77+
int m_bufferSize;
78+
79+
OwnedArray<SequentialBlockFile> m_DataFiles;
80+
Array<unsigned int> m_channelIndexes;
81+
Array<unsigned int> m_fileIndexes;
82+
OwnedArray<EventRecording> m_eventFiles;
83+
OwnedArray<EventRecording> m_spikeFiles;
84+
OwnedArray<NpyFile> m_dataTimestampFiles;
85+
ScopedPointer<FileOutputStream> m_syncTextFile;
86+
87+
Array<unsigned int> m_spikeFileIndexes;
88+
Array<uint16> m_spikeChannelIndexes;
89+
90+
int m_recordingNum;
91+
Array<int64> m_startTS;
92+
93+
94+
//Compile-time constants
95+
const int samplesPerBlock{ 4096 };
96+
97+
};
9898

9999
}
100100

101-
#endif
101+
#endif

‎Source/Plugins/BinaryWriter/FileMemoryBlock.h‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -29,40 +29,40 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
2929
namespace BinaryRecordingEngine
3030
{
3131

32-
template <class StorageType = int16>
33-
class FileMemoryBlock
34-
{
35-
public:
36-
FileMemoryBlock(FileOutputStream* file, int blockSize, uint64 offset) :
37-
m_data(blockSize, true),
38-
m_file(file),
39-
m_blockSize(blockSize),
40-
m_offset(offset)
41-
{};
42-
~FileMemoryBlock() {
43-
if (!m_flushed)
44-
{
45-
m_file->write(m_data, m_blockSize*sizeof(StorageType));
46-
}
47-
};
32+
template <class StorageType = int16>
33+
class FileMemoryBlock
34+
{
35+
public:
36+
FileMemoryBlock(FileOutputStream* file, int blockSize, uint64 offset) :
37+
m_data(blockSize, true),
38+
m_file(file),
39+
m_blockSize(blockSize),
40+
m_offset(offset)
41+
{};
42+
~FileMemoryBlock() {
43+
if (!m_flushed)
44+
{
45+
m_file->write(m_data, m_blockSize*sizeof(StorageType));
46+
}
47+
};
4848

49-
inline uint64 getOffset() { return m_offset; }
50-
inline StorageType* getData() { return m_data.getData(); }
51-
void partialFlush(size_t size, bool markFlushed = true)
52-
{
53-
std::cout << "flushing last block " << size << std::endl;
54-
m_file->write(m_data, size*sizeof(StorageType));
55-
if (markFlushed)
56-
m_flushed = true;
57-
}
49+
inline uint64 getOffset() { return m_offset; }
50+
inline StorageType* getData() { return m_data.getData(); }
51+
void partialFlush(size_t size, bool markFlushed = true)
52+
{
53+
std::cout << "flushing last block " << size << std::endl;
54+
m_file->write(m_data, size*sizeof(StorageType));
55+
if (markFlushed)
56+
m_flushed = true;
57+
}
5858

59-
private:
60-
HeapBlock<StorageType> m_data;
61-
FileOutputStream* const m_file;
62-
const int m_blockSize;
63-
const uint64 m_offset;
64-
bool m_flushed{ false };
65-
JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(FileMemoryBlock);
66-
};
59+
private:
60+
HeapBlock<StorageType> m_data;
61+
FileOutputStream* const m_file;
62+
const int m_blockSize;
63+
const uint64 m_offset;
64+
bool m_flushed{ false };
65+
JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR(FileMemoryBlock);
66+
};
6767
}
68-
#endif
68+
#endif

0 commit comments

Comments
 (0)