Skip to content

Commit 7fc3100

Browse files
committed
perf: use _DummyListBuffer in test_reads.py to avoid GIL contention
1 parent 64ac5f1 commit 7fc3100

1 file changed

Lines changed: 21 additions & 3 deletions

File tree

  • packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads

packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads/test_reads.py

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,24 @@ def _download_time_based_json(client, filename, params):
114114
return total_bytes_downloaded
115115

116116

117+
# _DummyListBuffer is used instead of io.BytesIO to avoid GIL contention
118+
# during profiling. io.BytesIO.write() holds the GIL while copying data,
119+
# which introduces significant noise and bottlenecks in performance tests
120+
# with high concurrency or large data transfers.
121+
# This buffer simply collects chunks in a list and tracks the total size.
122+
class _DummyListBuffer:
123+
def __init__(self):
124+
self.chunks = []
125+
self.size = 0
126+
127+
def write(self, data):
128+
self.chunks.append(data)
129+
self.size += len(data)
130+
131+
def getvalue(self):
132+
return b"".join(self.chunks)
133+
134+
117135
async def _download_time_based_async(client, filename, params):
118136
mrd = AsyncMultiRangeDownloader(client, params.bucket_name, filename)
119137
await mrd.open()
@@ -138,17 +156,17 @@ async def _worker_coro():
138156
offset = random.randint(
139157
0, params.file_size_bytes - params.chunk_size_bytes
140158
)
141-
ranges.append((offset, params.chunk_size_bytes, BytesIO()))
159+
ranges.append((offset, params.chunk_size_bytes, _DummyListBuffer()))
142160
else: # seq
143161
for _ in range(params.num_ranges):
144-
ranges.append((offset, params.chunk_size_bytes, BytesIO()))
162+
ranges.append((offset, params.chunk_size_bytes, _DummyListBuffer()))
145163
offset += params.chunk_size_bytes
146164
if offset + params.chunk_size_bytes > params.file_size_bytes:
147165
offset = 0 # Reset offset if end of file is reached
148166

149167
await mrd.download_ranges(ranges)
150168

151-
bytes_in_buffers = sum(r[2].getbuffer().nbytes for r in ranges)
169+
bytes_in_buffers = sum(r[2].size for r in ranges)
152170
assert bytes_in_buffers == params.chunk_size_bytes * params.num_ranges
153171

154172
if not is_warming_up:

0 commit comments

Comments
 (0)