Skip to content

Commit 5f87ce3

Browse files
committed
perf: use _DummyListBuffer in test_reads.py to avoid GIL contention
1 parent 2bfa5d6 commit 5f87ce3

1 file changed

Lines changed: 21 additions & 3 deletions

File tree

  • packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads

packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads/test_reads.py

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,24 @@ def _download_time_based_json(client, filename, params):
170170
)
171171

172172

173+
# _DummyListBuffer is used instead of io.BytesIO to avoid GIL contention
174+
# during profiling. io.BytesIO.write() holds the GIL while copying data,
175+
# which introduces significant noise and bottlenecks in performance tests
176+
# with high concurrency or large data transfers.
177+
# This buffer simply collects chunks in a list and tracks the total size.
178+
class _DummyListBuffer:
179+
def __init__(self):
180+
self.chunks = []
181+
self.size = 0
182+
183+
def write(self, data):
184+
self.chunks.append(data)
185+
self.size += len(data)
186+
187+
def getvalue(self):
188+
return b"".join(self.chunks)
189+
190+
173191
async def _download_time_based_async(client, filename, params):
174192
mrd = AsyncMultiRangeDownloader(client, params.bucket_name, filename)
175193
await mrd.open()
@@ -197,17 +215,17 @@ async def _worker_coro():
197215
offset = random.randint(
198216
0, params.file_size_bytes - params.chunk_size_bytes
199217
)
200-
ranges.append((offset, params.chunk_size_bytes, BytesIO()))
218+
ranges.append((offset, params.chunk_size_bytes, _DummyListBuffer()))
201219
else: # seq
202220
for _ in range(params.num_ranges):
203-
ranges.append((offset, params.chunk_size_bytes, BytesIO()))
221+
ranges.append((offset, params.chunk_size_bytes, _DummyListBuffer()))
204222
offset += params.chunk_size_bytes
205223
if offset + params.chunk_size_bytes > params.file_size_bytes:
206224
offset = 0 # Reset offset if end of file is reached
207225

208226
await mrd.download_ranges(ranges)
209227

210-
bytes_in_buffers = sum(r[2].getbuffer().nbytes for r in ranges)
228+
bytes_in_buffers = sum(r[2].size for r in ranges)
211229
assert bytes_in_buffers == params.chunk_size_bytes * params.num_ranges
212230

213231
if not is_warming_up:

0 commit comments

Comments
 (0)