feat: Add 'whole' pattern and update config to bytes in reads_regional benchmark

Author: chandra-siri

packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads_regional/config.py

@@ -34,7 +34,7 @@ def _get_params() -> Dict[str, List[TimeBasedReadParameters]]:
 
     common_params = config["common"]
     read_types = common_params["read_types"]
-    file_sizes_mib = common_params["file_sizes_mib"]
+    file_sizes = common_params["file_sizes"]
     chunk_sizes_kib = common_params["chunk_sizes_kib"]
     num_ranges = common_params["num_ranges"]
     rounds = common_params["rounds"]
@@ -56,7 +56,7 @@ def _get_params() -> Dict[str, List[TimeBasedReadParameters]]:
         # Create a product of all parameter combinations
         product = itertools.product(
             read_types,
-            file_sizes_mib,
+            file_sizes,
             chunk_sizes_kib,
             num_ranges,
             processes,
@@ -65,19 +65,19 @@ def _get_params() -> Dict[str, List[TimeBasedReadParameters]]:
 
         for (
             read_type,
-            file_size_mib,
+            file_size,
             chunk_size_kib,
             num_ranges_val,
             num_processes,
             num_coros,
         ) in product:
-            file_size_bytes = file_size_mib * 1024 * 1024
+            file_size_bytes = file_size
             chunk_size_bytes = chunk_size_kib * 1024
 
             num_files = num_processes
 
             # Create a descriptive name for the parameter set
-            name = f"{pattern}_{read_type}_{num_processes}p_{num_coros}c_{file_size_mib}MiB_{chunk_size_kib}KiB_{num_ranges_val}ranges"
+            name = f"{pattern}_{read_type}_{num_processes}p_{num_coros}c_{file_size / (1024 * 1024)}MiB_{chunk_size_kib}KiB_{num_ranges_val}ranges"
 
             params[workload_name].append(
                 TimeBasedReadParameters(

packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads_regional/config.yaml

@@ -3,8 +3,8 @@ common:
     - "async_json"
     - "async_grpc_dp"
     - "async_grpc_cp"
-  file_sizes_mib:
-    - 10240 # 10GiB
+  file_sizes:
+    - 10737418240 # 10GiB in bytes
   chunk_sizes_kib: [64]
   num_ranges: [1]
   rounds: 1
@@ -23,5 +23,10 @@ workload:
     coros: [1, 16]
     processes: [1]
 
+  - name: "read_whole_multi_process"
+    pattern: "whole"
+    coros: [1]
+    processes: [1]
+
 defaults:
   DEFAULT_STANDARD_BUCKET: "chandrasiri-benchmarks-rb"

packages/google-cloud-storage/tests/perf/microbenchmarks/time_based/reads_regional/test_reads.py

@@ -91,6 +91,15 @@ def _cleanup_session():
 
 async def _download_time_based_json_async(session, filename, params):
     """Performs time-based downloads using the JSON API via aiohttp."""
+    if params.pattern == "whole":
+        url = f"https://storage.googleapis.com/storage/v1/b/{params.bucket_name}/o/{filename}?alt=media"
+        headers = {
+            "Authorization": f"Bearer {token}",
+        }
+        async with session.get(url, headers=headers) as response:
+            data = await response.read()
+            return len(data)
+
     total_bytes_downloaded = 0
     offset = 0
     is_warming_up = True
@@ -137,6 +146,12 @@ async def _download_time_based_async(client, filename, params):
     mrd = AsyncMultiRangeDownloader(client, params.bucket_name, filename)
     await mrd.open()
 
+    if params.pattern == "whole":
+        ranges = [(0, params.file_size_bytes, BytesIO())]
+        await mrd.download_ranges(ranges)
+        await mrd.close()
+        return ranges[0][2].getbuffer().nbytes
+
     async def _worker_coro():
         total_bytes_downloaded = 0
         offset = 0
@@ -224,21 +239,28 @@ def target_wrapper(*args, **kwargs):
         download_bytes_list.append(download_files_mp_mc_wrapper(pool, *args, **kwargs))
         return
 
+    duration_pedantic = 0
     try:
         with monitor() as m:
+            start_pedantic = time.monotonic()
             benchmark.pedantic(
                 target=target_wrapper,
                 iterations=1,
                 rounds=params.rounds,
                 args=(files_names, params, params.read_type),
             )
+            end_pedantic = time.monotonic()
+            duration_pedantic = end_pedantic - start_pedantic
     finally:
         pool.close()
         pool.join()
         total_bytes_downloaded = sum(download_bytes_list)
-        throughput_mib_s = (
-            total_bytes_downloaded / params.duration / params.rounds
-        ) / (1024 * 1024)
+        if params.pattern == "whole":
+            throughput_mib_s = (total_bytes_downloaded / duration_pedantic) / (1024 * 1024)
+        else:
+            throughput_mib_s = (
+                total_bytes_downloaded / params.duration / params.rounds
+            ) / (1024 * 1024)
         benchmark.extra_info["avg_throughput_mib_s"] = f"{throughput_mib_s:.2f}"
         print(
             f"Avg Throughput of {params.rounds} round(s): {throughput_mib_s:.2f} MiB/s"