feat(Storage): list buckets partial success (#8745)

devarr1994 · web-flow · commit 694e94ad9919 · 2025-12-05T10:42:01.000-08:00
diff --git a/Storage/src/BucketIterator.php b/Storage/src/BucketIterator.php
@@ -0,0 +1,50 @@
+<?php
+/**
+ * Copyright 2025 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Google\Cloud\Storage;
+
+use ArrayObject;
+use Google\Cloud\Core\Iterator\ItemIterator;
+use Google\Cloud\Core\Iterator\PageIterator;
+
+/**
+ * Iterates over a set of buckets.
+ *
+ * Use the `unreachable()` method to get a list of bucket names that could not
+ * be retrieved. This is only populated when the `returnPartialSuccess` is set to true
+ */
+class BucketIterator extends ItemIterator
+{
+    /**
+     * @param PageIterator $iterator
+     * @param ArrayObject $unreachable
+     */
+    public function __construct(PageIterator $iterator, private ArrayObject $unreachable)
+    {
+        parent::__construct($iterator);
+    }
+
+    /**
+     * Get the list of unreachable buckets.
+     *
+     * @return array<string>
+     */
+    public function unreachable()
+    {
+        return $this->unreachable->getArrayCopy();
+    }
+}
diff --git a/Storage/src/Connection/ServiceDefinition/storage-v1.json b/Storage/src/Connection/ServiceDefinition/storage-v1.json
@@ -745,6 +745,13 @@
                         "$ref": "Bucket"
                     }
                 },
+                "unreachable": {
+                    "type": "array",
+                    "description": "The list of bucket resource names that could not be reached during the listing operation.",
+                    "items": {
+                        "type": "string"
+                    }
+                },
                 "kind": {
                     "type": "string",
                     "description": "The kind of item this is. For lists of buckets, this is always storage#buckets.",
@@ -2528,6 +2535,11 @@
                             "type": "boolean",
                             "description": "If set to true, only soft-deleted bucket versions are listed as distinct results in order of bucket name and generation number. The default value is false.",
                             "location": "query"
+                        },
+                        "returnPartialSuccess": {
+                            "type": "boolean",
+                            "description": "If true, returns a partial list of buckets. The `unreachable` field will contain buckets that were not reachable.",
+                            "location": "query"
                         }
                     },
                     "parameterOrder": [
diff --git a/Storage/src/StorageClient.php b/Storage/src/StorageClient.php
@@ -273,24 +273,37 @@ public function bucket($name, $userProject = false, array $options = [])
      *           If false, `$options.userProject` will be used ONLY for the
      *           listBuckets operation. If `$options.userProject` is not set,
      *           this option has no effect. **Defaults to** `true`.
+     *      @type bool $returnPartialSuccess If true, the returned iterator will contain an
+     *           `unreachable` property with a list of buckets that were not retrieved.
+     *           **Note:** If set to false (default) and unreachable buckets are found,
+     *           the operation will throw an exception.
+     *
      * }
-     * @return ItemIterator<Bucket>
+     * @return BucketIterator<Bucket>
      * @throws GoogleException When a project ID has not been detected.
      */
     public function buckets(array $options = [])
     {
         $this->requireProjectId();
-
         $resultLimit = $this->pluck('resultLimit', $options, false);
-        $bucketUserProject = $this->pluck('bucketUserProject', $options, false);
-        $bucketUserProject = !is_null($bucketUserProject)
-            ? $bucketUserProject
-            : true;
-        $userProject = (isset($options['userProject']) && $bucketUserProject)
-            ? $options['userProject']
-            : null;
-
-        return new ItemIterator(
+        $bucketUserProject = $this->pluck('bucketUserProject', $options, null) ?? true;
+        $userProject = $bucketUserProject ? ($options['userProject'] ?? null) : null;
+
+        $unreachable = new \ArrayObject();
+
+        $apiCall = [$this->connection, 'listBuckets'];
+        $callDelegate = function (array $args) use ($apiCall, $unreachable) {
+            $response = call_user_func($apiCall, $args);
+            if (isset($response['unreachable']) && is_array($response['unreachable'])) {
+                $current = $unreachable->getArrayCopy();
+                $updated = array_unique(array_merge($current, $response['unreachable']));
+                $unreachable->exchangeArray($updated);
+            }
+            return $response;
+        };
+
+        // Return the new BucketIterator with the wrapped unreachable bucket
+        return new BucketIterator(
             new PageIterator(
                 function (array $bucket) use ($userProject) {
                     return new Bucket(
@@ -299,10 +312,11 @@ function (array $bucket) use ($userProject) {
                         $bucket + ['requesterProjectId' => $userProject]
                     );
                 },
-                [$this->connection, 'listBuckets'],
+                $callDelegate,
                 $options + ['project' => $this->projectId],
                 ['resultLimit' => $resultLimit]
-            )
+            ),
+            $unreachable
         );
     }
 
diff --git a/Storage/tests/Unit/StorageClientTest.php b/Storage/tests/Unit/StorageClientTest.php
@@ -418,7 +418,7 @@ public function testRetriesConfiguration(
         }
 
         $objects = iterator_to_array($client->bucket('myBucket')->objects());
-        
+
         $this->assertEquals('file.txt', $objects[0]->name());
         $this->assertEquals($expectedRemainingResponses, $mockHandler->count());
     }
@@ -576,7 +576,7 @@ public function testDelayFunctionsConfiguration()
         ]);
 
         $objects = iterator_to_array($client->bucket('myBucket')->objects());
-        
+
         $this->assertEquals('file.txt', $objects[0]->name());
         $this->assertEquals([100, 200], $capturedDelays);
     }
@@ -624,7 +624,7 @@ public function testRetryListenerConfiguration()
         ];
         $mockHandler = new MockHandler($mockResponses);
         $handlerStack = HandlerStack::create($mockHandler);
-        
+
         $requestHistory = [];
         $handlerStack->push(\GuzzleHttp\Middleware::history($requestHistory));
         $guzzleClient = new \GuzzleHttp\Client(['handler' => $handlerStack]);
@@ -658,6 +658,100 @@ public function testRetryListenerConfiguration()
         $this->assertFalse($requestHistory[0]['request']->hasHeader('X-Retry-Attempt'));
         $this->assertEquals('1', $requestHistory[1]['request']->getHeaderLine('X-Retry-Attempt'));
     }
+
+    public function testListBucketsReturnPartialSuccess()
+    {
+        $expectedUnreachable = [
+            'projects/_/buckets/unreachable-1',
+            'projects/_/buckets/unreachable-2',
+        ];
+
+        $this->connection->listBuckets(
+            Argument::withEntry('returnPartialSuccess', true)
+        )->willReturn([
+            'nextPageToken' => 'token',
+            'unreachable' => $expectedUnreachable,
+            'items' => [
+                ['name' => 'bucket1']
+            ]
+        ], [
+            'items' => [
+                ['name' => 'bucket2']
+            ]
+        ]);
+
+        $this->connection->projectId()
+            ->willReturn(self::PROJECT);
+
+        $this->client->___setProperty('connection', $this->connection->reveal());
+        $responseWrapper = $this->client->buckets(['returnPartialSuccess' => true]);
+
+        $this->assertInstanceOf(
+            \Google\Cloud\Storage\BucketIterator::class,
+            $responseWrapper
+        );
+        $bucket = iterator_to_array($responseWrapper);
+
+        $this->assertCount(2, $bucket);
+        $this->assertEquals('bucket1', $bucket[0]->name());
+        $this->assertEquals('bucket2', $bucket[1]->name());
+        $this->assertNotEmpty($responseWrapper->unreachable());
+        $this->assertEquals($expectedUnreachable, $responseWrapper->unreachable());
+    }
+
+    public function testBucketsIgnoresUnreachableWhenPartialSuccessIsFalse()
+    {
+        $this->connection->listBuckets(
+            Argument::withEntry('returnPartialSuccess', false)
+        )->willReturn([
+            'nextPageToken' => 'token',
+            'items' => [
+                ['name' => 'bucket1']
+            ]
+        ], [
+            'items' => [
+                ['name' => 'bucket2']
+            ]
+        ]);
+
+        $this->connection->projectId()
+            ->willReturn(self::PROJECT);
+        $this->client->___setProperty('connection', $this->connection->reveal());
+        $responseWrapper = $this->client->buckets(['returnPartialSuccess' => false]);
+        $bucket = iterator_to_array($responseWrapper);
+
+        $this->assertCount(2, $bucket);
+        $this->assertEquals('bucket1', $bucket[0]->name());
+        $this->assertEquals('bucket2', $bucket[1]->name());
+        $this->assertEmpty($responseWrapper->unreachable());
+    }
+
+    public function testBucketsIgnoresUnreachableWhenOptionIsAbsent()
+    {
+        $this->connection->listBuckets(
+            Argument::withEntry('project', self::PROJECT)
+        )->willReturn([
+            'nextPageToken' => 'token',
+            'items' => [
+                ['name' => 'bucket1']
+            ]
+        ], [
+            'items' => [
+                ['name' => 'bucket2']
+            ]
+        ]);
+
+        $this->connection->projectId()
+            ->willReturn(self::PROJECT);
+        $this->client->___setProperty('connection', $this->connection->reveal());
+        $responseWrapper = $this->client->buckets();
+        $bucket = iterator_to_array($responseWrapper);
+
+        $this->assertCount(2, $bucket);
+        $this->assertEquals('bucket1', $bucket[0]->name());
+        $this->assertEquals('bucket2', $bucket[1]->name());
+        $this->assertEmpty($responseWrapper->unreachable());
+    }
 }
 
 //@codingStandardsIgnoreStart
