diff --git a/.github/workflows/conformance-test.yaml b/.github/workflows/conformance-test.yaml index 845d280462d3..8789e93fd02f 100644 --- a/.github/workflows/conformance-test.yaml +++ b/.github/workflows/conformance-test.yaml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 18 - run: node --version - run: cd handwritten/storage && npm install - run: cd handwritten/storage && npm run conformance-test diff --git a/handwritten/storage/.github/sync-repo-settings.yaml b/handwritten/storage/.github/sync-repo-settings.yaml index 556bfc53d5e2..9563dfc04dee 100644 --- a/handwritten/storage/.github/sync-repo-settings.yaml +++ b/handwritten/storage/.github/sync-repo-settings.yaml @@ -12,7 +12,6 @@ branchProtectionRules: requiredStatusCheckContexts: - "ci/kokoro: Samples test" - "ci/kokoro: System test" - - docs - lint - test (18) - test (20) diff --git a/handwritten/storage/.github/workflows/ci.yaml b/handwritten/storage/.github/workflows/ci.yaml index 8babaf86d550..8a89bb08dede 100644 --- a/handwritten/storage/.github/workflows/ci.yaml +++ b/handwritten/storage/.github/workflows/ci.yaml @@ -46,15 +46,3 @@ jobs: node-version: 18 - run: npm install - run: npm run lint - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 18 - - run: npm install - - run: npm run docs - - uses: JustinBeckwith/linkinator-action@v1 - with: - paths: docs/ diff --git a/handwritten/storage/conformance-test/conformanceCommon.ts b/handwritten/storage/conformance-test/conformanceCommon.ts index 3ffd0faa6daf..f0e0cb549843 100644 --- a/handwritten/storage/conformance-test/conformanceCommon.ts +++ b/handwritten/storage/conformance-test/conformanceCommon.ts @@ -13,24 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars import * as jsonToNodeApiMapping from './test-data/retryInvocationMap.json'; import * as libraryMethods from './libraryMethods'; -import { - Bucket, - File, - GaxiosOptions, - GaxiosOptionsPrepared, - HmacKey, - Notification, - Storage, -} from '../src'; +import {Bucket, File, Gaxios, HmacKey, Notification, Storage} from '../src'; import * as uuid from 'uuid'; import * as assert from 'assert'; import { StorageRequestOptions, StorageTransport, + StorageTransportCallback, } from '../src/storage-transport'; +import {getDirName} from '../src/util'; +import path from 'path'; +import {GoogleAuth} from 'google-auth-library'; interface RetryCase { instructions: String[]; } @@ -60,16 +55,31 @@ interface ConformanceTestResult { type LibraryMethodsModuleType = typeof import('./libraryMethods'); const methodMap: Map = new Map( - Object.entries({}), // TODO: replace with Object.entries(jsonToNodeApiMapping) + Object.entries(jsonToNodeApiMapping), ); const DURATION_SECONDS = 600; // 10 mins. const TESTS_PREFIX = `storage.retry.tests.${shortUUID()}.`; const TESTBENCH_HOST = process.env.STORAGE_EMULATOR_HOST || 'http://localhost:9000/'; -const CONF_TEST_PROJECT_ID = 'my-project-id'; +const CONF_TEST_PROJECT_ID = 'dummy-project-id'; const TIMEOUT_FOR_INDIVIDUAL_TEST = 20000; const RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS = 0.01; +const SERVICE_ACCOUNT = path.join( + getDirName(), + '../../../conformance-test/fixtures/signing-service-account.json', +); + +const authClient = new GoogleAuth({ + keyFilename: SERVICE_ACCOUNT, + scopes: ['https://www.googleapis.com/auth/devstorage.full_control'], +}).fromJSON(require(SERVICE_ACCOUNT)); + +authClient.getAccessToken = async () => ({token: 'unauthenticated-test-token'}); +authClient.request = async opts => { + const gaxios = new Gaxios(); + return gaxios.request(opts); +}; export function executeScenario(testCase: RetryTestCase) { for ( @@ -89,16 +99,17 @@ export function executeScenario(testCase: RetryTestCase) { let bucket: Bucket; let file: File; let notification: Notification; - let creationResult: {id: string}; + let creationResult: ConformanceTestCreationResult; let storage: Storage; let hmacKey: HmacKey; let storageTransport: StorageTransport; describe(`${storageMethodString}`, async () => { beforeEach(async () => { - storageTransport = new StorageTransport({ + const rawTransport = new StorageTransport({ apiEndpoint: TESTBENCH_HOST, - authClient: undefined, + authClient: authClient, + keyFilename: SERVICE_ACCOUNT, baseUrl: TESTBENCH_HOST, packageJson: {name: 'test-package', version: '1.0.0'}, retryOptions: { @@ -117,92 +128,71 @@ export function executeScenario(testCase: RetryTestCase) { timeout: DURATION_SECONDS, }); + creationResult = await createTestBenchRetryTest( + instructionSet.instructions, + jsonMethod?.name.toString(), + rawTransport, + ); + + // Create a Proxy around rawStorageTransport to intercept makeRequest + storageTransport = createRetryProxy( + rawTransport, + creationResult.id, + ); + storage = new Storage({ apiEndpoint: TESTBENCH_HOST, projectId: CONF_TEST_PROJECT_ID, + keyFilename: SERVICE_ACCOUNT, + authClient: authClient, retryOptions: { retryDelayMultiplier: RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS, }, }); - creationResult = await createTestBenchRetryTest( - instructionSet.instructions, - jsonMethod?.name.toString(), - storageTransport, + bucket = await createBucketForTest( + storage, + testCase.preconditionProvided && + !storageMethodString.includes('combine'), + storageMethodString, + ); + file = await createFileForTest( + testCase.preconditionProvided, + storageMethodString, + bucket, ); - if (storageMethodString.includes('InstancePrecondition')) { - bucket = await createBucketForTest( - storage, - testCase.preconditionProvided, - storageMethodString, - ); - file = await createFileForTest( - testCase.preconditionProvided, - storageMethodString, - bucket, - ); - } else { - bucket = await createBucketForTest( - storage, - false, - storageMethodString, - ); - file = await createFileForTest( - false, - storageMethodString, - bucket, - ); - } notification = bucket.notification(TESTS_PREFIX); await notification.create(); [hmacKey] = await storage.createHmacKey( `${TESTS_PREFIX}@email.com`, ); - - storage.interceptors.push({ - resolved: ( - requestConfig: GaxiosOptionsPrepared, - ): Promise => { - const config = requestConfig as GaxiosOptions; - config.headers = config.headers || {}; - Object.assign(config.headers, { - 'x-retry-test-id': creationResult.id, - }); - return Promise.resolve(config as GaxiosOptionsPrepared); - }, - rejected: error => { - return Promise.reject(error); - }, - }); }); it(`${instructionNumber}`, async () => { const methodParameters: libraryMethods.ConformanceTestOptions = { - storage: storage, - bucket: bucket, - file: file, - storageTransport: storageTransport, - notification: notification, - hmacKey: hmacKey, + storage, + bucket, + file, + storageTransport, + notification, + hmacKey, + projectId: CONF_TEST_PROJECT_ID, + preconditionRequired: testCase.preconditionProvided, }; - if (testCase.preconditionProvided) { - methodParameters.preconditionRequired = true; - } if (testCase.expectSuccess) { - assert.ifError(await storageMethodObject(methodParameters)); + await storageMethodObject(methodParameters); + const testBenchResult = await getTestBenchRetryTest( + creationResult.id, + storageTransport, + ); + assert.strictEqual(testBenchResult.completed, true); } else { await assert.rejects(async () => { await storageMethodObject(methodParameters); }, undefined); } - - const testBenchResult = await getTestBenchRetryTest( - creationResult.id, - storageTransport, - ); - assert.strictEqual(testBenchResult.completed, true); }).timeout(TIMEOUT_FOR_INDIVIDUAL_TEST); }); }); @@ -210,20 +200,51 @@ export function executeScenario(testCase: RetryTestCase) { } } +/** + * Creates a Proxy to automatically inject x-retry-test-id into all requests + */ +function createRetryProxy( + transport: StorageTransport, + retryId: string, +): StorageTransport { + return new Proxy(transport, { + get(target, prop, receiver) { + const original = Reflect.get(target, prop, receiver); + if (prop === 'makeRequest' && typeof original === 'function') { + return async ( + reqOpts: StorageRequestOptions, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + callback?: StorageTransportCallback, + ) => { + reqOpts.headers = reqOpts.headers || {}; + + if (reqOpts.headers instanceof Headers) { + reqOpts.headers.set('x-retry-test-id', retryId); + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (reqOpts.headers as any)['x-retry-test-id'] = retryId; + } + + return original.apply(target, [reqOpts, callback]); + }; + } + return original; + }, + }); +} + async function createBucketForTest( storage: Storage, - preconditionShouldBeOnInstance: boolean, - storageMethodString: String, + withPrecondition: boolean, + method: String, ) { - const name = generateName(storageMethodString, 'bucket'); - const bucket = storage.bucket(name); - await bucket.create(); + const bucket = storage.bucket(generateName(method, 'bucket')); + const [metadata] = await bucket.create(); await bucket.setRetentionPeriod(DURATION_SECONDS); - - if (preconditionShouldBeOnInstance) { + if (withPrecondition) { return new Bucket(storage, bucket.name, { preconditionOpts: { - ifMetagenerationMatch: 2, + ifMetagenerationMatch: metadata.metageneration, }, }); } @@ -231,61 +252,52 @@ async function createBucketForTest( } async function createFileForTest( - preconditionShouldBeOnInstance: boolean, - storageMethodString: String, + withPrecondition: boolean, + method: String, bucket: Bucket, ) { - const name = generateName(storageMethodString, 'file'); - const file = bucket.file(name); - await file.save(name); - if (preconditionShouldBeOnInstance) { + const file = bucket.file(generateName(method, 'file')); + await file.save('test-content'); + if (withPrecondition) { + const [metadata] = await file.getMetadata(); return new File(bucket, file.name, { preconditionOpts: { - ifMetagenerationMatch: file.metadata.metageneration, - ifGenerationMatch: file.metadata.generation, + ifMetagenerationMatch: metadata.metageneration, + ifGenerationMatch: metadata.generation, }, }); } return file; } -function generateName(storageMethodString: String, bucketOrFile: string) { - return `${TESTS_PREFIX}${storageMethodString.toLowerCase()}${bucketOrFile}.${shortUUID()}`; -} - async function createTestBenchRetryTest( instructions: String[], methodName: string, - storageTransport: StorageTransport, + transport: StorageTransport, ): Promise { - const requestBody = {instructions: {[methodName]: instructions}}; - - const requestOptions: StorageRequestOptions = { + return (await transport.makeRequest({ method: 'POST', url: 'retry_test', - body: JSON.stringify(requestBody), + body: JSON.stringify({instructions: {[methodName]: instructions}}), headers: {'Content-Type': 'application/json'}, - }; - - const response = await storageTransport.makeRequest(requestOptions); - return response as unknown as ConformanceTestCreationResult; + })) as ConformanceTestCreationResult; } async function getTestBenchRetryTest( testId: string, - storageTransport: StorageTransport, + transport: StorageTransport, ): Promise { - const response = await storageTransport.makeRequest({ + return (await transport.makeRequest({ url: `retry_test/${testId}`, method: 'GET', - retry: true, - headers: { - 'x-retry-test-id': testId, - }, - }); - return response as unknown as ConformanceTestResult; + headers: {'x-retry-test-id': testId}, + })) as ConformanceTestResult; +} + +function generateName(method: String, type: string) { + return `${TESTS_PREFIX}${method.toLowerCase()}${type}.${shortUUID()}`; } function shortUUID() { - return uuid.v1().split('-').shift(); + return uuid.v4().split('-').shift(); } diff --git a/handwritten/storage/conformance-test/libraryMethods.ts b/handwritten/storage/conformance-test/libraryMethods.ts index 26c466143b85..81f330829dee 100644 --- a/handwritten/storage/conformance-test/libraryMethods.ts +++ b/handwritten/storage/conformance-test/libraryMethods.ts @@ -21,18 +21,14 @@ import { Policy, GaxiosError, } from '../src'; -import * as path from 'path'; -import { - createTestBuffer, - createTestFileFromBuffer, - deleteTestFile, -} from './testBenchUtil'; +import {createTestBuffer} from './testBenchUtil'; import * as uuid from 'uuid'; -import {getDirName} from '../src/util.js'; -import {StorageTransport} from '../src/storage-transport'; +import { + StorageTransport, + StorageRequestOptions, +} from '../src/storage-transport'; const FILE_SIZE_BYTES = 9 * 1024 * 1024; -const CHUNK_SIZE_BYTES = 2 * 1024 * 1024; export interface ConformanceTestOptions { bucket?: Bucket; @@ -42,6 +38,8 @@ export interface ConformanceTestOptions { hmacKey?: HmacKey; preconditionRequired?: boolean; storageTransport?: StorageTransport; + projectId?: string; + retryTestId?: string; } ///////////////////////////////////////////////// @@ -51,63 +49,37 @@ export interface ConformanceTestOptions { export async function addLifecycleRuleInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.addLifecycleRule({ - action: { - type: 'Delete', - }, - condition: { - age: 365 * 3, // Specified in days. - }, - }); + return await addLifecycleRule(options); } export async function addLifecycleRule(options: ConformanceTestOptions) { - if (options.preconditionRequired) { - await options.bucket!.addLifecycleRule( - { - action: { - type: 'Delete', - }, - condition: { - age: 365 * 3, // Specified in days. - }, - }, - { - ifMetagenerationMatch: 2, - }, - ); - } else { - await options.bucket!.addLifecycleRule({ - action: { - type: 'Delete', - }, - condition: { - age: 365 * 3, // Specified in days. + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + lifecycle: { + rule: [ + { + action: {type: 'Delete'}, + condition: {age: 1095}, // Specified in days. + }, + ], }, - }); + }), + queryParameters: {}, + }; + + if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function combineInstancePrecondition( options: ConformanceTestOptions, ) { - const file1 = options.bucket!.file('file1.txt'); - const file2 = options.bucket!.file('file2.txt'); - await file1.save('file1 contents'); - await file2.save('file2 contents'); - let allFiles; - const sources = [file1, file2]; - if (options.preconditionRequired) { - allFiles = options.bucket!.file('all-files.txt', { - preconditionOpts: { - ifGenerationMatch: 0, - }, - }); - } else { - allFiles = options.bucket!.file('all-files.txt'); - } - - await options.bucket!.combine(sources, allFiles); + return await combine(options); } export async function combine(options: ConformanceTestOptions) { @@ -115,36 +87,122 @@ export async function combine(options: ConformanceTestOptions) { const file2 = options.bucket!.file('file2.txt'); await file1.save('file1 contents'); await file2.save('file2 contents'); - const sources = [file1, file2]; - const allFiles = options.bucket!.file('all-files.txt'); - await allFiles.save('allfiles contents'); + + const destinationFile = encodeURIComponent('all-files.txt'); + const body = { + sourceObjects: [{name: file1.name}, {name: file2.name}], + }; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${destinationFile}/compose`, + body: JSON.stringify(body), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.combine(sources, allFiles, { - ifGenerationMatch: allFiles.metadata.generation!, - }); + requestOptions.queryParameters!.ifGenerationMatch = 0; } else { - await options.bucket!.combine(sources, allFiles); + delete requestOptions.queryParameters!.ifGenerationMatch; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function create(options: ConformanceTestOptions) { - const [bucketExists] = await options.bucket!.exists(); + if (!options.storageTransport || !options.projectId || !options.bucket) { + throw new Error( + 'storageTransport, projectId, and bucket are required for the create test.', + ); + } + const bucketName = options.bucket.name; + let bucketExists = false; + const existsReq: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${bucketName}`, + }; + try { + await options.storageTransport.makeRequest(existsReq); + bucketExists = true; + } catch (e) { + const err = e as GaxiosError; + if (err.response?.status !== 404) { + throw e; + } + } + if (bucketExists) { - await options.bucket!.deleteFiles(); - await options.bucket!.delete({ - ignoreNotFound: true, - }); + let pageToken: string | undefined = undefined; + do { + const listReq: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${bucketName}/o`, + queryParameters: pageToken ? {pageToken} : undefined, + }; + const listResult = await options.storageTransport.makeRequest(listReq); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const objects = (listResult as any)?.items || []; + + for (const obj of objects) { + const deleteObjReq: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${bucketName}/o/${obj.name}`, + }; + await options.storageTransport.makeRequest(deleteObjReq); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + pageToken = (listResult as any)?.nextPageToken; + } while (pageToken); + + const deleteBucketReq: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${bucketName}`, + }; + await options.storageTransport.makeRequest(deleteBucketReq); } - await options.bucket!.create(); + + const createRequest: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b?project=${options.projectId}`, + body: JSON.stringify({name: bucketName}), + headers: {'Content-Type': 'application/json'}, + }; + await options.storageTransport.makeRequest(createRequest); } export async function createNotification(options: ConformanceTestOptions) { - await options.bucket!.createNotification('my-topic'); + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs`, + body: JSON.stringify({ + topic: 'my-topic', + }), + headers: {'Content-Type': 'application/json'}, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function deleteBucket(options: ConformanceTestOptions) { - await options.bucket!.deleteFiles(); - await options.bucket!.delete(); + try { + await options.bucket!.deleteFiles(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + const message = err.message || ''; + if (!message.includes('does not exist') && err.code !== 404) { + throw err; + } + } + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + queryParameters: {}, + }; + + if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = 1; + } + return await options.storageTransport!.makeRequest(requestOptions); } // Note: bucket.deleteFiles is missing from these tests @@ -153,331 +211,455 @@ export async function deleteBucket(options: ConformanceTestOptions) { export async function deleteLabelsInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.deleteLabels(); + return await deleteLabels(options); } export async function deleteLabels(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({labels: null}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.deleteLabels({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.deleteLabels(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function disableRequesterPaysInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.disableRequesterPays(); + return await disableRequesterPays(options); } export async function disableRequesterPays(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({billing: {requesterPays: false}}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.disableRequesterPays({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.disableRequesterPays(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function enableLoggingInstancePrecondition( options: ConformanceTestOptions, ) { - const config = { - prefix: 'log', - }; - await options.bucket!.enableLogging(config); + return await enableLogging(options); } export async function enableLogging(options: ConformanceTestOptions) { - let config; + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + logging: { + logBucket: options.bucket!.name, + logObjectPrefix: 'log', + }, + }), + queryParameters: {}, + }; + if (options.preconditionRequired) { - config = { - prefix: 'log', - ifMetagenerationMatch: 2, - }; - } else { - config = { - prefix: 'log', - }; + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } - await options.bucket!.enableLogging(config); + + return await options.storageTransport!.makeRequest(requestOptions); } export async function enableRequesterPaysInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.enableRequesterPays(); + return await enableRequesterPays(options); } export async function enableRequesterPays(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({billing: {requesterPays: true}}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.enableRequesterPays({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.enableRequesterPays(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketExists(options: ConformanceTestOptions) { - await options.bucket!.exists(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + }; + + try { + await options.storageTransport!.makeRequest(requestOptions); + return true; + } catch (err: unknown) { + const gaxiosError = err as GaxiosError; + if (gaxiosError.response?.status === 404) { + return false; + } + throw err; + } } export async function bucketGet(options: ConformanceTestOptions) { - await options.bucket!.get(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getFilesStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .bucket!.getFilesStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', (err: GaxiosError) => reject(err)); - }); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getLabels(options: ConformanceTestOptions) { - await options.bucket!.getLabels(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + queryParameters: { + fields: 'labels', + }, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketGetMetadata(options: ConformanceTestOptions) { - await options.bucket!.getMetadata(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + queryParameters: { + projection: 'full', + }, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getNotifications(options: ConformanceTestOptions) { - await options.bucket!.getNotifications(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function lock(options: ConformanceTestOptions) { - const metageneration = 0; - await options.bucket!.lock(metageneration); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const metadata: any = await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + headers: {...(retryId ? {'x-retry-test-id': retryId} : {})}, + }); + const currentMetageneration = metadata.metageneration; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/lockRetentionPolicy`, + queryParameters: { + ifMetagenerationMatch: currentMetageneration, + }, + headers: { + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketMakePrivateInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.makePrivate(); + return await bucketMakePrivate(options); } export async function bucketMakePrivate(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({acl: []}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.makePrivate({ - preconditionOpts: {ifMetagenerationMatch: 2}, - }); - } else { - await options.bucket!.makePrivate(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketMakePublic(options: ConformanceTestOptions) { - await options.bucket!.makePublic(); + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function removeRetentionPeriodInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.removeRetentionPeriod(); + return await removeRetentionPeriod(options); } export async function removeRetentionPeriod(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({retentionPolicy: null}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.removeRetentionPeriod({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.removeRetentionPeriod(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setCorsConfigurationInstancePrecondition( options: ConformanceTestOptions, ) { - const corsConfiguration = [{maxAgeSeconds: 3600}]; // 1 hour - await options.bucket!.setCorsConfiguration(corsConfiguration); + return await setCorsConfiguration(options); } export async function setCorsConfiguration(options: ConformanceTestOptions) { - const corsConfiguration = [{maxAgeSeconds: 3600}]; // 1 hour + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({cors: [{maxAgeSeconds: 3600}]}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.setCorsConfiguration(corsConfiguration, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setCorsConfiguration(corsConfiguration); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setLabelsInstancePrecondition( options: ConformanceTestOptions, ) { - const labels = { - labelone: 'labelonevalue', - labeltwo: 'labeltwovalue', - }; - await options.bucket!.setLabels(labels); + return await setLabels(options); } export async function setLabels(options: ConformanceTestOptions) { - const labels = { - labelone: 'labelonevalue', - labeltwo: 'labeltwovalue', + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + labels: {labelone: 'labelonevalue', labeltwo: 'labeltwovalue'}, + }), + queryParameters: {}, }; + if (options.preconditionRequired) { - await options.bucket!.setLabels(labels, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setLabels(labels); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketSetMetadataInstancePrecondition( options: ConformanceTestOptions, ) { - const metadata = { - website: { - mainPageSuffix: 'http://example.com', - notFoundPage: 'http://example.com/404.html', - }, - }; - await options.bucket!.setMetadata(metadata); + return await bucketSetMetadata(options); } export async function bucketSetMetadata(options: ConformanceTestOptions) { - const metadata = { - website: { - mainPageSuffix: 'http://example.com', - notFoundPage: 'http://example.com/404.html', - }, + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + website: { + mainPageSuffix: 'http://example.com', + notFoundPage: 'http://example.com/404.html', + }, + }), + queryParameters: {}, }; + if (options.preconditionRequired) { - await options.bucket!.setMetadata(metadata, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setMetadata(metadata); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setRetentionPeriodInstancePrecondition( options: ConformanceTestOptions, ) { - const DURATION_SECONDS = 15780000; // 6 months. - await options.bucket!.setRetentionPeriod(DURATION_SECONDS); + return await setRetentionPeriod(options); } export async function setRetentionPeriod(options: ConformanceTestOptions) { - const DURATION_SECONDS = 15780000; // 6 months. + const DURATION_SECONDS = 15780000; + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + retentionPolicy: {retentionPeriod: DURATION_SECONDS.toString()}, + }), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.setRetentionPeriod(DURATION_SECONDS, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setRetentionPeriod(DURATION_SECONDS); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketSetStorageClassInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.setStorageClass('nearline'); + return await bucketSetStorageClass(options); } export async function bucketSetStorageClass(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({storageClass: 'NEARLINE'}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.setStorageClass('nearline', { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setStorageClass('nearline'); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketUploadResumableInstancePrecondition( options: ConformanceTestOptions, ) { - const filePath = path.join( - getDirName(), - `../conformance-test/test-data/tmp-${uuid.v4()}.txt`, - ); - createTestFileFromBuffer(FILE_SIZE_BYTES, filePath); - if (options.bucket!.instancePreconditionOpts) { - options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0; - delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; - } - await options.bucket!.upload(filePath, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); - deleteTestFile(filePath); + return await bucketUploadResumable(options); } export async function bucketUploadResumable(options: ConformanceTestOptions) { - const filePath = path.join( - getDirName(), - `../conformance-test/test-data/tmp-${uuid.v4()}.txt`, - ); - createTestFileFromBuffer(FILE_SIZE_BYTES, filePath); + const fileName = `resumable-file-${uuid.v4()}.txt`; + const dataBuffer = Buffer.alloc(FILE_SIZE_BYTES, 'a'); + + const initiateOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'resumable', + name: fileName, + }, + headers: { + 'X-Upload-Content-Type': 'text/plain', + 'X-Upload-Content-Length': FILE_SIZE_BYTES.toString(), + }, + body: JSON.stringify({name: fileName}), + }; + if (options.preconditionRequired) { - await options.bucket!.upload(filePath, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.bucket!.upload(filePath, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); + initiateOptions.queryParameters = initiateOptions.queryParameters || {}; + initiateOptions.queryParameters.ifGenerationMatch = 0; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const response: any = + await options.storageTransport!.makeRequest(initiateOptions); + + let sessionUri: string | null = null; + + if (response.headers) { + if (typeof response.headers.get === 'function') { + sessionUri = response.headers.get('location'); + } + if (!sessionUri) { + sessionUri = response.headers.location || response.headers.Location; + } + } + + if (!sessionUri && response.data && typeof response.data === 'object') { + sessionUri = response.data.location; + } + + if (!sessionUri) { + throw new Error( + 'Failed to get session URI from resumable upload initiation.', + ); } - deleteTestFile(filePath); + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: sessionUri, + body: dataBuffer, + queryParameters: undefined, + headers: { + 'Content-Length': FILE_SIZE_BYTES.toString(), + 'Content-Range': `bytes 0-${FILE_SIZE_BYTES - 1}/${FILE_SIZE_BYTES}`, + }, + }); } export async function bucketUploadMultipartInstancePrecondition( options: ConformanceTestOptions, ) { - if (options.bucket!.instancePreconditionOpts) { - delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; - options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0; - } - await options.bucket!.upload( - path.join( - getDirName(), - '../../../conformance-test/test-data/retryStrategyTestData.json', - ), - {resumable: false}, - ); + return await bucketUploadMultipart(options); } export async function bucketUploadMultipart(options: ConformanceTestOptions) { - if (options.bucket!.instancePreconditionOpts) { - delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; - } + const fileName = 'retryStrategyTestData.json'; + const boundary = 'foo_bar_baz'; + + const metadata = JSON.stringify({ + name: fileName, + contentType: 'application/json', + }); + const media = JSON.stringify({some: 'data'}); + const body = + `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${metadata}\r\n` + + `--${boundary}\r\nContent-Type: application/json\r\n\r\n${media}\r\n` + + `--${boundary}--`; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'multipart', + name: fileName, + }, + headers: {'Content-Type': `multipart/related; boundary=${boundary}`}, + body: body, + }; if (options.preconditionRequired) { - await options.bucket!.upload( - path.join( - getDirName(), - '../../../conformance-test/test-data/retryStrategyTestData.json', - ), - {resumable: false, preconditionOpts: {ifGenerationMatch: 0}}, - ); - } else { - await options.bucket!.upload( - path.join( - getDirName(), - '../../../conformance-test/test-data/retryStrategyTestData.json', - ), - {resumable: false}, - ); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } ///////////////////////////////////////////////// @@ -485,195 +667,380 @@ export async function bucketUploadMultipart(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function copy(options: ConformanceTestOptions) { - const newFile = new File(options.bucket!, 'a-different-file.png'); - await newFile.save('a-different-file.png'); + const sourceBucket = options.bucket!.name; + const sourceFile = encodeURIComponent(options.file!.name); + const destinationFile = encodeURIComponent('a-different-file.png'); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${sourceBucket}/o/${sourceFile}/rewriteTo/b/${sourceBucket}/o/${destinationFile}`, + queryParameters: {}, + }; if (options.preconditionRequired) { - await options.file!.copy('a-different-file.png', { - preconditionOpts: { - ifGenerationMatch: newFile.metadata.generation!, - }, - }); - } else { - await options.file!.copy('a-different-file.png'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifSourceGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function createReadStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .file!.createReadStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', (err: GaxiosError) => reject(err)); - }); + return await download(options); } export async function createResumableUploadInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.createResumableUpload(); + return await createResumableUpload(options); } export async function createResumableUpload(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'resumable', + name: options.file!.name, + }, + }; + if (options.preconditionRequired) { - await options.file!.createResumableUpload({ - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.file!.createResumableUpload(); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileDeleteInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.delete(); + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + headers: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + 'x-retry-test-id': (options as any).retryTestId, + }, + }; + if (options.preconditionRequired) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifGenerationMatch = generation; + } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileDelete(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + headers: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + 'x-retry-test-id': (options as any).retryTestId, + }, + }; + if (options.preconditionRequired) { - await options.file!.delete({ - ifGenerationMatch: options.file!.metadata.generation, - }); - } else { - await options.file!.delete(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function download(options: ConformanceTestOptions) { - await options.file!.download(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {alt: 'media'}, + responseType: 'stream', + headers: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...(options as any).headers, + }, + }; + return await options.storageTransport!.makeRequest(requestOptions); } export async function exists(options: ConformanceTestOptions) { - await options.file!.exists(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + }; + + try { + await options.storageTransport!.makeRequest(requestOptions); + return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if (err.code === 404) return false; + throw err; + } } export async function get(options: ConformanceTestOptions) { - await options.file!.get(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getExpirationDate(options: ConformanceTestOptions) { - await options.file!.getExpirationDate(); + return await get(options); } export async function getMetadata(options: ConformanceTestOptions) { - await options.file!.getMetadata(); + return await get(options); } export async function isPublic(options: ConformanceTestOptions) { - await options.file!.isPublic(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${options.bucket!.name}/o/${encodeURIComponent(options.file!.name)}`, + }; + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileMakePrivateInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.makePrivate(); + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + body: JSON.stringify({acl: []}), + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + if (instanceOpts?.ifGenerationMatch !== undefined) { + requestOptions.queryParameters!.ifGenerationMatch = + instanceOpts.ifGenerationMatch; + } else if (instanceOpts?.ifMetagenerationMatch !== undefined) { + requestOptions.queryParameters!.ifMetagenerationMatch = + instanceOpts.ifMetagenerationMatch; + } else if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = + options.file?.metadata.metageneration; + } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileMakePrivate(options: ConformanceTestOptions) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + body: JSON.stringify({acl: []}), + queryParameters: {}, + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }; + if (options.preconditionRequired) { - await options.file!.makePrivate({ - preconditionOpts: { - ifMetagenerationMatch: options.file!.metadata.metageneration, - }, - }); - } else { - await options.file!.makePrivate(); + requestOptions.queryParameters!.ifMetagenerationMatch = + options.file!.metadata.metageneration ?? 1; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileMakePublic(options: ConformanceTestOptions) { - await options.file!.makePublic(); + const fileName = encodeURIComponent(options.file!.name); + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${fileName}/acl`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function move(options: ConformanceTestOptions) { + const sourceBucket = options.bucket!.name; + const sourceFile = encodeURIComponent(options.file!.name); + const destinationFile = encodeURIComponent('new-file'); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${sourceBucket}/o/${sourceFile}/rewriteTo/b/${sourceBucket}/o/${destinationFile}`, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.move('new-file', { - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.file!.move('new-file'); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function rename(options: ConformanceTestOptions) { + const sourceBucket = options.bucket!.name; + const sourceFile = encodeURIComponent(options.file!.name); + const destinationFile = encodeURIComponent('new-name'); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${sourceBucket}/o/${sourceFile}/rewriteTo/b/${sourceBucket}/o/${destinationFile}`, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.rename('new-name', { - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.file!.rename('new-name'); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function rotateEncryptionKey(options: ConformanceTestOptions) { - const crypto = require('crypto'); - const buffer = crypto.randomBytes(32); - const newKey = buffer.toString('base64'); + const bucketName = options.bucket!.name; + const fileName = encodeURIComponent(options.file!.name); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${bucketName}/o/${fileName}/rewriteTo/b/${bucketName}/o/${fileName}`, + headers: { + 'x-goog-copy-source-encryption-algorithm': 'AES256', + }, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.rotateEncryptionKey({ - encryptionKey: Buffer.from(newKey, 'base64'), - preconditionOpts: {ifGenerationMatch: options.file!.metadata.generation}, - }); - } else { - await options.file!.rotateEncryptionKey({ - encryptionKey: Buffer.from(newKey, 'base64'), - }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function saveResumableInstancePrecondition( options: ConformanceTestOptions, ) { - const buf = createTestBuffer(FILE_SIZE_BYTES); - await options.file!.save(buf, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); + return await saveResumable(options); } export async function saveResumable(options: ConformanceTestOptions) { - const buf = createTestBuffer(FILE_SIZE_BYTES); + const data = createTestBuffer(FILE_SIZE_BYTES); + const dataBuffer = Buffer.from(data); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + + const initiateOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'resumable', + name: options.file!.name, + }, + body: JSON.stringify({name: options.file!.name}), + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }; + if (options.preconditionRequired) { - await options.file!.save(buf, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - preconditionOpts: { - ifGenerationMatch: options.file!.metadata.generation, - ifMetagenerationMatch: options.file!.metadata.metageneration, - }, - }); - } else { - await options.file!.save(buf, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + initiateOptions.queryParameters!.ifGenerationMatch = generation; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const response: any = + await options.storageTransport!.makeRequest(initiateOptions); + let sessionUri: string | null = null; + if (response.headers) { + if (typeof response.headers.get === 'function') { + sessionUri = response.headers.get('location'); + } + if (!sessionUri) { + sessionUri = response.headers.location || response.headers.Location; + } } + + if (!sessionUri && response.data && typeof response.data === 'object') { + sessionUri = response.data.location; + } + + if (!sessionUri) throw new Error('Failed to get session URI'); + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: sessionUri, + body: dataBuffer, + queryParameters: undefined, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': dataBuffer.length.toString(), + 'Content-Range': `bytes 0-${dataBuffer.length - 1}/${dataBuffer.length}`, + 'x-retry-test-id': retryId, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any, + }); } export async function saveMultipartInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.save('testdata', {resumable: false}); + return await saveMultipart(options); } export async function saveMultipart(options: ConformanceTestOptions) { - if (options.preconditionRequired) { - await options.file!.save('testdata', { - resumable: false, - preconditionOpts: { - ifGenerationMatch: options.file!.metadata.generation, - }, - }); - } else { - await options.file!.save('testdata', { - resumable: false, - }); + const boundary = 'conformance_test_boundary'; + const fileName = options.file!.name; + + const metadata = JSON.stringify({name: fileName}); + const media = 'testdata'; + + const body = + `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${metadata}\r\n` + + `--${boundary}\r\nContent-Type: text/plain\r\n\r\n${media}\r\n` + + `--${boundary}--`; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'multipart', + name: options.file!.name, + }, + headers: {'Content-Type': `multipart/related; boundary=${boundary}`}, + body: body, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + if (instanceOpts?.ifGenerationMatch !== undefined) { + requestOptions.queryParameters!.ifGenerationMatch = + instanceOpts.ifGenerationMatch; + } else if (options.preconditionRequired) { + const generation = options.file?.metadata?.generation ?? 0; + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setMetadataInstancePrecondition( @@ -681,41 +1048,79 @@ export async function setMetadataInstancePrecondition( ) { const metadata = { contentType: 'application/x-font-ttf', - metadata: { - my: 'custom', - properties: 'go here', - }, + metadata: {my: 'custom', properties: 'go here'}, + }; + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + body: JSON.stringify(metadata), + headers: {'Content-Type': 'application/json'}, }; - await options.file!.setMetadata(metadata); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + + if (instanceOpts?.ifGenerationMatch !== undefined) { + requestOptions.queryParameters!.ifGenerationMatch = + instanceOpts.ifGenerationMatch; + } else if (instanceOpts?.ifMetagenerationMatch !== undefined) { + requestOptions.queryParameters!.ifMetagenerationMatch = + instanceOpts.ifMetagenerationMatch; + } else if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = + options.file?.metadata.metageneration; + } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setMetadata(options: ConformanceTestOptions) { const metadata = { contentType: 'application/x-font-ttf', - metadata: { - my: 'custom', - properties: 'go here', - }, + metadata: {my: 'custom', properties: 'go here'}, }; + + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${options.bucket!.name}/o/${encodeURIComponent(options.file!.name)}`, + body: JSON.stringify(metadata), + headers: {'Content-Type': 'application/json'}, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.setMetadata(metadata, { - ifMetagenerationMatch: options.file!.metadata.metageneration, - }); - } else { - await options.file!.setMetadata(metadata); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setStorageClass(options: ConformanceTestOptions) { + const bucketName = options.bucket!.name; + const fileName = encodeURIComponent(options.file!.name); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${bucketName}/o/${fileName}/rewriteTo/b/${bucketName}/o/${fileName}`, + body: JSON.stringify({storageClass: 'NEARLINE'}), + headers: {'Content-Type': 'application/json'}, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.setStorageClass('nearline', { - preconditionOpts: { - ifGenerationMatch: options.file!.metadata.generation, - }, - }); - } else { - await options.file!.setStorageClass('nearline'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifSourceGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } // ///////////////////////////////////////////////// @@ -723,26 +1128,49 @@ export async function setStorageClass(options: ConformanceTestOptions) { // ///////////////////////////////////////////////// export async function deleteHMAC(options: ConformanceTestOptions) { - const metadata = { - state: 'INACTIVE', - }; - await options.hmacKey!.setMetadata(metadata); - await options.hmacKey!.delete(); + await options.storageTransport!.makeRequest({ + method: 'PUT', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + body: JSON.stringify({state: 'INACTIVE'}), + headers: {'x-retry-test-id': ''}, + }); + return await options.storageTransport!.makeRequest({ + method: 'DELETE', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + }); } export async function getHMAC(options: ConformanceTestOptions) { - await options.hmacKey!.get(); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + }); } export async function getMetadataHMAC(options: ConformanceTestOptions) { - await options.hmacKey!.getMetadata(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setMetadataHMAC(options: ConformanceTestOptions) { - const metadata = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const body: any = { state: 'INACTIVE', }; - await options.hmacKey!.setMetadata(metadata); + + if (options.preconditionRequired && options.hmacKey?.metadata?.etag) { + body.etag = options.hmacKey.metadata.etag; + } + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + body: JSON.stringify(body), + }); } ///////////////////////////////////////////////// @@ -750,11 +1178,17 @@ export async function setMetadataHMAC(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function iamGetPolicy(options: ConformanceTestOptions) { - await options.bucket!.iam.getPolicy({requestedPolicyVersion: 1}); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam`, + queryParameters: {optionsRequestedPolicyVersion: 1}, + }); } export async function iamSetPolicy(options: ConformanceTestOptions) { - const testPolicy: Policy = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + const body: Policy = { bindings: [ { role: 'roles/storage.admin', @@ -762,16 +1196,40 @@ export async function iamSetPolicy(options: ConformanceTestOptions) { }, ], }; + if (options.preconditionRequired) { - const currentPolicy = await options.bucket!.iam.getPolicy(); - testPolicy.etag = currentPolicy[0].etag; + const getResponse = await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam`, + queryParameters: {optionsRequestedPolicyVersion: 1}, + headers: retryId ? {'x-retry-test-id': retryId} : {}, + }); + + const currentPolicy = getResponse as Policy; + const fetchedEtag = currentPolicy.etag; + + if (fetchedEtag) { + body.etag = fetchedEtag; + } } - await options.bucket!.iam.setPolicy(testPolicy); + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam`, + body: JSON.stringify(body), + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }); } export async function iamTestPermissions(options: ConformanceTestOptions) { - const permissionToTest = 'storage.buckets.delete'; - await options.bucket!.iam.testPermissions(permissionToTest); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam/testPermissions`, + queryParameters: {permissions: 'storage.buckets.delete'}, + }); } ///////////////////////////////////////////////// @@ -779,23 +1237,55 @@ export async function iamTestPermissions(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function notificationDelete(options: ConformanceTestOptions) { - await options.notification!.delete(); + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs/${options.notification!.id}`, + queryParameters: {}, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function notificationCreate(options: ConformanceTestOptions) { - await options.notification!.create(); + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs`, + body: JSON.stringify({ + topic: 'my-topic', + }), + headers: {'Content-Type': 'application/json'}, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function notificationExists(options: ConformanceTestOptions) { - await options.notification!.exists(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs/${options.notification!.id}`, + }; + + try { + await options.storageTransport!.makeRequest(requestOptions); + return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if (err.code === 404) return false; + throw err; + } } export async function notificationGet(options: ConformanceTestOptions) { - await options.notification!.get(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs/${options.notification!.id}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function notificationGetMetadata(options: ConformanceTestOptions) { - await options.notification!.getMetadata(); + return await notificationGet(options); } ///////////////////////////////////////////////// @@ -803,43 +1293,74 @@ export async function notificationGetMetadata(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function createBucket(options: ConformanceTestOptions) { - const bucket = options.storage!.bucket('test-creating-bucket'); - const [exists] = await bucket.exists(); - if (exists) { - await bucket.delete(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + const bucketName = options.bucket!.name; + + try { + return await options.storageTransport!.makeRequest({ + method: 'POST', + url: 'storage/v1/b', + queryParameters: {project: options.projectId}, + body: JSON.stringify({name: bucketName}), + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if ( + err.response?.status === 409 || + err.message?.includes('already exists') + ) { + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(bucketName)}`, + headers: { + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }); + } + throw err; } - await options.storage!.createBucket('test-creating-bucket'); } export async function createHMACKey(options: ConformanceTestOptions) { const serviceAccountEmail = 'my-service-account@appspot.gserviceaccount.com'; - await options.storage!.createHmacKey(serviceAccountEmail); + return await options.storageTransport!.makeRequest({ + method: 'POST', + url: `storage/v1/projects/${options.projectId}/hmacKeys`, + queryParameters: {serviceAccountEmail}, + }); } export async function getBuckets(options: ConformanceTestOptions) { - await options.storage!.getBuckets(); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: 'storage/v1/b', + queryParameters: {project: options.projectId}, + }); } export async function getBucketsStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .storage!.getBucketsStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', err => reject(err)); - }); + return await getBuckets(options); } -export function getHMACKeyStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .storage!.getHmacKeysStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', err => reject(err)); +export async function getHMACKeyStream(options: ConformanceTestOptions) { + const serviceAccountEmail = 'my-service-account@appspot.gserviceaccount.com'; + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/projects/${options.projectId}/hmacKeys`, + queryParameters: {serviceAccountEmail}, }); } export async function getServiceAccount(options: ConformanceTestOptions) { - await options.storage!.getServiceAccount(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/projects/${options.projectId}/serviceAccount`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } diff --git a/handwritten/storage/conformance-test/test-data/retryInvocationMap.json b/handwritten/storage/conformance-test/test-data/retryInvocationMap.json index 8dea345f12c3..c9a52c43649a 100644 --- a/handwritten/storage/conformance-test/test-data/retryInvocationMap.json +++ b/handwritten/storage/conformance-test/test-data/retryInvocationMap.json @@ -44,7 +44,7 @@ "storage.notifications.list": [ "getNotifications" ], - "storage.buckets.lockRententionPolicy": [ + "storage.buckets.lockRetentionPolicy": [ "lock" ], "storage.objects.patch": [ @@ -134,6 +134,7 @@ "getMetadataHMAC" ], "storage.hmacKey.update": [ + "setMetadataHMAC" ], "storage.hmacKey.create": [ "createHMACKey" diff --git a/handwritten/storage/conformance-test/testBenchUtil.ts b/handwritten/storage/conformance-test/testBenchUtil.ts index b66f83094588..c231c86cf0b2 100644 --- a/handwritten/storage/conformance-test/testBenchUtil.ts +++ b/handwritten/storage/conformance-test/testBenchUtil.ts @@ -22,7 +22,7 @@ const PORT = new URL(HOST).port; const CONTAINER_NAME = 'storage-testbench'; const DEFAULT_IMAGE_NAME = 'gcr.io/cloud-devrel-public-resources/storage-testbench'; -const DEFAULT_IMAGE_TAG = 'v0.35.0'; +const DEFAULT_IMAGE_TAG = 'v0.60.0'; const DOCKER_IMAGE = `${DEFAULT_IMAGE_NAME}:${DEFAULT_IMAGE_TAG}`; const PULL_CMD = `docker pull ${DOCKER_IMAGE}`; const RUN_CMD = `docker run --rm -d -p ${PORT}:${PORT} --name ${CONTAINER_NAME} ${DOCKER_IMAGE} && sleep 1`; diff --git a/handwritten/storage/package.json b/handwritten/storage/package.json index 9c3171359f93..5cb984e03a6a 100644 --- a/handwritten/storage/package.json +++ b/handwritten/storage/package.json @@ -83,7 +83,6 @@ "fast-xml-parser": "^5.2.0", "gaxios": "^7.0.0-rc.4", "google-auth-library": "^10.1.0", - "html-entities": "^2.6.0", "mime": "3.0.0", "p-limit": "3.1.0", "uuid": "^11.1.0" diff --git a/handwritten/storage/src/acl.ts b/handwritten/storage/src/acl.ts index 13f019a626fd..08c4c237c960 100644 --- a/handwritten/storage/src/acl.ts +++ b/handwritten/storage/src/acl.ts @@ -528,10 +528,10 @@ class Acl extends AclRoleAccessorMethods { if (this.parent instanceof File) { const file = this.parent as File; const bucket = file.parent; - url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`; + url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`; } else if (this.parent instanceof Bucket) { const bucket = this.parent as Bucket; - url = `${bucket.baseUrl}/${bucket.name}${url}`; + url = `/storage/v1/b/${bucket.name}${url}`; } this.storageTransport @@ -644,14 +644,14 @@ class Acl extends AclRoleAccessorMethods { query.userProject = options.userProject; } - let url = `${this.pathPrefix}/${options.entity}`; + let url = `${this.pathPrefix}/${encodeURIComponent(options.entity)}`; if (this.parent instanceof File) { const file = this.parent as File; const bucket = file.parent; - url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`; + url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`; } else if (this.parent instanceof Bucket) { const bucket = this.parent as Bucket; - url = `${bucket.baseUrl}/${bucket.name}${url}`; + url = `/storage/v1/b/${bucket.name}${url}`; } this.storageTransport @@ -768,7 +768,7 @@ class Acl extends AclRoleAccessorMethods { let url = `${this.pathPrefix}`; if (options) { - url = `${url}/${options.entity}`; + url = `${url}/${encodeURIComponent(options.entity)}`; if (options.generation) { query.generation = options.generation; } @@ -781,10 +781,10 @@ class Acl extends AclRoleAccessorMethods { if (this.parent instanceof File) { const file = this.parent as File; const bucket = file.parent; - url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`; + url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`; } else if (this.parent instanceof Bucket) { const bucket = this.parent as Bucket; - url = `${bucket.baseUrl}/${bucket.name}${url}`; + url = `/storage/v1/b/${bucket.name}${url}`; } this.storageTransport @@ -888,14 +888,14 @@ class Acl extends AclRoleAccessorMethods { query.userProject = options.userProject; } - let url = `${this.pathPrefix}/${options.entity}`; + let url = `${this.pathPrefix}/${encodeURIComponent(options.entity)}`; if (this.parent instanceof File) { const file = this.parent as File; const bucket = file.parent; - url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`; + url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`; } else if (this.parent instanceof Bucket) { const bucket = this.parent as Bucket; - url = `${bucket.baseUrl}/${bucket.name}${url}`; + url = `/storage/v1/b/${bucket.name}${url}`; } this.storageTransport diff --git a/handwritten/storage/src/bucket.ts b/handwritten/storage/src/bucket.ts index ae71afbd1972..22db62d6a25a 100644 --- a/handwritten/storage/src/bucket.ts +++ b/handwritten/storage/src/bucket.ts @@ -1232,7 +1232,7 @@ class Bucket extends ServiceObject { super({ storageTransport: storage.storageTransport, parent: storage, - baseUrl: '/b', + baseUrl: '/storage/v1/b', id: name, createMethod: storage.createBucket.bind(storage), methods, @@ -1688,7 +1688,7 @@ class Bucket extends ServiceObject { .makeRequest( { method: 'POST', - url: '/compose', + url: `/storage/v1/b/${this.name}/o/${encodeURIComponent(destinationFile.name)}/compose`, maxRetries, body: JSON.stringify({ destination: { @@ -1709,6 +1709,9 @@ class Bucket extends ServiceObject { return sourceObject; }), }), + headers: { + 'Content-Type': 'application/json', + }, queryParameters: options as unknown as StorageQueryParameters, }, (err, resp) => { @@ -2050,6 +2053,9 @@ class Bucket extends ServiceObject { body: JSON.stringify(convertObjKeysToSnakeCase(body)), queryParameters: query as unknown as StorageQueryParameters, retry: false, + headers: { + 'Content-Type': 'application/json', + }, }, (err, data, resp) => { if (err) { diff --git a/handwritten/storage/src/channel.ts b/handwritten/storage/src/channel.ts index cd3251bc1630..edf74e686b31 100644 --- a/handwritten/storage/src/channel.ts +++ b/handwritten/storage/src/channel.ts @@ -43,7 +43,7 @@ class Channel extends ServiceObject { const config = { parent: storage, storageTransport: storage.storageTransport, - baseUrl: '/channels', + baseUrl: '/storage/v1/channels', id: '', methods: {}, }; @@ -89,6 +89,9 @@ class Channel extends ServiceObject { method: 'POST', url: `${this.baseUrl}/stop`, body: JSON.stringify(this.metadata), + headers: { + 'Content-Type': 'application/json', + }, responseType: 'json', }, (err, data, resp) => { diff --git a/handwritten/storage/src/file.ts b/handwritten/storage/src/file.ts index f1edc193204b..459c0892cac5 100644 --- a/handwritten/storage/src/file.ts +++ b/handwritten/storage/src/file.ts @@ -81,7 +81,6 @@ import { StorageQueryParameters, StorageRequestOptions, } from './storage-transport.js'; -import * as gaxios from 'gaxios'; import mime from 'mime'; export type GetExpirationDateResponse = [Date]; @@ -1363,13 +1362,24 @@ class File extends ServiceObject { } if (newFile.encryptionKey !== undefined) { - this.setEncryptionKey(newFile.encryptionKey!); + headers.set('x-goog-encryption-algorithm', 'AES256'); + headers.set( + 'x-goog-encryption-key', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (newFile as any).encryptionKeyBase64 || '', + ); + headers.set( + 'x-goog-encryption-key-sha256', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (newFile as any).encryptionKeyHash || '', + ); } else if (options.destinationKmsKeyName !== undefined) { query.destinationKmsKeyName = options.destinationKmsKeyName; delete options.destinationKmsKeyName; } else if (newFile.kmsKeyName !== undefined) { query.destinationKmsKeyName = newFile.kmsKeyName; } + headers.set('Content-Type', 'application/json'); if (query.destinationKmsKeyName) { this.kmsKeyName = query.destinationKmsKeyName; @@ -1399,7 +1409,7 @@ class File extends ServiceObject { .makeRequest( { method: 'POST', - url: `/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/rewriteTo/b/${ + url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/rewriteTo/b/${ destBucket.name }/o/${encodeURIComponent(newFile.name)}`, queryParameters: query as unknown as StorageQueryParameters, @@ -1596,6 +1606,8 @@ class File extends ServiceObject { } const headers = response.headers; + const isStoredCompressed = + headers.get('x-goog-stored-content-encoding') === 'gzip'; const isCompressed = headers.get('content-encoding') === 'gzip'; const hashes: {crc32c?: string; md5?: string} = {}; @@ -1609,7 +1621,7 @@ class File extends ServiceObject { const transformStreams: Transform[] = []; - if (shouldRunValidation) { + if (shouldRunValidation && !isStoredCompressed) { // The x-goog-hash header should be set with a crc32c and md5 hash. // ex: headers.set('x-goog-hash', 'crc32c=xxxx,md5=xxxx') if (typeof headers.get('x-goog-hash') === 'string') { @@ -1678,8 +1690,13 @@ class File extends ServiceObject { const headers = { 'Accept-Encoding': 'gzip', 'Cache-Control': 'no-store', + ...(this.encryptionKeyHeaders || {}), } as Headers; + if (options.decompress === false) { + headers['Accept-Encoding'] = 'gzip'; + } + if (rangeRequest) { const start = typeof options.start === 'number' ? options.start : '0'; const end = typeof options.end === 'number' ? options.end : ''; @@ -1688,11 +1705,13 @@ class File extends ServiceObject { } const reqOpts: StorageRequestOptions = { - url: `${this.bucket.baseUrl}/${this.bucket.name}${this.baseUrl}/${this.name}`, + url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`, headers, queryParameters: query as unknown as StorageQueryParameters, responseType: 'stream', - }; + decompress: options.decompress, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; if (options[GCCL_GCS_CMD_KEY]) { reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY]; @@ -2368,6 +2387,18 @@ class File extends ServiceObject { } } + get encryptionKeyHeaders(): Record | undefined { + if (!this.encryptionKey) { + return undefined; + } + + return { + 'x-goog-encryption-algorithm': 'AES256', + 'x-goog-encryption-key': this.encryptionKey.toString('base64'), + 'x-goog-encryption-key-sha256': this.encryptionKeyHash!, + }; + } + /** * The Storage API allows you to use a custom key for server-side encryption. * @@ -3252,39 +3283,34 @@ class File extends ServiceObject { */ isPublic(callback?: IsPublicCallback): Promise | void { - // Build any custom headers based on the defined interceptors on the parent - // storage object and this object - const storageInterceptors = this.storage?.interceptors || []; - const fileInterceptors = this.interceptors || []; - const allInterceptors = storageInterceptors.concat(fileInterceptors); - - for (const curInter of allInterceptors) { - gaxios.instance.interceptors.request.add(curInter); - } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const {callback: cb} = normalize( + undefined, + callback, + ); + + const url = `/${this.bucket.name}/${encodeURIComponent(this.name)}`; + this.storageTransport + .makeRequest( + { + method: 'GET', + url: url, + }, + err => { + if (!err) { + cb(null, true); + return; + } - gaxios - .request({ - method: 'GET', - url: `${this.storage.apiEndpoint}/${ - this.bucket.name - }/${encodeURIComponent(this.name)}`, - retryConfig: { - retry: this.storage.retryOptions.maxRetries, - noResponseRetries: this.storage.retryOptions.maxRetries, - maxRetryDelay: this.storage.retryOptions.maxRetryDelay, - retryDelayMultiplier: this.storage.retryOptions.retryDelayMultiplier, - shouldRetry: this.storage.retryOptions.retryableErrorFn, - totalTimeout: this.storage.retryOptions.totalTimeout, + const status = err.response?.status; + if (status === 401 || status === 403) { + cb(null, false); + return; + } + cb(err); }, - }) - .then(() => callback!(null, true)) - .catch(err => { - if (err.status === 403) { - callback!(null, false); - } else { - callback!(err); - } - }); + ) + .catch(err => cb(err)); } makePrivate( @@ -3630,7 +3656,7 @@ class File extends ServiceObject { .makeRequest( { method: 'POST', - url: `/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/moveTo/o/${encodeURIComponent(newFile.name)}`, + url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/moveTo/o/${encodeURIComponent(newFile.name)}`, queryParameters: query as StorageQueryParameters, body: JSON.stringify(options), }, @@ -3961,7 +3987,7 @@ class File extends ServiceObject { async restore(options: RestoreOptions): Promise { const file = await this.storageTransport.makeRequest({ method: 'POST', - url: `/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/restore`, + url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/restore`, queryParameters: options as unknown as StorageQueryParameters, }); return file as File; @@ -4488,6 +4514,14 @@ class File extends ServiceObject { }, ]; + const headers: Record = {}; + if (this.encryptionKey) { + headers['x-goog-encryption-algorithm'] = 'AES256'; + headers['x-goog-encryption-key'] = this.encryptionKeyBase64!; + headers['x-goog-encryption-key-sha256'] = this.encryptionKeyHash!; + } + reqOpts.headers = headers; + this.storageTransport .makeRequest(reqOpts as StorageRequestOptions, (err, body, resp) => { if (err) { diff --git a/handwritten/storage/src/hmacKey.ts b/handwritten/storage/src/hmacKey.ts index 538264c0a497..52996219054f 100644 --- a/handwritten/storage/src/hmacKey.ts +++ b/handwritten/storage/src/hmacKey.ts @@ -354,7 +354,7 @@ export class HmacKey extends ServiceObject { storageTransport: storage.storageTransport, parent: storage, id: accessId, - baseUrl: `/projects/${projectId}/hmacKeys`, + baseUrl: `/storage/v1/projects/${projectId}/hmacKeys`, methods, }); diff --git a/handwritten/storage/src/iam.ts b/handwritten/storage/src/iam.ts index 45d7f17c2d07..61d9f340a3da 100644 --- a/handwritten/storage/src/iam.ts +++ b/handwritten/storage/src/iam.ts @@ -141,11 +141,11 @@ export enum IAMExceptionMessages { * ``` */ class Iam { - private resourceId_: string; + private bucket: Bucket; private storageTransport: StorageTransport; constructor(bucket: Bucket) { - this.resourceId_ = 'buckets/' + bucket.getId(); + this.bucket = bucket; this.storageTransport = bucket.storageTransport; } @@ -261,7 +261,8 @@ class Iam { this.storageTransport .makeRequest( { - url: '/iam', + method: 'GET', + url: `/storage/v1/b/${this.bucket.name}/iam`, queryParameters: qs as unknown as StorageQueryParameters, }, (err, data, resp) => { @@ -358,16 +359,10 @@ class Iam { .makeRequest( { method: 'PUT', - url: '/iam', + url: `/storage/v1/b/${this.bucket.name}/iam`, maxRetries, - body: JSON.stringify( - Object.assign( - { - resourceId: this.resourceId_, - }, - policy, - ), - ), + body: JSON.stringify(policy), + headers: {'Content-Type': 'application/json'}, queryParameters: options as unknown as StorageQueryParameters, }, (err, data, resp) => { @@ -467,17 +462,18 @@ class Iam { ? permissions : [permissions]; - const req = Object.assign( - { - permissions: permissionsArray, - }, - options, - ); + const req: any = { + permissions: permissionsArray, + }; + if (options.userProject) { + req.userProject = options.userProject; + } this.storageTransport .makeRequest( { - url: '/iam/testPermissions', + method: 'GET', + url: `/storage/v1/b/${this.bucket.name}/iam/testPermissions`, queryParameters: req as unknown as StorageQueryParameters, }, (err, data, resp) => { diff --git a/handwritten/storage/src/nodejs-common/service-object.ts b/handwritten/storage/src/nodejs-common/service-object.ts index 80ed207764d8..f37e41a47b1c 100644 --- a/handwritten/storage/src/nodejs-common/service-object.ts +++ b/handwritten/storage/src/nodejs-common/service-object.ts @@ -444,6 +444,20 @@ class ServiceObject extends EventEmitter { url = `${this.parent.baseUrl}/${this.parent.id}${url}`; } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const encryptionHeaders = (this as any).encryptionKeyHeaders || {}; + + const headers = { + ...encryptionHeaders, + ...methodConfig.reqOpts?.headers, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...(options as any).headers, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const query = {...options} as any; + delete query.headers; + this.storageTransport .makeRequest( { @@ -451,9 +465,10 @@ class ServiceObject extends EventEmitter { responseType: 'json', url, ...methodConfig.reqOpts, + headers, queryParameters: { ...methodConfig.reqOpts?.queryParameters, - ...options, + ...query, }, }, (err, data, resp) => { diff --git a/handwritten/storage/src/resumable-upload.ts b/handwritten/storage/src/resumable-upload.ts index d6ec8df5ed83..6e0c4c182c3c 100644 --- a/handwritten/storage/src/resumable-upload.ts +++ b/handwritten/storage/src/resumable-upload.ts @@ -1208,6 +1208,8 @@ export class Upload extends Writable { code: resp.status.toString(), message: resp.statusText, name: resp.statusText, + config: resp.config, + response: resp, } as GaxiosError) ) { void this.attemptDelayedRetry(resp); diff --git a/handwritten/storage/src/storage-transport.ts b/handwritten/storage/src/storage-transport.ts index 43070a73ff5e..653b4e6759d2 100644 --- a/handwritten/storage/src/storage-transport.ts +++ b/handwritten/storage/src/storage-transport.ts @@ -25,13 +25,13 @@ import { getModuleFormat, getRuntimeTrackingString, getUserAgentString, -} from './util'; +} from './util.js'; import {randomUUID} from 'crypto'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore import {getPackageJSON} from './package-json-helper.cjs'; -import {GCCL_GCS_CMD_KEY} from './nodejs-common/util'; -import {RetryOptions} from './storage'; +import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js'; +import {RETRYABLE_ERR_FN_DEFAULT, RetryOptions} from './storage.js'; export interface StandardStorageQueryParams { alt?: 'json' | 'media'; @@ -87,7 +87,6 @@ export interface StorageTransportCallback { fullResponse?: GaxiosResponse, ): void; } -let projectId: string; export class StorageTransport { authClient: GoogleAuth; @@ -113,7 +112,11 @@ export class StorageTransport { } this.providedUserAgent = options.userAgent; this.packageJson = getPackageJSON(); - this.retryOptions = options.retryOptions; + this.retryOptions = { + ...options.retryOptions, + retryableErrorFn: + options.retryOptions?.retryableErrorFn || RETRYABLE_ERR_FN_DEFAULT, + }; this.baseUrl = options.baseUrl; this.timeout = options.timeout; this.projectId = options.projectId; @@ -124,13 +127,16 @@ export class StorageTransport { reqOpts: StorageRequestOptions, callback?: StorageTransportCallback, ): Promise { - const headers = this.#buildRequestHeaders(reqOpts.headers); - if (reqOpts[GCCL_GCS_CMD_KEY]) { - headers.set( - 'x-goog-api-client', - `${headers.get('x-goog-api-client')} gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`, - ); + // Project ID Resolution + if (!this.projectId) { + this.projectId = + reqOpts.projectId || (await this.authClient.getProjectId()); } + + // Header Construction + const headers = this.#prepareHeaders(reqOpts); + + // Interceptor Management if (reqOpts.interceptors) { this.gaxiosInstance.interceptors.request.clear(); for (const inter of reqOpts.interceptors) { @@ -138,62 +144,85 @@ export class StorageTransport { } } - try { - const getProjectId = async () => { - if (reqOpts.projectId) return reqOpts.projectId; - projectId = await this.authClient.getProjectId(); - return projectId; - }; - const _projectId = await getProjectId(); - if (_projectId) { - projectId = _projectId; - this.projectId = projectId; - } + const urlString = reqOpts.url?.toString() || ''; + const isAbsolute = this.#isValidUrl(urlString); + + // Determine the base URL for the request + const requestUrl = isAbsolute + ? urlString + : new URL(urlString, this.baseUrl).toString(); + try { const requestPromise = this.authClient.request({ retryConfig: { retry: this.retryOptions.maxRetries, noResponseRetries: this.retryOptions.maxRetries, maxRetryDelay: this.retryOptions.maxRetryDelay, retryDelayMultiplier: this.retryOptions.retryDelayMultiplier, - shouldRetry: this.retryOptions.retryableErrorFn, totalTimeout: this.retryOptions.totalTimeout, + shouldRetry: err => !!this.retryOptions.retryableErrorFn?.(err), }, ...reqOpts, + data: reqOpts.body, + params: reqOpts.queryParameters, + paramsSerializer: this.#paramsSerializer, headers, - url: this.#buildUrl(reqOpts.url?.toString(), reqOpts.queryParameters), + url: requestUrl, timeout: this.timeout, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...({decompress: false} as any), + validateStatus: status => { + const isResumable = + reqOpts.queryParameters?.uploadType === 'resumable' || + reqOpts.url?.toString().includes('uploadType=resumable'); + + return ( + (status >= 200 && status < 300) || (isResumable && status === 308) + ); + }, }); - return callback - ? requestPromise - .then(resp => callback(null, resp.data, resp)) - .catch(err => callback(err, null, err.response)) - : (requestPromise.then(resp => resp.data) as Promise); + // Response Handling + const responseHandler = (resp: GaxiosResponse) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const data = resp.data as any; + if (data && typeof data === 'object') { + data.headers = resp.headers; + data.status = resp.status; + } + return data || resp; + }; + + if (callback) { + requestPromise + .then(resp => callback(null, responseHandler(resp), resp)) + .catch(err => callback(err, null, err.response)); + return; + } + + return requestPromise.then(responseHandler); } catch (e) { if (callback) return callback(e as GaxiosError); throw e; } } - #buildUrl(pathUri = '', queryParameters: StorageQueryParameters = {}): URL { - if ( - 'project' in queryParameters && - (queryParameters.project !== this.projectId || - queryParameters.project !== projectId) - ) { - queryParameters.project = this.projectId; - } - const qp = this.#buildRequestQueryParams(queryParameters); - let url: URL; - if (this.#isValidUrl(pathUri)) { - url = new URL(pathUri); - } else { - url = new URL(`${this.baseUrl}${pathUri}`); + #prepareHeaders(reqOpts: StorageRequestOptions): Record { + const headersObj = this.#buildRequestHeaders(reqOpts.headers); + + if (reqOpts[GCCL_GCS_CMD_KEY]) { + const current = headersObj.get('x-goog-api-client') || ''; + headersObj.set( + 'x-goog-api-client', + `${current} gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`, + ); } - url.search = qp; - return url; + const finalHeaders: Record = {}; + headersObj.forEach((v, k) => { + finalHeaders[k] = v; + }); + return finalHeaders; } #isValidUrl(url: string): boolean { @@ -204,32 +233,38 @@ export class StorageTransport { } } + /** + * Serializes query parameters into a string. + * Specifically handles arrays by appending each value individually + * to satisfy GCS "repeated key" requirements (e.g., for IAM permissions). + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + #paramsSerializer = (params: Record): string => { + const searchParams = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value === undefined) continue; + + if (Array.isArray(value)) { + value.forEach(v => searchParams.append(key, String(v))); + } else { + searchParams.set(key, String(value)); + } + } + return searchParams.toString(); + }; + #buildRequestHeaders(requestHeaders = {}) { const headers = new Headers(requestHeaders); - headers.set('User-Agent', this.#getUserAgentString()); headers.set( 'x-goog-api-client', `${getRuntimeTrackingString()} gccl/${this.packageJson.version}-${getModuleFormat()} gccl-invocation-id/${randomUUID()}`, ); - return headers; } - #buildRequestQueryParams(queryParameters: StorageQueryParameters): string { - const qp = new URLSearchParams( - queryParameters as unknown as Record, - ); - - return qp.toString(); - } - #getUserAgentString(): string { - let userAgent = getUserAgentString(); - if (this.providedUserAgent) { - userAgent = `${this.providedUserAgent} ${userAgent}`; - } - - return userAgent; + const base = getUserAgentString(); + return this.providedUserAgent ? `${this.providedUserAgent} ${base}` : base; } } diff --git a/handwritten/storage/src/storage.ts b/handwritten/storage/src/storage.ts index 55f20ed846cf..3d32d1ecd337 100644 --- a/handwritten/storage/src/storage.ts +++ b/handwritten/storage/src/storage.ts @@ -303,38 +303,100 @@ const IDEMPOTENCY_STRATEGY_DEFAULT = IdempotencyStrategy.RetryConditional; * @return {boolean} True if the API request should be retried, false otherwise. */ export const RETRYABLE_ERR_FN_DEFAULT = function (err?: GaxiosError) { - const isConnectionProblem = (reason: string) => { - return ( - reason.includes('eai_again') || // DNS lookup error - reason === 'econnreset' || - reason === 'unexpected connection closure' || - reason === 'epipe' || - reason === 'socket connection timeout' - ); - }; - - if (err) { - if ([408, 429, 500, 502, 503, 504].indexOf(err.status!) !== -1) { - return true; + if (!err || !err.config) return false; + + const config = err.config; + const method = (config.method || 'GET').toUpperCase(); + const url = config.url ? config.url.toString() : ''; + const params = config.params || {}; + const data = config.data; + + // Immediate exit for non-retryable status codes + const status = err.response?.status; + if (status && [401, 405, 412].includes(status)) return false; + + // Optimized Precondition Check + let bodyEtag = false; + try { + const parsedBody = typeof data === 'string' ? JSON.parse(data) : data; + if (parsedBody && parsedBody.etag) { + bodyEtag = true; } + } catch (e) { + // If parsing fails, we treat it as no etag and move on + bodyEtag = false; + } - if (typeof err.code === 'string') { - if (['408', '429', '500', '502', '503', '504'].indexOf(err.code) !== -1) { - return true; - } - const reason = (err.code as string).toLowerCase(); - if (isConnectionProblem(reason)) { - return true; - } + const hasPrecondition = !!( + params.ifGenerationMatch !== undefined || + params.ifMetagenerationMatch !== undefined || + params.ifSourceGenerationMatch !== undefined || + bodyEtag + ); + + // Granular Idempotency Logic + let isIdempotent = false; + if (['GET', 'HEAD'].includes(method) || hasPrecondition) { + isIdempotent = true; + } else if (method === 'PUT') { + // Resumable uploads (upload_id) are idempotent. + // IAM/HMAC are only idempotent if they have an etag (handled in hasPrecondition). + const isResumable = url.includes('upload_id='); + const isSpecialMutation = + url.includes('/iam') || url.includes('/hmacKeys/'); + isIdempotent = isResumable || !isSpecialMutation; + } else if (method === 'DELETE') { + // Deleting a specific object is only idempotent with a precondition. + // Deleting a bucket/HMAC is generally safe to retry. + if (!url.includes('/o/')) { + isIdempotent = true; } + } else if (method === 'POST') { + // Bucket creation is safe to retry. + // Object mutations (rewrite/copy) must have a precondition (handled above). + isIdempotent = url.includes('/v1/b') && !url.includes('/o'); + } + if (!isIdempotent) return false; + + // Unified Error Detection + const retryableCodes = [408, 429, 500, 502, 503, 504]; + const errCode = err.code?.toString().toUpperCase() || ''; + const message = err.message?.toLowerCase() || ''; + + // Check HTTP Status + if (status && retryableCodes.includes(status)) return true; + + // Check Gaxios/Node Error Codes + if (retryableCodes.includes(Number(errCode))) return true; + + const connectionErrors = [ + 'ECONNRESET', + 'EPIPE', + 'ETIMEDOUT', + 'EADDRINUSE', + 'ECONNREFUSED', + 'ENOTFOUND', + 'ENETUNREACH', + 'EAI_AGAIN', + ]; + + if ( + connectionErrors.includes(errCode) || + message.includes('socket hang up') + ) { + return true; + } - if (err) { - const reason = err?.code?.toString().toLowerCase(); - if (reason && isConnectionProblem(reason)) { - return true; - } - } + // Handle malformed responses or stream interruptions + if ( + message.includes('unexpected end of json input') || + message.includes('unexpected token') || + message.includes('operation was aborted') || + message.includes('unexpected connection closure') + ) { + return true; } + return false; }; @@ -1097,7 +1159,7 @@ export class Storage { method: 'POST', queryParameters: query, body: JSON.stringify(body), - url: '/b', + url: '/storage/v1/b', responseType: 'json', headers: { 'Content-Type': 'application/json', @@ -1224,7 +1286,7 @@ export class Storage { .makeRequest( { method: 'POST', - url: `/projects/${projectId}/hmacKeys`, + url: `/storage/v1/projects/${projectId}/hmacKeys`, queryParameters: query as unknown as StorageQueryParameters, retry: false, responseType: 'json', @@ -1359,7 +1421,7 @@ export class Storage { items: BucketMetadata[]; }>( { - url: '/b', + url: '/storage/v1/b', method: 'GET', queryParameters: options as unknown as StorageQueryParameters, responseType: 'json', @@ -1488,7 +1550,7 @@ export class Storage { items: HmacKeyMetadata[]; }>( { - url: `/projects/${projectId}/hmacKeys`, + url: `/storage/v1/projects/${projectId}/hmacKeys`, responseType: 'json', queryParameters: query as unknown as StorageQueryParameters, method: 'GET', @@ -1589,8 +1651,8 @@ export class Storage { .makeRequest( { method: 'GET', - url: `/projects/${this.projectId}/serviceAccount`, - queryParameters: options as unknown as StorageQueryParameters, + url: `/storage/v1/projects/${this.projectId}/serviceAccount`, + queryParameters: (options || {}) as StorageQueryParameters, responseType: 'json', }, (err, data, resp) => { diff --git a/handwritten/storage/test/acl.ts b/handwritten/storage/test/acl.ts index 2cf6c47b3885..922d05d313ba 100644 --- a/handwritten/storage/test/acl.ts +++ b/handwritten/storage/test/acl.ts @@ -62,7 +62,7 @@ describe('storage/acl', () => { it('should make the correct api request', () => { acl.storageTransport.makeRequest = sandbox.stub().callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'POST'); - assert.strictEqual(reqOpts.url, '/bucket/acl'); + assert.strictEqual(reqOpts.url, '/storage/v1/b/bucket/acl'); assert.deepStrictEqual(JSON.parse(reqOpts.body), { entity: ENTITY, role: ROLE, @@ -166,7 +166,10 @@ describe('storage/acl', () => { it('should make the correct api request', () => { acl.storageTransport.makeRequest = sandbox.stub().callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'DELETE'); - assert.strictEqual(reqOpts.url, `/bucket/acl/${ENTITY}`); + assert.strictEqual( + reqOpts.url, + `/storage/v1/b/bucket/acl/${encodeURIComponent(ENTITY)}`, + ); return Promise.resolve(); }); @@ -240,7 +243,7 @@ describe('storage/acl', () => { describe('all ACL objects', () => { it('should make the correct API request', () => { acl.storageTransport.makeRequest = sandbox.stub().callsFake(reqOpts => { - assert.strictEqual(reqOpts.url, '/bucket/acl'); + assert.strictEqual(reqOpts.url, '/storage/v1/b/bucket/acl'); return Promise.resolve(); }); @@ -295,7 +298,10 @@ describe('storage/acl', () => { describe('ACL object for an entity', () => { it('should get a specific ACL object', () => { acl.storageTransport.makeRequest = sandbox.stub().callsFake(reqOpts => { - assert.strictEqual(reqOpts.url, `/bucket/acl/${ENTITY}`); + assert.strictEqual( + reqOpts.url, + `/storage/v1/b/bucket/acl/${encodeURIComponent(ENTITY)}`, + ); return Promise.resolve(); }); @@ -408,7 +414,10 @@ describe('storage/acl', () => { it('should make the correct API request', () => { acl.storageTransport.makeRequest = sandbox.stub().callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'PUT'); - assert.strictEqual(reqOpts.url, `/bucket/acl/${ENTITY}`); + assert.strictEqual( + reqOpts.url, + `/storage/v1/b/bucket/acl/${encodeURIComponent(ENTITY)}`, + ); assert.deepStrictEqual(JSON.parse(reqOpts.body), {role: ROLE}); return Promise.resolve(); }); diff --git a/handwritten/storage/test/bucket.ts b/handwritten/storage/test/bucket.ts index 42c93ae4a6fe..be1c6849d70c 100644 --- a/handwritten/storage/test/bucket.ts +++ b/handwritten/storage/test/bucket.ts @@ -99,7 +99,7 @@ describe('Bucket', () => { .stub() .callsFake((reqOpts, callback) => { assert.strictEqual(reqOpts.method, 'POST'); - assert.strictEqual(reqOpts.url, '/b'); + assert.strictEqual(reqOpts.url, '/storage/v1/b'); assert.deepStrictEqual( reqOpts.queryParameters!.userProject, options.userProject, @@ -127,7 +127,7 @@ describe('Bucket', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'DELETE'); - assert.strictEqual(reqOpts.url, `/b/${BUCKET_NAME}`); + assert.strictEqual(reqOpts.url, `/storage/v1/b/${BUCKET_NAME}`); assert.deepStrictEqual( reqOpts.queryParameters!.userProject, options.userProject, @@ -156,7 +156,7 @@ describe('Bucket', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'GET'); - assert.strictEqual(reqOpts.url, `/b/${BUCKET_NAME}`); + assert.strictEqual(reqOpts.url, `/storage/v1/b/${BUCKET_NAME}`); assert.deepStrictEqual( reqOpts.queryParameters!.userProject, options.userProject, @@ -185,7 +185,7 @@ describe('Bucket', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'GET'); - assert.strictEqual(reqOpts.url, `/b/${BUCKET_NAME}`); + assert.strictEqual(reqOpts.url, `/storage/v1/b/${BUCKET_NAME}`); assert.deepStrictEqual( reqOpts.queryParameters!.userProject, options.userProject, @@ -214,7 +214,7 @@ describe('Bucket', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'GET'); - assert.strictEqual(reqOpts.url, `/b/${BUCKET_NAME}`); + assert.strictEqual(reqOpts.url, `/storage/v1/b/${BUCKET_NAME}`); assert.deepStrictEqual( reqOpts.queryParameters!.userProject, options.userProject, @@ -247,7 +247,7 @@ describe('Bucket', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'PATCH'); - assert.strictEqual(reqOpts.url, `/b/${BUCKET_NAME}`); + assert.strictEqual(reqOpts.url, `/storage/v1/b/${BUCKET_NAME}`); assert.deepStrictEqual( reqOpts.queryParameters!.versioning, options.versioning, @@ -570,7 +570,10 @@ describe('Bucket', () => { .callsFake(reqOpts => { const body = JSON.parse(reqOpts.body); assert.strictEqual(reqOpts.method, 'POST'); - assert.strictEqual(reqOpts.url, '/compose'); + assert.strictEqual( + reqOpts.url, + '/storage/v1/b/test-bucket/o/destination.txt/compose', + ); assert.strictEqual(body.sourceObjects[0].name, file1.name); assert.strictEqual(body.sourceObjects[1].name, file2.name); done(); @@ -640,7 +643,10 @@ describe('Bucket', () => { storageTransport.makeRequest = sandbox.stub().callsFake(reqOpts => { const body = JSON.parse(reqOpts.body); - assert.strictEqual(reqOpts.url, '/compose'); + assert.strictEqual( + reqOpts.url, + '/storage/v1/b/test-bucket/o/destination.foo/compose', + ); assert.deepStrictEqual(body, { destination: {}, sourceObjects: [{name: sources[0].name}, {name: sources[1].name}], @@ -827,7 +833,10 @@ describe('Bucket', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'POST'); - assert.strictEqual(reqOpts.url, `/b/${BUCKET_NAME}/o/watch`); + assert.strictEqual( + reqOpts.url, + `/storage/v1/b/${BUCKET_NAME}/o/watch`, + ); const expectedJson = Object.assign({}, config, { id: ID, @@ -955,7 +964,7 @@ describe('Bucket', () => { assert.strictEqual(reqOpts.method, 'POST'); assert.strictEqual( reqOpts.url, - `/b/${BUCKET_NAME}/notificationConfigs`, + `/storage/v1/b/${BUCKET_NAME}/notificationConfigs`, ); assert.deepStrictEqual(JSON.parse(reqOpts.body), expectedJson); assert.notStrictEqual(reqOpts.body, options); @@ -1644,7 +1653,7 @@ describe('Bucket', () => { bucket.storageTransport.makeRequest = sandbox .stub() .callsFake(reqOpts => { - assert.strictEqual(reqOpts.url, `/b/${BUCKET_NAME}/o`); + assert.strictEqual(reqOpts.url, `/storage/v1/b/${BUCKET_NAME}/o`); assert.deepStrictEqual(reqOpts.queryParameters, {}); }); @@ -1917,7 +1926,7 @@ describe('Bucket', () => { .callsFake(reqOpts => { assert.strictEqual( reqOpts.url, - `/b/${BUCKET_NAME}/notificationConfigs`, + `/storage/v1/b/${BUCKET_NAME}/notificationConfigs`, ); assert.strictEqual(reqOpts.queryParameters, options); done(); @@ -2072,7 +2081,7 @@ describe('Bucket', () => { .callsFake((reqOpts, callback) => { assert.deepStrictEqual(reqOpts, { method: 'POST', - url: `/b/${BUCKET_NAME}/lockRetentionPolicy`, + url: `/storage/v1/b/${BUCKET_NAME}/lockRetentionPolicy`, queryParameters: { ifMetagenerationMatch: metageneration, }, @@ -2293,7 +2302,7 @@ describe('Bucket', () => { .callsFake(reqOpts => { assert.deepStrictEqual(reqOpts, { method: 'POST', - url: `/b/${BUCKET_NAME}/restore`, + url: `/storage/v1/b/${BUCKET_NAME}/restore`, queryParameters: {generation: '123456789'}, }); return []; diff --git a/handwritten/storage/test/channel.ts b/handwritten/storage/test/channel.ts index 0e9f80349571..90f2813cfbfa 100644 --- a/handwritten/storage/test/channel.ts +++ b/handwritten/storage/test/channel.ts @@ -62,7 +62,7 @@ describe('Channel', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'POST'); - assert.strictEqual(reqOpts.url, '/channels/stop'); + assert.strictEqual(reqOpts.url, '/storage/v1/channels/stop'); assert.deepStrictEqual(JSON.parse(reqOpts.body), channel.metadata); return Promise.resolve(); diff --git a/handwritten/storage/test/file.ts b/handwritten/storage/test/file.ts index ca58296702bb..f6e664c138d8 100644 --- a/handwritten/storage/test/file.ts +++ b/handwritten/storage/test/file.ts @@ -156,7 +156,10 @@ describe('File', () => { .stub() .callsFake(reqOpts => { assert.strictEqual(reqOpts.method, 'DELETE'); - assert.strictEqual(reqOpts.url, '/b/bucket-name/o/file-name.png'); + assert.strictEqual( + reqOpts.url, + '/storage/v1/b/bucket-name/o/file-name.png', + ); assert.deepStrictEqual( reqOpts.queryParameters.generation, options.generation, @@ -214,7 +217,10 @@ describe('File', () => { .stub() .callsFake((reqOpts, callback) => { assert.strictEqual(reqOpts.method, 'GET'); - assert.strictEqual(reqOpts.url, '/b/bucket-name/o/file-name.png'); + assert.strictEqual( + reqOpts.url, + '/storage/v1/b/bucket-name/o/file-name.png', + ); assert.deepStrictEqual( reqOpts.queryParameters.generation, options.generation, @@ -272,7 +278,10 @@ describe('File', () => { .stub() .callsFake((reqOpts, callback) => { assert.strictEqual(reqOpts.method, 'GET'); - assert.strictEqual(reqOpts.url, '/b/bucket-name/o/file-name.png'); + assert.strictEqual( + reqOpts.url, + '/storage/v1/b/bucket-name/o/file-name.png', + ); assert.deepStrictEqual( reqOpts.queryParameters.generation, options.generation, @@ -330,7 +339,10 @@ describe('File', () => { .stub() .callsFake((reqOpts, callback) => { assert.strictEqual(reqOpts.method, 'GET'); - assert.strictEqual(reqOpts.url, '/b/bucket-name/o/file-name.png'); + assert.strictEqual( + reqOpts.url, + '/storage/v1/b/bucket-name/o/file-name.png', + ); assert.deepStrictEqual( reqOpts.queryParameters.generation, options.generation, @@ -382,7 +394,10 @@ describe('File', () => { .callsFake((reqOpts, callback) => { const body = JSON.parse(reqOpts.body); assert.strictEqual(reqOpts.method, 'PATCH'); - assert.strictEqual(reqOpts.url, '/b/bucket-name/o/file-name.png'); + assert.strictEqual( + reqOpts.url, + '/storage/v1/b/bucket-name/o/file-name.png', + ); assert.deepStrictEqual(body.temporaryHold, options.temporaryHold); callback(null); return Promise.resolve(); @@ -448,7 +463,7 @@ describe('File', () => { it('should URI encode file names', done => { const newFile = new File(BUCKET, 'nested/file.jpg'); - const expectedPath = `/b/${BUCKET.name}/o/${encodeURIComponent(directoryFile.name)}/rewriteTo/b/${ + const expectedPath = `/storage/v1/b/${BUCKET.name}/o/${encodeURIComponent(directoryFile.name)}/rewriteTo/b/${ file.bucket.name }/o/${encodeURIComponent(newFile.name)}`; @@ -553,6 +568,7 @@ describe('File', () => { assert.deepStrictEqual( Object.fromEntries((reqOpts.headers as Headers).entries()), { + 'content-type': 'application/json', 'x-goog-copy-source-encryption-algorithm': 'AES256', 'x-goog-copy-source-encryption-key': file.encryptionKeyBase64, 'x-goog-copy-source-encryption-key-sha256': file.encryptionKeyHash, @@ -566,19 +582,42 @@ describe('File', () => { it('should set encryption key on the new File instance', done => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - let file: any; - // eslint-disable-next-line prefer-const, @typescript-eslint/no-explicit-any - file = new (File as any)(BUCKET, FILE_NAME); + const file = new (File as any)(BUCKET, FILE_NAME); + Object.assign(file, { + encryptionKey: 'source-key', + encryptionKeyBase64: 'base64', + encryptionKeyHash: 'hash', + }); // eslint-disable-next-line @typescript-eslint/no-explicit-any const newFile = new (File as any)(BUCKET, 'new-file'); - newFile.encryptionKey = 'encryptionKey'; - - file.setEncryptionKey = sandbox.stub().callsFake(encryptionKey => { - assert.strictEqual(encryptionKey, newFile.encryptionKey); - done(); + Object.assign(newFile, { + encryptionKey: 'dest-key', + encryptionKeyBase64: 'base64-dest', + encryptionKeyHash: 'hash-dest', }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + storageTransport.makeRequest = async (reqOpts: any, callback: any) => { + const actualHeaders = Object.fromEntries(reqOpts.headers.entries()); + + try { + assert.deepStrictEqual(actualHeaders, { + 'content-type': 'application/json', + 'x-goog-copy-source-encryption-algorithm': 'AES256', + 'x-goog-copy-source-encryption-key': 'base64', + 'x-goog-copy-source-encryption-key-sha256': 'hash', + 'x-goog-encryption-algorithm': 'AES256', + 'x-goog-encryption-key': 'base64-dest', + 'x-goog-encryption-key-sha256': 'hash-dest', + }); + callback?.(null, {done: true}, {}); + done(); + } catch (e) { + done(e); + } + }; + file.copy(newFile, assert.ifError); }); @@ -685,7 +724,7 @@ describe('File', () => { it('should allow a string', done => { const newFileName = 'new-file-name.png'; const newFile = new File(BUCKET, newFileName); - const expectedPath = `/b/${BUCKET.name}/o/${encodeURIComponent(file.name)}/rewriteTo/b/${file.bucket.name}/o/${newFile.name}`; + const expectedPath = `/storage/v1/b/${BUCKET.name}/o/${encodeURIComponent(file.name)}/rewriteTo/b/${file.bucket.name}/o/${newFile.name}`; assertPathEquals(file, expectedPath, done); file.copy(newFileName, done); }); @@ -694,7 +733,7 @@ describe('File', () => { const newFileName = '/new-file-name.png'; const newFile = new File(BUCKET, newFileName); // File uri encodes file name when calling this.request during copy - const expectedPath = `/b/${BUCKET.name}/o/${encodeURIComponent(file.name)}/rewriteTo/b/${ + const expectedPath = `/storage/v1/b/${BUCKET.name}/o/${encodeURIComponent(file.name)}/rewriteTo/b/${ file.bucket.name }/o/${encodeURIComponent(newFile.name)}`; assertPathEquals(file, expectedPath, done); @@ -703,20 +742,20 @@ describe('File', () => { it('should allow a "gs://..." string', done => { const newFileName = 'gs://other-bucket/new-file-name.png'; - const expectedPath = `/b/${BUCKET.name}/o/${file.name}/rewriteTo/b/other-bucket/o/new-file-name.png`; + const expectedPath = `/storage/v1/b/${BUCKET.name}/o/${file.name}/rewriteTo/b/other-bucket/o/new-file-name.png`; assertPathEquals(file, expectedPath, done); file.copy(newFileName, done); }); it('should allow a Bucket', done => { - const expectedPath = `/b/${BUCKET.name}/o/${file.name}/rewriteTo/b/${BUCKET.name}/o/${file.name}`; + const expectedPath = `/storage/v1/b/${BUCKET.name}/o/${file.name}/rewriteTo/b/${BUCKET.name}/o/${file.name}`; assertPathEquals(file, expectedPath, done); file.copy(BUCKET, done); }); it('should allow a File', done => { const newFile = new File(BUCKET, 'new-file'); - const expectedPath = `/b/${BUCKET.name}/o/${file.name}/rewriteTo/b/${BUCKET.name}/o/${newFile.name}`; + const expectedPath = `/storage/v1/b/${BUCKET.name}/o/${file.name}/rewriteTo/b/${BUCKET.name}/o/${newFile.name}`; assertPathEquals(file, expectedPath, done); file.copy(newFile, done); }); @@ -963,11 +1002,12 @@ describe('File', () => { it('should create an authenticated request', () => { file.storageTransport.makeRequest = sandbox.stub().callsFake(opts => { assert.deepStrictEqual(opts, { - url: '/b/bucket-name/o/file-name.png', + url: '/storage/v1/b/bucket-name/o/file-name.png', headers: { 'Accept-Encoding': 'gzip', 'Cache-Control': 'no-store', }, + decompress: true, responseType: 'stream', queryParameters: { alt: 'media', @@ -3649,81 +3689,99 @@ describe('File', () => { }); describe('isPublic', () => { - it('should execute callback with `true` in response', () => { + it('should execute callback with `true` in response', done => { + file.storageTransport.makeRequest = sandbox + .stub() + .callsFake((reqOpts, callback) => { + callback(null, {}, {}); + return Promise.resolve(); + }); + file.isPublic((err, resp) => { assert.ifError(err); assert.strictEqual(resp, true); + done(); }); }); - it('should execute callback with `false` in response', () => { + it('should execute callback with `false` in response on 403', done => { file.storageTransport.makeRequest = sandbox .stub() - .callsFake((reqOpts, config, callback) => { + .callsFake((reqOpts, callback) => { const error = new GaxiosError( 'Permission Denied.', {} as GaxiosOptionsPrepared, ); - error.status = 403; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + error.response = {status: 403} as any; callback(error); + return Promise.resolve(); }); file.isPublic((err, resp) => { assert.ifError(err); assert.strictEqual(resp, false); + done(); }); }); - it('should propagate non-403 errors to user', () => { + it('should propagate non-403 errors to user', done => { const error = new GaxiosError('400 Error.', {} as GaxiosOptionsPrepared); - error.status = 400; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + error.response = {status: 400} as any; file.storageTransport.makeRequest = sandbox .stub() - .callsFake((reqOpts, config, callback) => { + .callsFake((reqOpts, callback) => { callback(error); + return Promise.resolve(); }); file.isPublic(err => { assert.strictEqual(err, error); + done(); }); }); it('should correctly send a GET request', () => { file.storageTransport.makeRequest = sandbox .stub() - .callsFake((reqOpts, config, callback) => { + .callsFake((reqOpts, callback) => { assert.strictEqual(reqOpts.method, 'GET'); callback(null); + return Promise.resolve(); }); file.isPublic(err => { assert.ifError(err); }); }); - it('should correctly format URL in the request', () => { + it('should correctly format URL in the request', done => { file = new File(BUCKET, 'my#file$.png'); - const expectedURL = `https://storage.googleapis.com/${ - BUCKET.name - }/${encodeURIComponent(file.name)}`; + const expectedPath = `/${BUCKET.name}/${encodeURIComponent(file.name)}`; file.storageTransport.makeRequest = sandbox .stub() - .callsFake((reqOpts, config, callback) => { - assert.strictEqual(reqOpts.uri, expectedURL); + .callsFake((reqOpts, callback) => { + assert.strictEqual(reqOpts.method, 'GET'); + assert.strictEqual(reqOpts.url, expectedPath); callback(null); + return Promise.resolve(); }); file.isPublic(err => { assert.ifError(err); + done(); }); }); - it('should not set any headers when there are no interceptors', () => { + it('should not set any headers when there are no interceptors', done => { file.storageTransport.makeRequest = sandbox .stub() - .callsFake((reqOpts, config, callback) => { - assert.deepStrictEqual(reqOpts.headers, {}); + .callsFake((reqOpts, callback) => { + assert.deepStrictEqual(reqOpts.headers, undefined); callback(null); + return Promise.resolve(); }); file.isPublic(err => { assert.ifError(err); + done(); }); }); }); @@ -3755,7 +3813,7 @@ describe('File', () => { it('should URI encode file names', async () => { const newFile = new File(BUCKET, 'nested/file.jpg'); - const expectedPath = `/b/${BUCKET.id}/o/${directoryFile.id}/moveTo/o/${encodeURIComponent(newFile.name)}`; + const expectedPath = `/storage/v1/b/${BUCKET.id}/o/${directoryFile.id}/moveTo/o/${encodeURIComponent(newFile.name)}`; directoryFile.storageTransport.makeRequest = sandbox .stub() @@ -3858,7 +3916,7 @@ describe('File', () => { it('should allow a string', async done => { const newFileName = 'new-file-name.png'; const newFile = new File(BUCKET, newFileName); - const expectedPath = `/b/${BUCKET.id}/o/${file.id}/moveTo/o/${newFile.name}`; + const expectedPath = `/storage/v1/b/${BUCKET.id}/o/${file.id}/moveTo/o/${newFile.name}`; assertPathEquals(file, expectedPath, done); await file.moveFileAtomic(newFileName); }); @@ -3866,21 +3924,21 @@ describe('File', () => { it('should allow a string with leading slash.', async done => { const newFileName = '/new-file-name.png'; const newFile = new File(BUCKET, newFileName); - const expectedPath = `/b/${BUCKET.id}/o/${file.id}/moveTo/o/${encodeURIComponent(newFile.name)}`; + const expectedPath = `/storage/v1/b/${BUCKET.id}/o/${file.id}/moveTo/o/${encodeURIComponent(newFile.name)}`; assertPathEquals(file, expectedPath, done); await file.moveFileAtomic(newFileName); }); it('should allow a "gs://..." string', async done => { const newFileName = 'gs://other-bucket/new-file-name.png'; - const expectedPath = `/b/${BUCKET.id}/o/${file.id}/moveTo/o/new-file-name.png`; + const expectedPath = `/storage/v1/b/${BUCKET.id}/o/${file.id}/moveTo/o/new-file-name.png`; assertPathEquals(file, expectedPath, done); await file.moveFileAtomic(newFileName); }); it('should allow a File', async done => { const newFile = new File(BUCKET, 'new-file'); - const expectedPath = `/b/${BUCKET.id}/o/${file.id}/moveTo/o/${newFile.name}`; + const expectedPath = `/storage/v1/b/${BUCKET.id}/o/${file.id}/moveTo/o/${newFile.name}`; assertPathEquals(file, expectedPath, done); await file.moveFileAtomic(newFile); }); @@ -4153,7 +4211,7 @@ describe('File', () => { .callsFake((reqOpts, callback_) => { assert.deepStrictEqual(reqOpts, { method: 'POST', - url: `/b/${file.bucket.name}/o/${encodeURIComponent(file.name)}/restore`, + url: `/storage/v1/b/${file.bucket.name}/o/${encodeURIComponent(file.name)}/restore`, queryParameters: {generation: 123}, }); assert.strictEqual(callback_, undefined); diff --git a/handwritten/storage/test/headers.ts b/handwritten/storage/test/headers.ts index a3632ea85438..a9826f933709 100644 --- a/handwritten/storage/test/headers.ts +++ b/handwritten/storage/test/headers.ts @@ -76,9 +76,18 @@ describe('headers', () => { it('populates x-goog-api-client header (node)', async () => { const bucket = storage.bucket('foo-bucket'); authClient.request = opts => { + let apiClientHeader: string | null = ''; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (typeof (opts.headers as any).get === 'function') { + apiClientHeader = (opts.headers as Headers).get('x-goog-api-client'); + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + apiClientHeader = (opts.headers as any)['x-goog-api-client']; + } assert.ok( /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - (opts.headers as Headers).get('x-goog-api-client')!, + apiClientHeader!, ), ); return Promise.resolve(gaxiosResponse); @@ -94,9 +103,18 @@ describe('headers', () => { it('populates x-goog-api-client header (deno)', async () => { const bucket = storage.bucket('foo-bucket'); authClient.request = opts => { + let apiClientHeader: string | null = ''; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (typeof (opts.headers as any).get === 'function') { + apiClientHeader = (opts.headers as Headers).get('x-goog-api-client'); + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + apiClientHeader = (opts.headers as any)['x-goog-api-client']; + } assert.ok( /^gl-deno\/0.00.0 gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - (opts.headers as Headers).get('x-goog-api-client')!, + apiClientHeader!, ), ); return Promise.resolve(gaxiosResponse); diff --git a/handwritten/storage/test/iam.ts b/handwritten/storage/test/iam.ts index ab2c6d9d0ccf..89d480785dc1 100644 --- a/handwritten/storage/test/iam.ts +++ b/handwritten/storage/test/iam.ts @@ -51,7 +51,8 @@ describe('storage/iam', () => { .stub() .callsFake((reqOpts, callback) => { assert.deepStrictEqual(reqOpts, { - url: '/iam', + method: 'GET', + url: `/storage/v1/b/${BUCKET_INSTANCE.name}/iam`, queryParameters: {}, }); callback(null); @@ -107,9 +108,12 @@ describe('storage/iam', () => { reqOpts.body = JSON.parse(reqOpts.body); assert.deepStrictEqual(reqOpts, { method: 'PUT', - url: '/iam', + url: `/storage/v1/b/${BUCKET_INSTANCE.name}/iam`, maxRetries: 0, - body: Object.assign(policy, {resourceId: `buckets/${id}`}), + headers: { + 'Content-Type': 'application/json', + }, + body: Object.assign(policy), queryParameters: {}, }); callback(null); @@ -147,7 +151,8 @@ describe('storage/iam', () => { .stub() .callsFake(reqOpts => { assert.deepStrictEqual(reqOpts, { - url: '/iam/testPermissions', + method: 'GET', + url: `/storage/v1/b/${BUCKET_INSTANCE.name}/iam/testPermissions`, queryParameters: { permissions: [permissions], }, diff --git a/handwritten/storage/test/index.ts b/handwritten/storage/test/index.ts index d15b9e710f6d..ac19bab68f00 100644 --- a/handwritten/storage/test/index.ts +++ b/handwritten/storage/test/index.ts @@ -187,8 +187,17 @@ describe('Storage', () => { const storage = new Storage({ projectId: PROJECT_ID, }); - const error = new GaxiosError('502 Error', {} as GaxiosOptionsPrepared); + const mockConfig = { + method: 'GET', + url: 'http://127.0.0.1/test', + params: {}, + headers: {}, + } as unknown as GaxiosOptionsPrepared; + + const error = new GaxiosError('502 Error', mockConfig); error.status = 502; + error.code = '502'; + assert.strictEqual(storage.retryOptions.retryableErrorFn!(error), true); }); @@ -225,8 +234,15 @@ describe('Storage', () => { const storage = new Storage({ projectId: PROJECT_ID, }); - const error = new GaxiosError('Broken pipe', {} as GaxiosOptionsPrepared); - error.code = 'Socket connection timeout'; + const mockConfig = { + method: 'GET', + url: 'http://127.0.0.1/test', + headers: {}, + } as unknown as GaxiosOptionsPrepared; + + const error = new GaxiosError('socket connection timeout', mockConfig); + + error.code = 'ETIMEDOUT'; assert.strictEqual(storage.retryOptions.retryableErrorFn!(error), true); }); @@ -497,7 +513,7 @@ describe('Storage', () => { assert.strictEqual(reqOpts.method, 'POST'); assert.strictEqual( reqOpts.url, - `/projects/${storage.projectId}/hmacKeys`, + `/storage/v1/projects/${storage.projectId}/hmacKeys`, ); assert.strictEqual( reqOpts.queryParameters!.serviceAccountEmail, @@ -625,7 +641,7 @@ describe('Storage', () => { .callsFake((reqOpts, callback) => { const body = JSON.parse(reqOpts.body); assert.strictEqual(reqOpts.method, 'POST'); - assert.strictEqual(reqOpts.url, '/b'); + assert.strictEqual(reqOpts.url, '/storage/v1/b'); assert.strictEqual( reqOpts.queryParameters!.project, storage.projectId, @@ -946,7 +962,7 @@ describe('Storage', () => { storage.storageTransport.makeRequest = sandbox .stub() .callsFake(reqOpts => { - assert.strictEqual(reqOpts.url, '/b'); + assert.strictEqual(reqOpts.url, '/storage/v1/b'); assert.deepStrictEqual(reqOpts.queryParameters, { project: storage.projectId, }); @@ -1073,7 +1089,10 @@ describe('Storage', () => { it('should get HmacKeys without a query', done => { storage.storageTransport.makeRequest = sandbox.stub().callsFake(opts => { - assert.strictEqual(opts.uri, `/projects/${storage.projectId}/hmacKeys`); + assert.strictEqual( + opts.uri, + `/storage/v1/projects/${storage.projectId}/hmacKeys`, + ); assert.deepStrictEqual(opts.queryParameters, {}); }); storage.getHmacKeys(() => { @@ -1090,7 +1109,10 @@ describe('Storage', () => { }; storage.storageTransport.makeRequest = sandbox.stub().callsFake(opts => { - assert.strictEqual(opts.url, `/projects/${storage.projectId}/hmacKeys`); + assert.strictEqual( + opts.url, + `/storage/v1/projects/${storage.projectId}/hmacKeys`, + ); assert.deepStrictEqual(opts.queryParameters, query); done(); }); @@ -1184,7 +1206,7 @@ describe('Storage', () => { .callsFake(reqOpts => { assert.strictEqual( reqOpts.url, - `/projects/${storage.projectId}/serviceAccount`, + `/storage/v1/projects/${storage.projectId}/serviceAccount`, ); assert.deepStrictEqual(reqOpts.queryParameters, {}); done(); @@ -1202,7 +1224,7 @@ describe('Storage', () => { storage.storageTransport.makeRequest = sandbox .stub() .callsFake(reqOpts => { - assert.strictEqual(reqOpts.queryParameters, options); + assert.deepStrictEqual(reqOpts.queryParameters, options); done(); }); diff --git a/handwritten/storage/test/resumable-upload.ts b/handwritten/storage/test/resumable-upload.ts index a8d09faa6b1d..dd2a6e7ddef0 100644 --- a/handwritten/storage/test/resumable-upload.ts +++ b/handwritten/storage/test/resumable-upload.ts @@ -59,7 +59,9 @@ function mockAuthorizeRequest( access_token: 'abc123', }, ) { - return nock('https://oauth2.googleapis.com').post('/token').reply(code, data); + return nock('https://oauth2.googleapis.com') + .post('/token', () => true) + .reply(code, data); } describe('resumable-upload', () => { @@ -1929,7 +1931,18 @@ describe('resumable-upload', () => { }); describe('500s', () => { - const RESP = {status: 500, data: 'error message from server'}; + const RESP = { + status: 500, + statusText: 'Internal Server Error', + data: 'error message from server', + config: { + method: 'GET', + url: `${BASE_URI}/${BUCKET}/o`, + params: { + ifGenerationMatch: 0, + }, + }, + }; it('should increase the retry count if less than limit', () => { up.getRetryDelay = () => 1; diff --git a/handwritten/storage/test/storage-transport.ts b/handwritten/storage/test/storage-transport.ts index 4b71c8fa9d66..92593a2f6f48 100644 --- a/handwritten/storage/test/storage-transport.ts +++ b/handwritten/storage/test/storage-transport.ts @@ -21,7 +21,7 @@ import {GoogleAuth} from 'google-auth-library'; import sinon from 'sinon'; import assert from 'assert'; import {GCCL_GCS_CMD_KEY} from '../src/nodejs-common/util'; -import {Gaxios} from 'gaxios'; +import {RETRYABLE_ERR_FN_DEFAULT} from '../src/storage'; describe('Storage Transport', () => { let sandbox: sinon.SinonSandbox; @@ -46,7 +46,7 @@ describe('Storage Transport', () => { retryDelayMultiplier: 2, maxRetryDelay: 100, totalTimeout: 1000, - retryableErrorFn: () => true, + retryableErrorFn: RETRYABLE_ERR_FN_DEFAULT, }, scopes: ['https://www.googleapis.com/auth/could-platform'], packageJson: {name: 'test-package', version: '1.0.0'}, @@ -58,7 +58,12 @@ describe('Storage Transport', () => { }); it('should make a request with the correct parameters', async () => { - const response = {data: {success: true}}; + const response = { + data: {success: true}, + headers: new Map(), + status: 200, + statusText: 'OK', + }; const requestStub = authClientStub.request as sinon.SinonStub; requestStub.resolves(response); @@ -71,20 +76,19 @@ describe('Storage Transport', () => { assert.strictEqual(requestStub.calledOnce, true); const calledWith = requestStub.getCall(0).args[0]; - assert.strictEqual( - calledWith.url.href, - `${baseUrl}/bucket/object?alt=json&userProject=user-project`, - ); - assert.strictEqual(calledWith.headers.get('content-encoding'), 'gzip'); - assert.ok( - calledWith.headers.get('User-Agent').includes('gcloud-node-storage/'), - ); + assert.strictEqual(calledWith.headers['content-encoding'], 'gzip'); + const headers = calledWith.headers; + const userAgent = headers['User-Agent'] || headers['user-agent']; + assert.ok(userAgent.includes('gcloud-node-storage/')); assert.deepStrictEqual(_response, response.data); }); it('should handle retry options correctly', async () => { const requestStub = authClientStub.request as sinon.SinonStub; - requestStub.resolves({}); + requestStub.resolves({ + data: {}, + headers: new Map(), + }); const reqOpts: StorageRequestOptions = { url: '/bucket/object', }; @@ -105,7 +109,10 @@ describe('Storage Transport', () => { [GCCL_GCS_CMD_KEY]: 'test-key', }; - (authClientStub.request as sinon.SinonStub).resolves({data: {}}); + (authClientStub.request as sinon.SinonStub).resolves({ + data: {}, + headers: new Map(), + }); await transport.makeRequest(reqOpts); @@ -113,35 +120,10 @@ describe('Storage Transport', () => { .args[0]; assert.ok( - calledWith.headers - .get('x-goog-api-client') - .includes('gccl-gcs-cmd/test-key'), + calledWith.headers['x-goog-api-client'].includes('gccl-gcs-cmd/test-key'), ); }); - // TODO: Undo this skip once the gaxios interceptor issue is resolved. - it.skip('should clear and add interceptors if provided', async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const interceptorStub: any = sandbox.stub(); - const reqOpts: StorageRequestOptions = { - url: '/bucket/object', - interceptors: [interceptorStub], - }; - - const clearStub = sandbox.stub(); - const addStub = sandbox.stub(); - (authClientStub.request as sinon.SinonStub).resolves({data: {}}); - const transportInstance = new Gaxios(); - transportInstance.interceptors.request.clear = clearStub; - transportInstance.interceptors.request.add = addStub; - - await transport.makeRequest(reqOpts); - - assert.strictEqual(clearStub.calledOnce, true); - assert.strictEqual(addStub.calledOnce, true); - assert.strictEqual(addStub.calledWith(interceptorStub), true); - }); - it('should initialize a new GoogleAuth instance when authClient is not an instance of GoogleAuth', async () => { const mockAuthClient = undefined; @@ -167,4 +149,207 @@ describe('Storage Transport', () => { const transport = new StorageTransport(options); assert.ok(transport.authClient instanceof GoogleAuth); }); + + it('should handle absolute URLs and project validation', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({url: 'https://my-custom-endpoint.com/v1/b'}); + assert.strictEqual( + requestStub.getCall(0).args[0].url, + 'https://my-custom-endpoint.com/v1/b', + ); + }); + + describe('Storage Transport shouldRetry logic', () => { + it('should retry POST if preconditions are present', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({ + method: 'POST', + url: '/b/bucket/o', + queryParameters: {ifGenerationMatch: 123}, + }); + + const retryConfig = requestStub.getCall(0).args[0].retryConfig; + const error503 = { + response: {status: 503}, + config: { + method: 'POST', + url: '/b/bucket/o', + params: {ifGenerationMatch: 123}, + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + assert.strictEqual(retryConfig.shouldRetry(error503), true); + }); + + it('should retry on malformed JSON responses (SyntaxError)', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({url: '/test'}); + + const retryConfig = requestStub.getCall(0).args[0].retryConfig; + + const malformedError = new Error( + 'Unexpected token < in JSON at position 0', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ) as any; + malformedError.stack = 'SyntaxError: Unexpected token <'; + malformedError.config = {method: 'GET', url: '/test'}; + + assert.strictEqual(retryConfig.shouldRetry(malformedError), true); + }); + + it('should retry on 503 for idempotent PUT requests', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({ + method: 'PUT', + url: '/bucket/object', + }); + + const retryConfig = requestStub.getCall(0).args[0].retryConfig; + + const error503 = { + response: {status: 503}, + config: {url: '/bucket/object'}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + assert.strictEqual(retryConfig.shouldRetry(error503), true); + }); + + it('should NOT retry on 401 Unauthorized', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({url: '/test'}); + + const retryConfig = requestStub.getCall(0).args[0].retryConfig; + + const error401 = { + response: {status: 401}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + assert.strictEqual(retryConfig.shouldRetry(error401), false); + }); + + it('should treat 308 as a valid status for resumable uploads', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: '308-metadata', headers: new Map()}); + + await transport.makeRequest({ + url: '/upload/storage/v1/b/bucket/o?uploadType=resumable', + queryParameters: {uploadType: 'resumable'}, + }); + + const callArgs = requestStub.getCall(0).args[0]; + + assert.strictEqual(callArgs.validateStatus(308), true); + }); + + it('should retry when GCS reason is rateLimitExceeded', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({url: '/test'}); + const retryConfig = requestStub.getCall(0).args[0].retryConfig; + + const rateLimitError = { + response: { + status: 429, + data: { + error: { + errors: [{reason: 'rateLimitExceeded'}], + }, + }, + }, + config: {method: 'GET', url: '/test'}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + assert.strictEqual(retryConfig.shouldRetry(rateLimitError), true); + }); + + it('should retry on transient network errors (no response)', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({url: '/test'}); + const retryConfig = requestStub.getCall(0).args[0].retryConfig; + + const connReset = { + code: 'ECONNRESET', + config: {method: 'GET', url: '/test'}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + assert.strictEqual(retryConfig.shouldRetry(connReset), true); + }); + + it('should allow retries for bucket creation and safe deletes', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + await transport.makeRequest({method: 'POST', url: '/v1/b'}); + const retryConfig = requestStub.getCall(0).args[0].retryConfig; + + // No status code (network error) on bucket create should retry + assert.strictEqual( + retryConfig.shouldRetry({ + code: 'ECONNRESET', + config: {method: 'POST', url: '/v1/b'}, + }), + true, + ); + }); + + it('should handle HMAC and IAM retry logic', async () => { + const requestStub = authClientStub.request as sinon.SinonStub; + requestStub.resolves({data: {}, headers: new Map()}); + + // Test HMAC PUT without ETag (should NOT retry) + await transport.makeRequest({ + method: 'PUT', + url: '/hmacKeys/test', + body: JSON.stringify({noEtag: true}), + }); + let retryConfig = requestStub.getCall(0).args[0].retryConfig; + assert.strictEqual( + retryConfig.shouldRetry({ + response: {status: 503}, + config: { + method: 'PUT', + url: '/hmacKeys/test', + data: JSON.stringify({noEtag: true}), + }, + }), + false, + ); + + // Test IAM PUT with ETag (should retry) + await transport.makeRequest({ + method: 'PUT', + url: '/iam/test', + body: JSON.stringify({etag: '123'}), + }); + retryConfig = requestStub.getCall(1).args[0].retryConfig; + assert.strictEqual( + retryConfig.shouldRetry({ + response: {status: 503}, + config: { + method: 'PUT', + url: '/iam/test', + data: JSON.stringify({etag: '123'}), + }, + }), + true, + ); + }); + }); });