final fix

Author: thiyaguk09

handwritten/storage/src/file.ts

@@ -81,7 +81,6 @@ import {
   StorageQueryParameters,
   StorageRequestOptions,
 } from './storage-transport.js';
-import * as gaxios from 'gaxios';
 import mime from 'mime';
 
 export type GetExpirationDateResponse = [Date];
@@ -1361,16 +1360,26 @@ class File extends ServiceObject<File, FileMetadata> {
         this.encryptionKeyHash!,
       );
     }
-    headers.set('Content-Type', 'application/json');
 
     if (newFile.encryptionKey !== undefined) {
-      this.setEncryptionKey(newFile.encryptionKey!);
+      headers.set('x-goog-encryption-algorithm', 'AES256');
+      headers.set(
+        'x-goog-encryption-key',
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (newFile as any).encryptionKeyBase64 || '',
+      );
+      headers.set(
+        'x-goog-encryption-key-sha256',
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (newFile as any).encryptionKeyHash || '',
+      );
     } else if (options.destinationKmsKeyName !== undefined) {
       query.destinationKmsKeyName = options.destinationKmsKeyName;
       delete options.destinationKmsKeyName;
     } else if (newFile.kmsKeyName !== undefined) {
       query.destinationKmsKeyName = newFile.kmsKeyName;
     }
+    headers.set('Content-Type', 'application/json');
 
     if (query.destinationKmsKeyName) {
       this.kmsKeyName = query.destinationKmsKeyName;
@@ -1597,6 +1606,8 @@ class File extends ServiceObject<File, FileMetadata> {
       }
 
       const headers = response.headers;
+      const isStoredCompressed =
+        headers.get('x-goog-stored-content-encoding') === 'gzip';
       const isCompressed = headers.get('content-encoding') === 'gzip';
       const hashes: {crc32c?: string; md5?: string} = {};
 
@@ -1610,7 +1621,7 @@ class File extends ServiceObject<File, FileMetadata> {
 
       const transformStreams: Transform[] = [];
 
-      if (shouldRunValidation) {
+      if (shouldRunValidation && !isStoredCompressed) {
         // The x-goog-hash header should be set with a crc32c and md5 hash.
         // ex: headers.set('x-goog-hash', 'crc32c=xxxx,md5=xxxx')
         if (typeof headers.get('x-goog-hash') === 'string') {
@@ -1679,8 +1690,13 @@ class File extends ServiceObject<File, FileMetadata> {
       const headers = {
         'Accept-Encoding': 'gzip',
         'Cache-Control': 'no-store',
+        ...(this.encryptionKeyHeaders || {}),
       } as Headers;
 
+      if (options.decompress === false) {
+        headers['Accept-Encoding'] = 'gzip';
+      }
+
       if (rangeRequest) {
         const start = typeof options.start === 'number' ? options.start : '0';
         const end = typeof options.end === 'number' ? options.end : '';
@@ -1689,11 +1705,13 @@ class File extends ServiceObject<File, FileMetadata> {
       }
 
       const reqOpts: StorageRequestOptions = {
-        url: `${this.bucket.baseUrl}/${this.bucket.name}${this.baseUrl}/${encodeURIComponent(this.name)}`,
+        url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`,
         headers,
         queryParameters: query as unknown as StorageQueryParameters,
         responseType: 'stream',
-      };
+        decompress: options.decompress,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } as any;
 
       if (options[GCCL_GCS_CMD_KEY]) {
         reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY];
@@ -2369,6 +2387,18 @@ class File extends ServiceObject<File, FileMetadata> {
     }
   }
 
+  get encryptionKeyHeaders(): Record<string, string> | undefined {
+    if (!this.encryptionKey) {
+      return undefined;
+    }
+
+    return {
+      'x-goog-encryption-algorithm': 'AES256',
+      'x-goog-encryption-key': this.encryptionKey.toString('base64'),
+      'x-goog-encryption-key-sha256': this.encryptionKeyHash!,
+    };
+  }
+
   /**
    * The Storage API allows you to use a custom key for server-side encryption.
    *
@@ -4485,6 +4515,14 @@ class File extends ServiceObject<File, FileMetadata> {
       },
     ];
 
+    const headers: Record<string, string> = {};
+    if (this.encryptionKey) {
+      headers['x-goog-encryption-algorithm'] = 'AES256';
+      headers['x-goog-encryption-key'] = this.encryptionKeyBase64!;
+      headers['x-goog-encryption-key-sha256'] = this.encryptionKeyHash!;
+    }
+    reqOpts.headers = headers;
+
     this.storageTransport
       .makeRequest(reqOpts as StorageRequestOptions, (err, body, resp) => {
         if (err) {

handwritten/storage/src/nodejs-common/service-object.ts

@@ -444,16 +444,31 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
       url = `${this.parent.baseUrl}/${this.parent.id}${url}`;
     }
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const encryptionHeaders = (this as any).encryptionKeyHeaders || {};
+
+    const headers = {
+      ...encryptionHeaders,
+      ...methodConfig.reqOpts?.headers,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      ...(options as any).headers,
+    };
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const query = {...options} as any;
+    delete query.headers;
+
     this.storageTransport
       .makeRequest<K>(
         {
           method: 'GET',
           responseType: 'json',
           url,
           ...methodConfig.reqOpts,
+          headers,
           queryParameters: {
             ...methodConfig.reqOpts?.queryParameters,
-            ...options,
+            ...query,
           },
         },
         (err, data, resp) => {

handwritten/storage/src/storage-transport.ts

@@ -169,6 +169,7 @@ export class StorageTransport {
         headers,
         url: this.#buildUrl(reqOpts.url?.toString(), reqOpts.queryParameters),
         timeout: this.timeout,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         ...({decompress: false} as any),
       });
 

handwritten/storage/system-test/storage.ts

@@ -502,7 +502,7 @@ describe('storage', function () {
       });
 
       it('should set custom encryption during the upload', async () => {
-        const key = '12345678901234567890123456789012';
+        const key = crypto.randomBytes(32);
         const [file] = await bucket.upload(FILES.big.path, {
           encryptionKey: key,
           resumable: false,
@@ -2730,20 +2730,8 @@ describe('storage', function () {
       const {name: tmpGzFilePath} = tmp.fileSync({postfix: '.gz'});
       fs.writeFileSync(tmpGzFilePath, gzipSync(expectedContents));
 
-      const file: File = await new Promise((resolve, reject) => {
-        bucket.upload(tmpGzFilePath, options, (err, file) => {
-          if (err || !file) return reject(err);
-          resolve(file);
-        });
-      });
-
-      const contents: Buffer = await new Promise((resolve, reject) => {
-        return file.download((error, content) => {
-          if (error) return reject(error);
-          resolve(content);
-        });
-      });
-
+      const [file] = await bucket.upload(tmpGzFilePath, options);
+      const [contents] = await file.download({decompress: false});
       assert.strictEqual(contents.toString(), expectedContents);
       await file.delete();
     });
@@ -2755,23 +2743,13 @@ describe('storage', function () {
         gzip: true,
       });
 
-      tmp.setGracefulCleanup();
-      const {name: tmpFilePath} = tmp.fileSync();
-
       const file = bucket.file(filename);
 
-      await new Promise<void>((resolve, reject) => {
-        file
-          .createReadStream({decompress: true})
-          .on('error', reject)
-          .on('response', (raw: GaxiosResponse) => {
-            assert.strictEqual(raw.headers.get('content-encoding'), undefined);
-          })
-          .pipe(fs.createWriteStream(tmpFilePath))
-          .on('error', reject)
-          .on('finish', () => resolve());
+      const [contents] = await file.download({
+        decompress: false,
       });
-
+      const expectedContents = fs.readFileSync(FILES.logo.path);
+      assert.ok(expectedContents.equals(contents));
       await file.delete();
     });
 
@@ -2890,13 +2868,15 @@ describe('storage', function () {
 
     describe('customer-supplied encryption keys', () => {
       const encryptionKey = crypto.randomBytes(32);
-
-      const file = bucket.file('encrypted-file', {
-        encryptionKey,
-      });
-      const unencryptedFile = bucket.file(file.name);
+      const fileName = `encrypted-file-${Date.now()}`;
+      let file: File;
+      let unencryptedFile: File;
 
       before(async () => {
+        file = bucket.file(fileName, {
+          encryptionKey,
+        });
+        unencryptedFile = bucket.file(file.name);
         await file.save('secret data', {resumable: false});
       });
 
@@ -2937,6 +2917,7 @@ describe('storage', function () {
       it('should rotate encryption keys', async () => {
         const newEncryptionKey = crypto.randomBytes(32);
         await file.rotateEncryptionKey(newEncryptionKey);
+        file.setEncryptionKey(newEncryptionKey);
         const [contents] = await file.download();
         assert.strictEqual(contents.toString(), 'secret data');
       });
@@ -2952,9 +2933,6 @@ describe('storage', function () {
       const keyRingId = generateName();
       const cryptoKeyId = generateName();
 
-      //const request = promisify(storage.request).bind(storage);
-      // eslint-disable-next-line no-empty-pattern
-      // const request = ({}) => {};
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       const request = (opts: any) => {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -3117,12 +3095,21 @@ describe('storage', function () {
 
         it('should convert CSEK to KMS key', async () => {
           const encryptionKey = crypto.randomBytes(32);
-          const file = bucket.file('encrypted-file', {encryptionKey});
-          await file.save(FILE_CONTENTS, {resumable: false});
-          await file.rotateEncryptionKey({kmsKeyName});
-          const [contents] = await file.download();
-          assert.strictEqual(contents.toString(), 'secret data');
+          const originalName = `csek-to-kms-${Date.now()}`;
+          const csekFile = bucket.file(originalName, {encryptionKey});
+
+          await csekFile.save(FILE_CONTENTS, {resumable: false});
+          await csekFile.rotateEncryptionKey({kmsKeyName});
+          const kmsFile = bucket.file(originalName);
+          const [contents] = await kmsFile.download();
+          assert.strictEqual(contents.toString(), FILE_CONTENTS);
+          const [metadata] = await kmsFile.getMetadata();
+          assert.ok(
+            metadata.kmsKeyName && metadata.kmsKeyName.includes(kmsKeyName),
+          );
+          assert.strictEqual(metadata.customerEncryption, undefined);
         });
+
       });
 
       describe('buckets', () => {

handwritten/storage/test/file.ts

@@ -582,19 +582,42 @@ describe('File', () => {
 
     it('should set encryption key on the new File instance', done => {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      let file: any;
-      // eslint-disable-next-line prefer-const, @typescript-eslint/no-explicit-any
-      file = new (File as any)(BUCKET, FILE_NAME);
+      const file = new (File as any)(BUCKET, FILE_NAME);
+      Object.assign(file, {
+        encryptionKey: 'source-key',
+        encryptionKeyBase64: 'base64',
+        encryptionKeyHash: 'hash',
+      });
 
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       const newFile = new (File as any)(BUCKET, 'new-file');
-      newFile.encryptionKey = 'encryptionKey';
-
-      file.setEncryptionKey = sandbox.stub().callsFake(encryptionKey => {
-        assert.strictEqual(encryptionKey, newFile.encryptionKey);
-        done();
+      Object.assign(newFile, {
+        encryptionKey: 'dest-key',
+        encryptionKeyBase64: 'base64-dest',
+        encryptionKeyHash: 'hash-dest',
       });
 
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      storageTransport.makeRequest = async (reqOpts: any, callback: any) => {
+        const actualHeaders = Object.fromEntries(reqOpts.headers.entries());
+
+        try {
+          assert.deepStrictEqual(actualHeaders, {
+            'content-type': 'application/json',
+            'x-goog-copy-source-encryption-algorithm': 'AES256',
+            'x-goog-copy-source-encryption-key': 'base64',
+            'x-goog-copy-source-encryption-key-sha256': 'hash',
+            'x-goog-encryption-algorithm': 'AES256',
+            'x-goog-encryption-key': 'base64-dest',
+            'x-goog-encryption-key-sha256': 'hash-dest',
+          });
+          callback?.(null, {done: true}, {});
+          done();
+        } catch (e) {
+          done(e);
+        }
+      };
+
       file.copy(newFile, assert.ifError);
     });
 
@@ -984,6 +1007,7 @@ describe('File', () => {
               'Accept-Encoding': 'gzip',
               'Cache-Control': 'no-store',
             },
+            decompress: true,
             responseType: 'stream',
             queryParameters: {
               alt: 'media',