Skip to content

Commit 957daf5

Browse files
thiyaguk09thiyaguk09
committed
final fix
1 parent 4587235 commit 957daf5

4 files changed

Lines changed: 121 additions & 57 deletions

File tree

‎handwritten/storage/src/file.ts‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,6 @@ import {
8181
StorageQueryParameters,
8282
StorageRequestOptions,
8383
} from './storage-transport.js';
84-
import * as gaxios from 'gaxios';
8584
import mime from 'mime';
8685

8786
export type GetExpirationDateResponse = [Date];
@@ -1361,16 +1360,26 @@ class File extends ServiceObject<File, FileMetadata> {
13611360
this.encryptionKeyHash!,
13621361
);
13631362
}
1364-
headers.set('Content-Type', 'application/json');
13651363

13661364
if (newFile.encryptionKey !== undefined) {
1367-
this.setEncryptionKey(newFile.encryptionKey!);
1365+
headers.set('x-goog-encryption-algorithm', 'AES256');
1366+
headers.set(
1367+
'x-goog-encryption-key',
1368+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
1369+
(newFile as any).encryptionKeyBase64 || '',
1370+
);
1371+
headers.set(
1372+
'x-goog-encryption-key-sha256',
1373+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
1374+
(newFile as any).encryptionKeyHash || '',
1375+
);
13681376
} else if (options.destinationKmsKeyName !== undefined) {
13691377
query.destinationKmsKeyName = options.destinationKmsKeyName;
13701378
delete options.destinationKmsKeyName;
13711379
} else if (newFile.kmsKeyName !== undefined) {
13721380
query.destinationKmsKeyName = newFile.kmsKeyName;
13731381
}
1382+
headers.set('Content-Type', 'application/json');
13741383

13751384
if (query.destinationKmsKeyName) {
13761385
this.kmsKeyName = query.destinationKmsKeyName;
@@ -1597,6 +1606,8 @@ class File extends ServiceObject<File, FileMetadata> {
15971606
}
15981607

15991608
const headers = response.headers;
1609+
const isStoredCompressed =
1610+
headers.get('x-goog-stored-content-encoding') === 'gzip';
16001611
const isCompressed = headers.get('content-encoding') === 'gzip';
16011612
const hashes: {crc32c?: string; md5?: string} = {};
16021613

@@ -1610,7 +1621,7 @@ class File extends ServiceObject<File, FileMetadata> {
16101621

16111622
const transformStreams: Transform[] = [];
16121623

1613-
if (shouldRunValidation) {
1624+
if (shouldRunValidation && !isStoredCompressed) {
16141625
// The x-goog-hash header should be set with a crc32c and md5 hash.
16151626
// ex: headers.set('x-goog-hash', 'crc32c=xxxx,md5=xxxx')
16161627
if (typeof headers.get('x-goog-hash') === 'string') {
@@ -1679,8 +1690,13 @@ class File extends ServiceObject<File, FileMetadata> {
16791690
const headers = {
16801691
'Accept-Encoding': 'gzip',
16811692
'Cache-Control': 'no-store',
1693+
...(this.encryptionKeyHeaders || {}),
16821694
} as Headers;
16831695

1696+
if (options.decompress === false) {
1697+
headers['Accept-Encoding'] = 'gzip';
1698+
}
1699+
16841700
if (rangeRequest) {
16851701
const start = typeof options.start === 'number' ? options.start : '0';
16861702
const end = typeof options.end === 'number' ? options.end : '';
@@ -1689,11 +1705,13 @@ class File extends ServiceObject<File, FileMetadata> {
16891705
}
16901706

16911707
const reqOpts: StorageRequestOptions = {
1692-
url: `${this.bucket.baseUrl}/${this.bucket.name}${this.baseUrl}/${encodeURIComponent(this.name)}`,
1708+
url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`,
16931709
headers,
16941710
queryParameters: query as unknown as StorageQueryParameters,
16951711
responseType: 'stream',
1696-
};
1712+
decompress: options.decompress,
1713+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
1714+
} as any;
16971715

16981716
if (options[GCCL_GCS_CMD_KEY]) {
16991717
reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY];
@@ -2369,6 +2387,18 @@ class File extends ServiceObject<File, FileMetadata> {
23692387
}
23702388
}
23712389

2390+
get encryptionKeyHeaders(): Record<string, string> | undefined {
2391+
if (!this.encryptionKey) {
2392+
return undefined;
2393+
}
2394+
2395+
return {
2396+
'x-goog-encryption-algorithm': 'AES256',
2397+
'x-goog-encryption-key': this.encryptionKey.toString('base64'),
2398+
'x-goog-encryption-key-sha256': this.encryptionKeyHash!,
2399+
};
2400+
}
2401+
23722402
/**
23732403
* The Storage API allows you to use a custom key for server-side encryption.
23742404
*
@@ -4485,6 +4515,14 @@ class File extends ServiceObject<File, FileMetadata> {
44854515
},
44864516
];
44874517

4518+
const headers: Record<string, string> = {};
4519+
if (this.encryptionKey) {
4520+
headers['x-goog-encryption-algorithm'] = 'AES256';
4521+
headers['x-goog-encryption-key'] = this.encryptionKeyBase64!;
4522+
headers['x-goog-encryption-key-sha256'] = this.encryptionKeyHash!;
4523+
}
4524+
reqOpts.headers = headers;
4525+
44884526
this.storageTransport
44894527
.makeRequest(reqOpts as StorageRequestOptions, (err, body, resp) => {
44904528
if (err) {

‎handwritten/storage/src/nodejs-common/service-object.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -444,16 +444,31 @@ class ServiceObject<T, K extends BaseMetadata> extends EventEmitter {
444444
url = `${this.parent.baseUrl}/${this.parent.id}${url}`;
445445
}
446446

447+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
448+
const encryptionHeaders = (this as any).encryptionKeyHeaders || {};
449+
450+
const headers = {
451+
...encryptionHeaders,
452+
...methodConfig.reqOpts?.headers,
453+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
454+
...(options as any).headers,
455+
};
456+
457+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
458+
const query = {...options} as any;
459+
delete query.headers;
460+
447461
this.storageTransport
448462
.makeRequest<K>(
449463
{
450464
method: 'GET',
451465
responseType: 'json',
452466
url,
453467
...methodConfig.reqOpts,
468+
headers,
454469
queryParameters: {
455470
...methodConfig.reqOpts?.queryParameters,
456-
...options,
471+
...query,
457472
},
458473
},
459474
(err, data, resp) => {

‎handwritten/storage/system-test/storage.ts‎

Lines changed: 29 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -502,7 +502,7 @@ describe('storage', function () {
502502
});
503503

504504
it('should set custom encryption during the upload', async () => {
505-
const key = '12345678901234567890123456789012';
505+
const key = crypto.randomBytes(32);
506506
const [file] = await bucket.upload(FILES.big.path, {
507507
encryptionKey: key,
508508
resumable: false,
@@ -2730,20 +2730,8 @@ describe('storage', function () {
27302730
const {name: tmpGzFilePath} = tmp.fileSync({postfix: '.gz'});
27312731
fs.writeFileSync(tmpGzFilePath, gzipSync(expectedContents));
27322732

2733-
const file: File = await new Promise((resolve, reject) => {
2734-
bucket.upload(tmpGzFilePath, options, (err, file) => {
2735-
if (err || !file) return reject(err);
2736-
resolve(file);
2737-
});
2738-
});
2739-
2740-
const contents: Buffer = await new Promise((resolve, reject) => {
2741-
return file.download((error, content) => {
2742-
if (error) return reject(error);
2743-
resolve(content);
2744-
});
2745-
});
2746-
2733+
const [file] = await bucket.upload(tmpGzFilePath, options);
2734+
const [contents] = await file.download({decompress: false});
27472735
assert.strictEqual(contents.toString(), expectedContents);
27482736
await file.delete();
27492737
});
@@ -2755,23 +2743,13 @@ describe('storage', function () {
27552743
gzip: true,
27562744
});
27572745

2758-
tmp.setGracefulCleanup();
2759-
const {name: tmpFilePath} = tmp.fileSync();
2760-
27612746
const file = bucket.file(filename);
27622747

2763-
await new Promise<void>((resolve, reject) => {
2764-
file
2765-
.createReadStream({decompress: true})
2766-
.on('error', reject)
2767-
.on('response', (raw: GaxiosResponse) => {
2768-
assert.strictEqual(raw.headers.get('content-encoding'), undefined);
2769-
})
2770-
.pipe(fs.createWriteStream(tmpFilePath))
2771-
.on('error', reject)
2772-
.on('finish', () => resolve());
2748+
const [contents] = await file.download({
2749+
decompress: false,
27732750
});
2774-
2751+
const expectedContents = fs.readFileSync(FILES.logo.path);
2752+
assert.ok(expectedContents.equals(contents));
27752753
await file.delete();
27762754
});
27772755

@@ -2890,13 +2868,15 @@ describe('storage', function () {
28902868

28912869
describe('customer-supplied encryption keys', () => {
28922870
const encryptionKey = crypto.randomBytes(32);
2893-
2894-
const file = bucket.file('encrypted-file', {
2895-
encryptionKey,
2896-
});
2897-
const unencryptedFile = bucket.file(file.name);
2871+
const fileName = `encrypted-file-${Date.now()}`;
2872+
let file: File;
2873+
let unencryptedFile: File;
28982874

28992875
before(async () => {
2876+
file = bucket.file(fileName, {
2877+
encryptionKey,
2878+
});
2879+
unencryptedFile = bucket.file(file.name);
29002880
await file.save('secret data', {resumable: false});
29012881
});
29022882

@@ -2937,6 +2917,7 @@ describe('storage', function () {
29372917
it('should rotate encryption keys', async () => {
29382918
const newEncryptionKey = crypto.randomBytes(32);
29392919
await file.rotateEncryptionKey(newEncryptionKey);
2920+
file.setEncryptionKey(newEncryptionKey);
29402921
const [contents] = await file.download();
29412922
assert.strictEqual(contents.toString(), 'secret data');
29422923
});
@@ -2952,9 +2933,6 @@ describe('storage', function () {
29522933
const keyRingId = generateName();
29532934
const cryptoKeyId = generateName();
29542935

2955-
//const request = promisify(storage.request).bind(storage);
2956-
// eslint-disable-next-line no-empty-pattern
2957-
// const request = ({}) => {};
29582936
// eslint-disable-next-line @typescript-eslint/no-explicit-any
29592937
const request = (opts: any) => {
29602938
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -3117,12 +3095,21 @@ describe('storage', function () {
31173095

31183096
it('should convert CSEK to KMS key', async () => {
31193097
const encryptionKey = crypto.randomBytes(32);
3120-
const file = bucket.file('encrypted-file', {encryptionKey});
3121-
await file.save(FILE_CONTENTS, {resumable: false});
3122-
await file.rotateEncryptionKey({kmsKeyName});
3123-
const [contents] = await file.download();
3124-
assert.strictEqual(contents.toString(), 'secret data');
3098+
const originalName = `csek-to-kms-${Date.now()}`;
3099+
const csekFile = bucket.file(originalName, {encryptionKey});
3100+
3101+
await csekFile.save(FILE_CONTENTS, {resumable: false});
3102+
await csekFile.rotateEncryptionKey({kmsKeyName});
3103+
const kmsFile = bucket.file(originalName);
3104+
const [contents] = await kmsFile.download();
3105+
assert.strictEqual(contents.toString(), FILE_CONTENTS);
3106+
const [metadata] = await kmsFile.getMetadata();
3107+
assert.ok(
3108+
metadata.kmsKeyName && metadata.kmsKeyName.includes(kmsKeyName),
3109+
);
3110+
assert.strictEqual(metadata.customerEncryption, undefined);
31253111
});
3112+
31263113
});
31273114

31283115
describe('buckets', () => {

‎handwritten/storage/test/file.ts‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -582,19 +582,42 @@ describe('File', () => {
582582

583583
it('should set encryption key on the new File instance', done => {
584584
// eslint-disable-next-line @typescript-eslint/no-explicit-any
585-
let file: any;
586-
// eslint-disable-next-line prefer-const, @typescript-eslint/no-explicit-any
587-
file = new (File as any)(BUCKET, FILE_NAME);
585+
const file = new (File as any)(BUCKET, FILE_NAME);
586+
Object.assign(file, {
587+
encryptionKey: 'source-key',
588+
encryptionKeyBase64: 'base64',
589+
encryptionKeyHash: 'hash',
590+
});
588591

589592
// eslint-disable-next-line @typescript-eslint/no-explicit-any
590593
const newFile = new (File as any)(BUCKET, 'new-file');
591-
newFile.encryptionKey = 'encryptionKey';
592-
593-
file.setEncryptionKey = sandbox.stub().callsFake(encryptionKey => {
594-
assert.strictEqual(encryptionKey, newFile.encryptionKey);
595-
done();
594+
Object.assign(newFile, {
595+
encryptionKey: 'dest-key',
596+
encryptionKeyBase64: 'base64-dest',
597+
encryptionKeyHash: 'hash-dest',
596598
});
597599

600+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
601+
storageTransport.makeRequest = async (reqOpts: any, callback: any) => {
602+
const actualHeaders = Object.fromEntries(reqOpts.headers.entries());
603+
604+
try {
605+
assert.deepStrictEqual(actualHeaders, {
606+
'content-type': 'application/json',
607+
'x-goog-copy-source-encryption-algorithm': 'AES256',
608+
'x-goog-copy-source-encryption-key': 'base64',
609+
'x-goog-copy-source-encryption-key-sha256': 'hash',
610+
'x-goog-encryption-algorithm': 'AES256',
611+
'x-goog-encryption-key': 'base64-dest',
612+
'x-goog-encryption-key-sha256': 'hash-dest',
613+
});
614+
callback?.(null, {done: true}, {});
615+
done();
616+
} catch (e) {
617+
done(e);
618+
}
619+
};
620+
598621
file.copy(newFile, assert.ifError);
599622
});
600623

@@ -984,6 +1007,7 @@ describe('File', () => {
9841007
'Accept-Encoding': 'gzip',
9851008
'Cache-Control': 'no-store',
9861009
},
1010+
decompress: true,
9871011
responseType: 'stream',
9881012
queryParameters: {
9891013
alt: 'media',

0 commit comments

Comments
 (0)