diff --git a/storage/disableRequesterPays.js b/storage/disableRequesterPays.js new file mode 100644 index 0000000000..7e10fe703d --- /dev/null +++ b/storage/disableRequesterPays.js @@ -0,0 +1,64 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +/** + * This application demonstrates how to perform basic operations on buckets with + * the Google Cloud Storage API. + * + * For more information, see the README.md under /storage and the documentation + * at https://cloud.google.com/storage/docs. + */ + +function main(projectId, bucketName = 'my-bucket') { + // [START storage_disable_requester_pays] + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your Google Cloud project + // const projectId = 'your-project-id'; + + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // Imports the Google Cloud client library + const {Storage} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + async function disableRequesterPays() { + try { + // Disables requester-pays requests + await storage + .bucket(bucketName) + .disableRequesterPays({userProject: projectId}); + + console.log( + `Requester-pays requests have been disabled for bucket ${bucketName}` + ); + } catch (error) { + console.error( + 'Error executing disable requester pays:', + error.message || error + ); + } + } + + disableRequesterPays(); + // [END storage_disable_requester_pays] +} +main(...process.argv.slice(2)); diff --git a/storage/downloadFileUsingRequesterPays.js b/storage/downloadFileUsingRequesterPays.js new file mode 100644 index 0000000000..7e27c8fb06 --- /dev/null +++ b/storage/downloadFileUsingRequesterPays.js @@ -0,0 +1,81 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +/** + * This application demonstrates how to perform basic operations on buckets with + * the Google Cloud Storage API. + * + * For more information, see the README.md under /storage and the documentation + * at https://cloud.google.com/storage/docs. + */ + +const uuid = require('uuid'); +const path = require('path'); + +function main( + projectId = 'cloud-devrel-public-resources', + bucketName = `nodejs-storage-samples-${uuid.v4()}`, + srcFileName = 'test.txt', + destFileName = path.join(__dirname, `test_${uuid.v4()}.txt`) +) { + // [START storage_download_file_requester_pays] + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The project ID to bill + // const projectId = 'my-billable-project-id'; + + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // The ID of your GCS file + // const srcFileName = 'your-file-name'; + + // The path to which the file should be downloaded + // const destFileName = '/local/path/to/file.txt'; + + // Imports the Google Cloud client library + const {Storage} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + async function downloadFileUsingRequesterPays() { + try { + const options = { + destination: destFileName, + userProject: projectId, + }; + + // Downloads the file + await storage.bucket(bucketName).file(srcFileName).download(options); + + console.log( + `gs://${bucketName}/${srcFileName} downloaded to ${destFileName} using requester-pays requests` + ); + } catch (error) { + console.error( + 'Error executing download file using requester pays:', + error.message || error + ); + } + } + + downloadFileUsingRequesterPays(); + // [END storage_download_file_requester_pays] +} +main(...process.argv.slice(2)); diff --git a/storage/enableRequesterPays.js b/storage/enableRequesterPays.js new file mode 100644 index 0000000000..4763f75b92 --- /dev/null +++ b/storage/enableRequesterPays.js @@ -0,0 +1,57 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +/** + * This application demonstrates how to perform basic operations on buckets with + * the Google Cloud Storage API. + * + * For more information, see the README.md under /storage and the documentation + * at https://cloud.google.com/storage/docs. + */ + +function main(bucketName = 'my-bucket') { + // [START storage_enable_requester_pays] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // Imports the Google Cloud client library + const {Storage} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + async function enableRequesterPays() { + try { + await storage.bucket(bucketName).enableRequesterPays(); + + console.log( + `Requester-pays requests have been enabled for bucket ${bucketName}` + ); + } catch (error) { + console.error( + 'Error executing enable requester pays:', + error.message || error + ); + } + } + + enableRequesterPays(); + // [END storage_enable_requester_pays] +} +main(...process.argv.slice(2)); diff --git a/storage/getRequesterPaysStatus.js b/storage/getRequesterPaysStatus.js new file mode 100644 index 0000000000..fae1b9a544 --- /dev/null +++ b/storage/getRequesterPaysStatus.js @@ -0,0 +1,69 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +/** + * This application demonstrates how to perform basic operations on buckets with + * the Google Cloud Storage API. + * + * For more information, see the README.md under /storage and the documentation + * at https://cloud.google.com/storage/docs. + */ + +function main(projectId, bucketName = 'my-bucket') { + // [START storage_get_requester_pays_status] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your Google Cloud project + // const projectId = 'your-project-id'; + + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // Imports the Google Cloud client library + const {Storage} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + async function getRequesterPaysStatus() { + try { + // Gets the requester-pays status of a bucket + const [metadata] = await storage + .bucket(bucketName) + .getMetadata({userProject: projectId}); + + let status; + if (metadata && metadata.billing && metadata.billing.requesterPays) { + status = 'enabled'; + } else { + status = 'disabled'; + } + console.log( + `Requester-pays requests are ${status} for bucket ${bucketName}.` + ); + } catch (error) { + console.error( + 'Error executing get requester pays status:', + error.message || error + ); + } + } + + getRequesterPaysStatus(); + // [END storage_get_requester_pays_status] +} +main(...process.argv.slice(2)); diff --git a/storage/getServiceAccount.js b/storage/getServiceAccount.js new file mode 100644 index 0000000000..e3acc72e43 --- /dev/null +++ b/storage/getServiceAccount.js @@ -0,0 +1,55 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// sample-metadata: +// title: Storage Get Service Account. +// description: Get Service Account. +// usage: node getServiceAccount.js + +function main(projectId = 'serviceAccountProjectId') { + // [START storage_get_service_account] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCP project + // const projectId = 'your-project-id'; + + // Imports the Google Cloud client library + const {Storage} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage({ + projectId, + }); + + async function getServiceAccount() { + try { + const [serviceAccount] = await storage.getServiceAccount(); + console.log( + `The GCS service account for project ${projectId} is: ${serviceAccount.emailAddress}` + ); + } catch (error) { + console.error( + 'Error executing get service account:', + error.message || error + ); + } + } + + getServiceAccount(); + // [END storage_get_service_account] +} +main(...process.argv.slice(2)); diff --git a/storage/package.json b/storage/package.json new file mode 100644 index 0000000000..71acc18ab5 --- /dev/null +++ b/storage/package.json @@ -0,0 +1,30 @@ +{ + "name": "@google-cloud/storage-samples", + "description": "Samples for the Cloud Storage Client Library for Node.js.", + "license": "Apache-2.0", + "author": "Google Inc.", + "engines": { + "node": ">=12" + }, + "repository": "googleapis/nodejs-storage", + "private": true, + "files": [ + "*.js" + ], + "scripts": { + "cleanup": "node scripts/cleanup", + "test": "mocha system-test/*.js --timeout 800000" + }, + "dependencies": { + "@google-cloud/pubsub": "^4.0.0", + "@google-cloud/storage": "^7.19.0", + "node-fetch": "^2.6.7", + "uuid": "^8.0.0", + "yargs": "^16.0.0" + }, + "devDependencies": { + "chai": "^4.2.0", + "mocha": "^8.0.0", + "p-limit": "^3.1.0" + } +} diff --git a/storage/resources/.gitignore b/storage/resources/.gitignore new file mode 100644 index 0000000000..6738013702 --- /dev/null +++ b/storage/resources/.gitignore @@ -0,0 +1 @@ +downloaded.txt diff --git a/storage/resources/resourcesSub1/testSub1.txt b/storage/resources/resourcesSub1/testSub1.txt new file mode 100644 index 0000000000..51f4b307d5 --- /dev/null +++ b/storage/resources/resourcesSub1/testSub1.txt @@ -0,0 +1,2 @@ +Sub1 +Hello World! \ No newline at end of file diff --git a/storage/resources/test.txt b/storage/resources/test.txt new file mode 100644 index 0000000000..c57eff55eb --- /dev/null +++ b/storage/resources/test.txt @@ -0,0 +1 @@ +Hello World! \ No newline at end of file diff --git a/storage/resources/test2.txt b/storage/resources/test2.txt new file mode 100644 index 0000000000..010302410b --- /dev/null +++ b/storage/resources/test2.txt @@ -0,0 +1 @@ +Hello World 2! \ No newline at end of file diff --git a/storage/scripts/cleanup b/storage/scripts/cleanup new file mode 100644 index 0000000000..61bd73114f --- /dev/null +++ b/storage/scripts/cleanup @@ -0,0 +1,44 @@ +#!/usr/bin/env node + +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {Storage} = require('@google-cloud/storage'); +const storage = new Storage(); +const NAME_REG_EXP = /^nodejs-storage-samples-[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}$/; + +storage + .getBuckets() + .then(([buckets]) => { + let promise = Promise.resolve(); + + buckets + .filter((bucket) => NAME_REG_EXP.test(bucket.name)) + .forEach((bucket) => { + promise = promise.then(() => { + return bucket.deleteFiles() + .then(() => bucket.deleteFiles(), console.error) + .then(() => { + console.log(`Deleting ${bucket.name}`); + return bucket.delete(); + }, console.error) + .catch(console.error); + }); + }); + }) + .catch((err) => { + console.error('ERROR:', err); + }); diff --git a/storage/system-test/requesterPays.test.js b/storage/system-test/requesterPays.test.js new file mode 100644 index 0000000000..d57abdee77 --- /dev/null +++ b/storage/system-test/requesterPays.test.js @@ -0,0 +1,111 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const fs = require('fs'); +const {Storage} = require('@google-cloud/storage'); +const {assert} = require('chai'); +const {before, after, it} = require('mocha'); +const cp = require('child_process'); +const uuid = require('uuid'); +const path = require('path'); +const {promisify} = require('util'); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const storage = new Storage(); +const cwd = path.join(__dirname, '..'); +const bucketName = `nodejs-storage-samples-${uuid.v4()}`; +const fileName = 'test.txt'; +const bucket = storage.bucket(bucketName); +const projectId = process.env.GCLOUD_PROJECT; + +const uploadFilePath = path.join(cwd, 'resources', fileName); +const downloadFilePath = path.join(__dirname, `test_${uuid.v4()}.txt`); + +before(async () => { + await bucket.create(); + // Upload a test file (to download later) + await bucket.upload(uploadFilePath); +}); + +after(async () => { + await promisify(fs.unlink)(downloadFilePath).catch(console.error); + // Try deleting all files twice, just to make sure + await bucket.deleteFiles({force: true}).catch(console.error); + await bucket.deleteFiles({force: true}).catch(console.error); + await bucket.delete().catch(console.error); +}); + +it.skip('should error on requester-pays requests if they are disabled', () => { + const result = execSync( + `node downloadFileUsingRequesterPays.js ${projectId} ${bucketName} ${fileName} ${downloadFilePath}` + ); + assert.match(result, /User project prohibited for non requester pays bucket/); +}); + +it('should fetch requester-pays status on a default bucket', () => { + const out = execSync( + `node getRequesterPaysStatus.js ${projectId} ${bucketName}` + ); + assert.include( + out, + `Requester-pays requests are disabled for bucket ${bucketName}` + ); +}); + +it('should enable requester-pays requests', () => { + const out = execSync(`node enableRequesterPays.js ${bucketName}`); + assert.include( + out, + `Requester-pays requests have been enabled for bucket ${bucketName}` + ); +}); + +it('should fetch requester-pays status on a modified bucket', () => { + const out = execSync( + `node getRequesterPaysStatus.js ${projectId} ${bucketName}` + ); + assert.include( + out, + `Requester-pays requests are enabled for bucket ${bucketName}.` + ); +}); + +it('should download a file using requester-pays requests', () => { + const out = execSync( + `node downloadFileUsingRequesterPays.js ${projectId} ${bucketName} ${fileName} ${downloadFilePath}` + ); + assert.include( + out, + `gs://${bucketName}/${fileName} downloaded to ${downloadFilePath} using requester-pays requests` + ); + fs.statSync(downloadFilePath); +}); + +it('should disable requester-pays requests', () => { + const out = execSync( + `node disableRequesterPays.js ${projectId} ${bucketName}` + ); + assert.include( + out, + `Requester-pays requests have been disabled for bucket ${bucketName}` + ); +}); + +it('should get service account', () => { + const out = execSync(`node getServiceAccount.js ${projectId}`); + assert.include(out, '@gs-project-accounts.iam.gserviceaccount.com'); +}); diff --git a/storage/system-test/test_9d800329-00da-4cdd-9a3e-7ac6743d5813.txt b/storage/system-test/test_9d800329-00da-4cdd-9a3e-7ac6743d5813.txt new file mode 100644 index 0000000000..e69de29bb2