Skip to content

Commit 5c64fc6

Browse files
thiyaguk09thiyaguk09
authored
feat(storage transfer): Added samples for storage transfer (#2059)
1 parent 0a13650 commit 5c64fc6

10 files changed

Lines changed: 834 additions & 14 deletions

‎storagetransfer/composer.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
{
22
"require": {
3-
"google/cloud-storage-transfer": "^1.4",
3+
"google/cloud-storage-transfer": "^2.0",
44
"paragonie/random_compat": "^9.0.0"
55
},
66
"require-dev": {
7-
"google/cloud-storage": "^1.20.1"
7+
"google/cloud-storage": "^1.20.1",
8+
"google/cloud-pubsub": "^2.0"
89
}
910
}

‎storagetransfer/src/check_latest_transfer_operation.php‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
<?php
2+
/**
3+
* Copyright 2024 Google Inc.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
namespace Google\Cloud\Samples\StorageTransfer;
19+
20+
# [START storagetransfer_get_latest_transfer_operation]
21+
use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
22+
use Google\Cloud\StorageTransfer\V1\GetTransferJobRequest;
23+
24+
/**
25+
* Checks the latest transfer operation for a given transfer job.
26+
*
27+
* @param string $projectId Your Google Cloud project ID.
28+
* @param string $jobName Storage Transfer Service job name.
29+
*/
30+
function check_latest_transfer_operation(
31+
string $projectId,
32+
string $jobName
33+
): void {
34+
// $project = 'my-project-id';
35+
// $jobName = 'myJob/1234567890';
36+
$transferJob = new GetTransferJobRequest([
37+
'project_id' => $projectId,
38+
'job_name' => $jobName
39+
]);
40+
41+
$client = new StorageTransferServiceClient();
42+
$request = $client->getTransferJob($transferJob);
43+
$latestOperationName = $request->getLatestOperationName();
44+
45+
if ($latestOperationName) {
46+
$transferOperation = $client->resumeOperation($latestOperationName);
47+
$operation = $transferOperation->getLastProtoResponse();
48+
49+
printf('Latest transfer operation for %s is: %s ' . PHP_EOL, $jobName, $operation->serializeToJsonString());
50+
} else {
51+
printf('Transfer job %s has not ran yet.' . PHP_EOL, $jobName);
52+
}
53+
}
54+
# [END storagetransfer_get_latest_transfer_operation]
55+
56+
// The following 2 lines are only needed to run the samples
57+
require_once __DIR__ . '/../../testing/sample_helpers.php';
58+
\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);

‎storagetransfer/src/event_driven_gcs_transfer.php‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?php
2+
3+
/**
4+
* Copyright 2024 Google Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License");
7+
* you may not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
namespace Google\Cloud\Samples\StorageTransfer;
20+
21+
# [START storagetransfer_create_event_driven_gcs_transfer]
22+
23+
use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
24+
use Google\Cloud\StorageTransfer\V1\CreateTransferJobRequest;
25+
use Google\Cloud\StorageTransfer\V1\EventStream;
26+
use Google\Cloud\StorageTransfer\V1\GcsData;
27+
use Google\Cloud\StorageTransfer\V1\TransferJob;
28+
use Google\Cloud\StorageTransfer\V1\TransferJob\Status;
29+
use Google\Cloud\StorageTransfer\V1\TransferSpec;
30+
31+
/**
32+
* Creates an event driven transfer that tracks a Pubsub subscription.
33+
*
34+
* @param string $projectId Your Google Cloud project ID.
35+
* @param string $sourceGcsBucketName The name of the GCS bucket to transfer objects from.
36+
* @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to.
37+
* @param string $pubsubId The subscription ID to a Pubsub queue to track.
38+
*/
39+
function event_driven_gcs_transfer(
40+
string $projectId,
41+
string $sourceGcsBucketName,
42+
string $sinkGcsBucketName,
43+
string $pubsubId
44+
): void {
45+
// $project = 'my-project-id';
46+
// $sourceGcsBucketName = 'my-source-bucket';
47+
// $sinkGcsBucketName = 'my-sink-bucket';
48+
// $pubsubId = 'projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID';
49+
50+
$transferJob = new TransferJob([
51+
'project_id' => $projectId,
52+
'transfer_spec' => new TransferSpec([
53+
'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]),
54+
'gcs_data_source' => new GcsData(['bucket_name' => $sourceGcsBucketName])
55+
]),
56+
'event_stream' => new EventStream(['name' => $pubsubId]),
57+
'status' => Status::ENABLED
58+
]);
59+
60+
$client = new StorageTransferServiceClient();
61+
$createRequest = (new CreateTransferJobRequest())
62+
->setTransferJob($transferJob);
63+
$response = $client->createTransferJob($createRequest);
64+
65+
printf('Created an event driven transfer from %s to %s with name %s .' . PHP_EOL, $sourceGcsBucketName, $sinkGcsBucketName, $response->getName());
66+
}
67+
# [END storagetransfer_create_event_driven_gcs_transfer]
68+
69+
// The following 2 lines are only needed to run the samples
70+
require_once __DIR__ . '/../../testing/sample_helpers.php';
71+
\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);

‎storagetransfer/src/manifest_request.php‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?php
2+
/**
3+
* Copyright 2024 Google Inc.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
namespace Google\Cloud\Samples\StorageTransfer;
19+
20+
# [START storagetransfer_manifest_request]
21+
22+
use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
23+
use Google\Cloud\StorageTransfer\V1\CreateTransferJobRequest;
24+
use Google\Cloud\StorageTransfer\V1\GcsData;
25+
use Google\Cloud\StorageTransfer\V1\PosixFilesystem;
26+
use Google\Cloud\StorageTransfer\V1\RunTransferJobRequest;
27+
use Google\Cloud\StorageTransfer\V1\TransferJob;
28+
use Google\Cloud\StorageTransfer\V1\TransferJob\Status;
29+
use Google\Cloud\StorageTransfer\V1\TransferManifest;
30+
use Google\Cloud\StorageTransfer\V1\TransferSpec;
31+
32+
/**
33+
* Creates and runs a transfer from the local file system to the sink bucket
34+
*
35+
* @param string $projectId Your Google Cloud project ID.
36+
* @param string $sourceAgentPoolName The agent pool associated with the POSIX data source.
37+
* @param string $rootDirectory The root directory path on the source filesystem.
38+
* @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to.
39+
* @param string $manifestLocation Transfer manifest location. Must be a `gs:` URL.
40+
*/
41+
function manifest_request(
42+
string $projectId,
43+
string $sourceAgentPoolName,
44+
string $rootDirectory,
45+
string $sinkGcsBucketName,
46+
string $manifestLocation
47+
): void {
48+
// $project = 'my-project-id';
49+
// $sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default';
50+
// $rootDirectory = '/directory/to/transfer/source';
51+
// $sinkGcsBucketName = 'my-sink-bucket';
52+
// $manifestLocation = 'gs://my-bucket/sample_manifest.csv';
53+
$transferJob = new TransferJob([
54+
'project_id' => $projectId,
55+
'transfer_spec' => new TransferSpec([
56+
'source_agent_pool_name' => $sourceAgentPoolName,
57+
'posix_data_source' => new PosixFilesystem(['root_directory' => $rootDirectory]),
58+
'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]),
59+
'transfer_manifest' => new TransferManifest(['location' => $manifestLocation])
60+
]),
61+
'status' => Status::ENABLED
62+
]);
63+
64+
$client = new StorageTransferServiceClient();
65+
$createRequest = (new CreateTransferJobRequest())
66+
->setTransferJob($transferJob);
67+
$response = $client->createTransferJob($createRequest);
68+
$runRequest = (new RunTransferJobRequest())
69+
->setJobName($response->getName())
70+
->setProjectId($projectId);
71+
$client->runTransferJob($runRequest);
72+
73+
printf('Created and ran transfer job from %s to %s using manifest %s with name %s ' . PHP_EOL, $rootDirectory, $sinkGcsBucketName, $manifestLocation, $response->getName());
74+
}
75+
# [END storagetransfer_manifest_request]
76+
77+
// The following 2 lines are only needed to run the samples
78+
require_once __DIR__ . '/../../testing/sample_helpers.php';
79+
\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);

‎storagetransfer/src/nearline_request.php‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
<?php
2+
/**
3+
* Copyright 2024 Google Inc.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
namespace Google\Cloud\Samples\StorageTransfer;
19+
20+
# [START storagetransfer_transfer_to_nearline]
21+
22+
use DateTime;
23+
use Google\Cloud\StorageTransfer\V1\Client\StorageTransferServiceClient;
24+
use Google\Cloud\StorageTransfer\V1\CreateTransferJobRequest;
25+
use Google\Cloud\StorageTransfer\V1\GcsData;
26+
use Google\Cloud\StorageTransfer\V1\ObjectConditions;
27+
use Google\Cloud\StorageTransfer\V1\RunTransferJobRequest;
28+
use Google\Cloud\StorageTransfer\V1\Schedule;
29+
use Google\Cloud\StorageTransfer\V1\TransferJob;
30+
use Google\Cloud\StorageTransfer\V1\TransferJob\Status;
31+
use Google\Cloud\StorageTransfer\V1\TransferOptions;
32+
use Google\Cloud\StorageTransfer\V1\TransferSpec;
33+
use Google\Protobuf\Duration as ProtobufDuration;
34+
use Google\Type\Date;
35+
use Google\Type\TimeOfDay;
36+
37+
/**
38+
* Create a daily migration from a GCS bucket to another GCS bucket for objects untouched for 30+ days.
39+
*
40+
* @param string $projectId Your Google Cloud project ID.
41+
* @param string $description A useful description for your transfer job.
42+
* @param string $sourceGcsBucketName The name of the GCS bucket to transfer objects from.
43+
* @param string $sinkGcsBucketName The name of the GCS bucket to transfer objects to.
44+
* @param string $startDate Date to start daily migration.
45+
*/
46+
function nearline_request(
47+
string $projectId,
48+
string $description,
49+
string $sourceGcsBucketName,
50+
string $sinkGcsBucketName,
51+
string $startDate
52+
): void {
53+
// $project = 'my-project-id';
54+
// $description = 'My transfer job';
55+
// $sourceGcsBucketName = 'my-source-bucket';
56+
// $sinkGcsBucketName = 'my-sink-bucket';
57+
// $startDate = new DateTime();
58+
59+
$dateTime = new DateTime($startDate);
60+
$date = new Date([
61+
'year' => $dateTime->format('Y'),
62+
'month' => $dateTime->format('m'),
63+
'day' => $dateTime->format('d'),
64+
]);
65+
66+
$time = new TimeOfDay([
67+
'hours' => $dateTime->format('H'),
68+
'minutes' => $dateTime->format('i'),
69+
'seconds' => $dateTime->format('s'),
70+
]);
71+
72+
$transferJob = new TransferJob([
73+
'project_id' => $projectId,
74+
'description' => $description,
75+
'schedule' => new Schedule([
76+
'schedule_start_date' => $date,
77+
'start_time_of_day' => $time
78+
]),
79+
'transfer_spec' => new TransferSpec([
80+
'gcs_data_source' => new GcsData(['bucket_name' => $sourceGcsBucketName]),
81+
'gcs_data_sink' => new GcsData(['bucket_name' => $sinkGcsBucketName]),
82+
'object_conditions' => new ObjectConditions([
83+
'min_time_elapsed_since_last_modification' => new ProtobufDuration([
84+
'seconds' => 2592000
85+
])
86+
]),
87+
'transfer_options' => new TransferOptions(['delete_objects_from_source_after_transfer' => true])
88+
]),
89+
'status' => Status::ENABLED
90+
]);
91+
92+
$client = new StorageTransferServiceClient();
93+
$createRequest = (new CreateTransferJobRequest())
94+
->setTransferJob($transferJob);
95+
$response = $client->createTransferJob($createRequest);
96+
$runRequest = (new RunTransferJobRequest())
97+
->setJobName($response->getName())
98+
->setProjectId($projectId);
99+
$client->runTransferJob($runRequest);
100+
101+
printf('Created and ran transfer job : %s' . PHP_EOL, $response->getName());
102+
}
103+
# [END storagetransfer_transfer_to_nearline]
104+
105+
// The following 2 lines are only needed to run the samples
106+
require_once __DIR__ . '/../../testing/sample_helpers.php';
107+
\Google\Cloud\Samples\execute_sample(__FILE__, __NAMESPACE__, $argv);

0 commit comments

Comments
 (0)