Skip to content

Commit 18a3b05

Browse files
gcf-owl-bot[bot]pearigeeshivanee-p
authored
chore: [dataflow] migrate generator to google-cloud-node (#7923)
* feat: Add Pausing/Yaml capabilities to public protos PiperOrigin-RevId: 891736643 Source-Link: googleapis/googleapis@3aab7be Source-Link: googleapis/googleapis-gen@3b98246 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWRhdGFmbG93Ly5Pd2xCb3QueWFtbCIsImgiOiIzYjk4MjQ2NGExZmRlNjQ5MzNlMDA3MzY5MmI1NTFkZjk1OGNkYTY4In0= * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: add sha256 field to Package proto PiperOrigin-RevId: 891887357 Source-Link: googleapis/googleapis@54ba514 Source-Link: googleapis/googleapis-gen@54b07e8 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWRhdGFmbG93Ly5Pd2xCb3QueWFtbCIsImgiOiI1NGIwN2U4YTIwZjU4MTU2YjA0MmY2YjA2ZGYzYmFkNzI4OTI0ZDMwIn0= * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: [Memorystore for Redis Cluster] Add support for Flexible CA feature PiperOrigin-RevId: 891808664 Source-Link: googleapis/googleapis@4a55538 Source-Link: googleapis/googleapis-gen@22c11cd Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWRhdGFmbG93Ly5Pd2xCb3QueWFtbCIsImgiOiIyMmMxMWNkN2YwMGJmMzRmMDMwMDEyZmIyZGRiNDZjZDkyYzgwOWM3In0= * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: add sha256 field to Package proto PiperOrigin-RevId: 891887357 Source-Link: googleapis/googleapis@0673555 Source-Link: googleapis/googleapis-gen@601430f Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWRhdGFmbG93Ly5Pd2xCb3QueWFtbCIsImgiOiI2MDE0MzBmOTJmN2M5NGIxNDYyN2E1OGU2ZjM0ODliY2NlODdkMTQ1In0= * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: migrate generator to google-cloud-node chore: update generator system tests PiperOrigin-RevId: 896174961 Source-Link: googleapis/googleapis@2233f63 Source-Link: googleapis/googleapis-gen@c4a8c4e Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWRhdGFmbG93Ly5Pd2xCb3QueWFtbCIsImgiOiJjNGE4YzRlZWY5NTlhYTc0NGZiMWM2NzE0ODc1ZmEyOTVlMDZhYWQzIn0= * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Gabe Pearhill <86282859+pearigee@users.noreply.github.com> Co-authored-by: Shivanee <46910562+shivanee-p@users.noreply.github.com>
1 parent 0cc6a9b commit 18a3b05

16 files changed

Lines changed: 598 additions & 34 deletions

‎packages/google-dataflow/.eslintrc.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
{
2-
"extends": "./node_modules/gts"
2+
"extends": "./node_modules/gts",
3+
"root": true
34
}

‎packages/google-dataflow/protos/google/dataflow/v1beta3/environment.proto‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 Google LLC
1+
// Copyright 2026 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -73,8 +73,9 @@ message Environment {
7373
// specified in order for the job to have workers.
7474
repeated WorkerPool worker_pools = 4;
7575

76-
// A description of the process that generated the request.
77-
google.protobuf.Struct user_agent = 5;
76+
// Optional. A description of the process that generated the request.
77+
google.protobuf.Struct user_agent = 5
78+
[(google.api.field_behavior) = OPTIONAL];
7879

7980
// A structure describing which components and their versions of the service
8081
// are required in order to run the job.
@@ -139,6 +140,9 @@ message Environment {
139140
// [Set the pipeline streaming
140141
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
141142
StreamingMode streaming_mode = 19 [(google.api.field_behavior) = OPTIONAL];
143+
144+
// Optional. True when any worker pool that uses public IPs is present.
145+
bool use_public_ips = 20 [(google.api.field_behavior) = OPTIONAL];
142146
}
143147

144148
// The packages that must be installed in order for a worker to run the
@@ -161,6 +165,12 @@ message Package {
161165
// storage.googleapis.com/{bucket}
162166
// bucket.storage.googleapis.com/
163167
string location = 2;
168+
169+
// Optional. The hex-encoded SHA256 checksum of the package.
170+
// If the checksum is provided, the worker will verify the checksum of the
171+
// package before using it. If the checksum does not match, the worker will
172+
// fail to start.
173+
string sha256 = 3 [(google.api.field_behavior) = OPTIONAL];
164174
}
165175

166176
// Specifies the processing model used by a
@@ -494,6 +504,13 @@ message WorkerPool {
494504
// attempt to choose a reasonable default.
495505
string disk_type = 16;
496506

507+
// Optional. IOPS provisioned for the root disk for VMs.
508+
int64 disk_provisioned_iops = 23 [(google.api.field_behavior) = OPTIONAL];
509+
510+
// Optional. Throughput provisioned for the root disk for VMs.
511+
int64 disk_provisioned_throughput_mibps = 24
512+
[(google.api.field_behavior) = OPTIONAL];
513+
497514
// Fully qualified source image for disks.
498515
string disk_source_image = 8;
499516

‎packages/google-dataflow/protos/google/dataflow/v1beta3/jobs.proto‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 Google LLC
1+
// Copyright 2026 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -34,15 +34,15 @@ option java_package = "com.google.dataflow.v1beta3";
3434
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
3535
option ruby_package = "Google::Cloud::Dataflow::V1beta3";
3636

37-
// Provides a method to create and modify Google Cloud Dataflow jobs.
38-
// A Job is a multi-stage computation graph run by the Cloud Dataflow service.
37+
// Provides a method to create and modify Dataflow jobs.
38+
// A Job is a multi-stage computation graph run by the Dataflow service.
3939
service JobsV1Beta3 {
4040
option (google.api.default_host) = "dataflow.googleapis.com";
4141
option (google.api.oauth_scopes) =
4242
"https://www.googleapis.com/auth/cloud-platform,"
4343
"https://www.googleapis.com/auth/compute";
4444

45-
// Creates a Cloud Dataflow job.
45+
// Creates a Dataflow job.
4646
//
4747
// To create a job, we recommend using `projects.locations.jobs.create` with a
4848
// [regional endpoint]
@@ -315,6 +315,9 @@ message Job {
315315
// Output only. Resources used by the Dataflow Service to run the job.
316316
optional ServiceResources service_resources = 28
317317
[(google.api.field_behavior) = OUTPUT_ONLY];
318+
319+
// Output only. Indicates whether the job can be paused.
320+
bool pausable = 29 [(google.api.field_behavior) = OUTPUT_ONLY];
318321
}
319322

320323
// Resources used by the Dataflow Service to run the job.
@@ -343,6 +346,16 @@ message RuntimeUpdatableParams {
343346
// [Update an existing
344347
// pipeline](https://cloud.google.com/dataflow/docs/guides/updating-a-pipeline).
345348
optional double worker_utilization_hint = 3;
349+
350+
// Optional. Deprecated: Use `autoscaling_tier` instead.
351+
// The backlog threshold duration in seconds for autoscaling. Value must be
352+
// non-negative.
353+
optional google.protobuf.Duration acceptable_backlog_duration = 4
354+
[deprecated = true, (google.api.field_behavior) = OPTIONAL];
355+
356+
// Optional. The backlog threshold tier for autoscaling. Value must be one of
357+
// "low-latency", "medium-latency", or "high-latency".
358+
optional string autoscaling_tier = 5 [(google.api.field_behavior) = OPTIONAL];
346359
}
347360

348361
// Metadata for a Datastore connector used by the job.
@@ -836,6 +849,12 @@ enum JobState {
836849
// Currently, this is an opt-in feature, please reach out to Cloud support
837850
// team if you are interested.
838851
JOB_STATE_RESOURCE_CLEANING_UP = 12;
852+
853+
// `JOB_STATE_PAUSING` is not implemented yet.
854+
JOB_STATE_PAUSING = 13;
855+
856+
// `JOB_STATE_PAUSED` is not implemented yet.
857+
JOB_STATE_PAUSED = 14;
839858
}
840859

841860
// Additional information about how a Cloud Dataflow job will be executed that

‎packages/google-dataflow/protos/google/dataflow/v1beta3/messages.proto‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 Google LLC
1+
// Copyright 2026 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -29,8 +29,7 @@ option java_package = "com.google.dataflow.v1beta3";
2929
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
3030
option ruby_package = "Google::Cloud::Dataflow::V1beta3";
3131

32-
// The Dataflow Messages API is used for monitoring the progress of
33-
// Dataflow jobs.
32+
// The Dataflow Messages API is used to monitor the progress of Dataflow jobs.
3433
service MessagesV1Beta3 {
3534
option (google.api.default_host) = "dataflow.googleapis.com";
3635
option (google.api.oauth_scopes) =

‎packages/google-dataflow/protos/google/dataflow/v1beta3/metrics.proto‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 Google LLC
1+
// Copyright 2026 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -30,8 +30,7 @@ option java_package = "com.google.dataflow.v1beta3";
3030
option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3";
3131
option ruby_package = "Google::Cloud::Dataflow::V1beta3";
3232

33-
// The Dataflow Metrics API lets you monitor the progress of Dataflow
34-
// jobs.
33+
// The Dataflow Metrics API lets you monitor the progress of Dataflow jobs.
3534
service MetricsV1Beta3 {
3635
option (google.api.default_host) = "dataflow.googleapis.com";
3736
option (google.api.oauth_scopes) =
@@ -96,7 +95,6 @@ message MetricStructuredName {
9695
}
9796

9897
// Describes the state of a metric.
99-
// Next ID: 14
10098
message MetricUpdate {
10199
// Name of the metric.
102100
MetricStructuredName name = 1;
@@ -141,6 +139,12 @@ message MetricUpdate {
141139
// possible value type is a BoundedTrieNode.
142140
google.protobuf.Value trie = 13;
143141

142+
// Worker-computed aggregate value for the "Trie" aggregation kind. The only
143+
// possible value type is a BoundedTrieNode.
144+
// Introduced this field to avoid breaking older SDKs when Dataflow service
145+
// starts to populate the `bounded_trie` field.
146+
google.protobuf.Value bounded_trie = 14;
147+
144148
// A struct value describing properties of a distribution of numeric values.
145149
google.protobuf.Value distribution = 11;
146150

‎packages/google-dataflow/protos/google/dataflow/v1beta3/snapshots.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 Google LLC
1+
// Copyright 2026 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.

‎packages/google-dataflow/protos/google/dataflow/v1beta3/streaming.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 Google LLC
1+
// Copyright 2026 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.

‎packages/google-dataflow/protos/google/dataflow/v1beta3/templates.proto‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2025 Google LLC
1+
// Copyright 2026 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -299,6 +299,10 @@ message FlexTemplateRuntimeEnvironment {
299299
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
300300
optional StreamingMode streaming_mode = 26
301301
[(google.api.field_behavior) = OPTIONAL];
302+
303+
// Optional. Additional pipeline option flags for the job.
304+
repeated string additional_pipeline_options = 27
305+
[(google.api.field_behavior) = OPTIONAL];
302306
}
303307

304308
// A request to launch a Cloud Dataflow job from a FlexTemplate.
@@ -414,6 +418,10 @@ message RuntimeEnvironment {
414418
// mode](https://cloud.google.com/dataflow/docs/guides/streaming-modes).
415419
optional StreamingMode streaming_mode = 19
416420
[(google.api.field_behavior) = OPTIONAL];
421+
422+
// Optional. Additional pipeline option flags for the job.
423+
repeated string additional_pipeline_options = 20
424+
[(google.api.field_behavior) = OPTIONAL];
417425
}
418426

419427
// ParameterMetadataEnumOption specifies the option shown in the enum form.
@@ -523,6 +531,9 @@ message SDKInfo {
523531

524532
// Go.
525533
GO = 3;
534+
535+
// YAML.
536+
YAML = 4;
526537
}
527538

528539
// Required. The SDK Language.

0 commit comments

Comments
 (0)