Skip to content

Commit 43739c5

Browse files
cy-yuncy-yun
authored
feat(BigQuery): Use lossless Int64 Timestamps (#8375)
1 parent 67ab2bf commit 43739c5

10 files changed

Lines changed: 235 additions & 13 deletions

File tree

‎BigQuery/composer.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"minimum-stability": "stable",
66
"require": {
77
"php": "^8.0",
8-
"google/cloud-core": "^1.57",
8+
"google/cloud-core": "^1.64",
99
"ramsey/uuid": "^3.0|^4.0"
1010
},
1111
"require-dev": {

‎BigQuery/src/BigQueryClient.php‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,9 @@ public function queryConfig($query, array $options = [])
347347
* until the job is complete. By default, will poll indefinitely.
348348
* @type bool $returnRawResults Returns the raw data types returned from
349349
* BigQuery without converting their values into native PHP types or
350-
* the custom type classes supported by this library.
350+
* the custom type classes supported by this library. Default is false.
351+
* @type boolean $formatOptions.useInt64Timestamp Optional. Output
352+
* timestamp as usec int64. Default is false.
351353
* }
352354
* @return QueryResults
353355
* @throws JobException If the maximum number of retries while waiting for
@@ -361,6 +363,7 @@ public function runQuery(JobConfigurationInterface $query, array $options = [])
361363
'timeoutMs',
362364
'maxRetries',
363365
'returnRawResults',
366+
'formatOptions.useInt64Timestamp'
364367
], $options);
365368
$queryResultsOptions['initialTimeoutMs'] = 10000;
366369

‎BigQuery/src/Job.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,11 @@ public function cancel(array $options = [])
186186
* Please note that this option is used when iterating on the
187187
* returned class, and will not block immediately upon calling of
188188
* this method.
189+
* @type bool $returnRawResults Returns the raw data types returned from
190+
* BigQuery without converting their values into native PHP types or
191+
* the custom type classes supported by this library. Default is false
192+
* @type boolean $formatOptions.useInt64Timestamp Optional. Output
193+
* timestamp as usec int64. Default is false.
189194
* }
190195
* @return QueryResults
191196
*/

‎BigQuery/src/QueryResults.php‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/**
34
* Copyright 2016 Google Inc. All Rights Reserved.
45
*
@@ -78,7 +79,9 @@ class QueryResults implements \IteratorAggregate
7879
* @param ValueMapper $mapper Maps values between PHP and BigQuery.
7980
* @param Job $job The job from which the query results originated.
8081
* @param array $queryResultsOptions Default options to be used for calls to
81-
* get query results.
82+
* get query results. See
83+
* [documentation](https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults#query-parameters)
84+
* for available options.
8285
*/
8386
public function __construct(
8487
ConnectionInterface $connection,
@@ -151,7 +154,9 @@ public function __construct(
151154
* until the job is complete. By default, will poll indefinitely.
152155
* @type bool $returnRawResults Returns the raw data types returned from
153156
* BigQuery without converting their values into native PHP types or
154-
* the custom type classes supported by this library.
157+
* the custom type classes supported by this library. Default is false.
158+
* @type boolean $formatOptions.useInt64Timestamp Optional. Output
159+
* timestamp as usec int64. Default is false.
155160
* }
156161
* @return ItemIterator
157162
* @throws JobException If the maximum number of retries while waiting for
@@ -221,6 +226,8 @@ function (array $row) use ($schema, $returnRawResults) {
221226
* **Defaults to** `10000` milliseconds (10 seconds).
222227
* @type int $maxRetries The number of times to poll the Job status,
223228
* until the job is complete. By default, will poll indefinitely.
229+
* @type boolean $formatOptions.useInt64Timestamp Optional. Output
230+
* timestamp as usec int64. Default is false.
224231
* }
225232
* @throws JobException If the maximum number of retries while waiting for
226233
* query completion has been exceeded.
@@ -278,6 +285,8 @@ public function info()
278285
* @type int $startIndex Zero-based index of the starting row.
279286
* @type int $timeoutMs How long to wait for the query to complete, in
280287
* milliseconds. **Defaults to** `10000` milliseconds (10 seconds).
288+
* @type boolean $formatOptions.useInt64Timestamp Optional. Output
289+
* timestamp as usec int64. Default is false.
281290
* }
282291
* @return array
283292
*/

‎BigQuery/src/Table.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/**
34
* Copyright 2016 Google Inc. All Rights Reserved.
45
*
@@ -224,6 +225,8 @@ public function update(array $metadata, array $options = [])
224225
* @type string $pageToken A previously-returned page token used to
225226
* resume the loading of results from a specific point.
226227
* @type int $startIndex Zero-based index of the starting row.
228+
* @type boolean $formatOptions.useInt64Timestamp Optional. Output
229+
* timestamp as usec int64. Default is true.
227230
* }
228231
* @return ItemIterator<array>
229232
* @throws GoogleException
@@ -232,6 +235,7 @@ public function rows(array $options = [])
232235
{
233236
$resultLimit = $this->pluck('resultLimit', $options, false);
234237
$schema = $this->info()['schema']['fields'];
238+
$options += ['formatOptions.useInt64Timestamp' => true];
235239

236240
return new ItemIterator(
237241
new PageIterator(

‎BigQuery/src/ValueMapper.php‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,12 @@ public function fromBigQuery(array $value, array $schema)
114114
case self::TYPE_TIME:
115115
return new Time(new \DateTime($value));
116116
case self::TYPE_TIMESTAMP:
117-
return $this->timestampFromBigQuery($value);
117+
if ($value == '') {
118+
return null;
119+
}
120+
return $this->isInt($value)
121+
? $this->int64TimestampFromBigQuery($value)
122+
: $this->floatTimestampFromBigQuery($value);
118123
case self::TYPE_RECORD:
119124
return $this->recordFromBigQuery($value, $schema['fields']);
120125
case self::TYPE_GEOGRAPHY:
@@ -336,13 +341,13 @@ private function assocArrayToParameter(array $struct)
336341
}
337342

338343
/**
339-
* Converts a timestamp in string format received from BigQuery to a
344+
* Converts a float timestamp in string format received from BigQuery to a
340345
* {@see Timestamp}.
341346
*
342-
* @param string $value The timestamp.
347+
* @param string $value The float timestamp.
343348
* @return Timestamp
344349
*/
345-
private function timestampFromBigQuery($value)
350+
private function floatTimestampFromBigQuery($value)
346351
{
347352
// If the string contains 'E' convert from exponential notation to
348353
// decimal notation. This doesn't cast to a float because precision can
@@ -381,4 +386,43 @@ private function timestampFromBigQuery($value)
381386
)
382387
);
383388
}
389+
390+
/**
391+
* Converts an int64 microseconds in string format received from BigQuery to a
392+
* {@see Timestamp}.
393+
*
394+
* @param string $value The Int64 microseconds in string format.
395+
* @return Timestamp
396+
*/
397+
private function int64TimestampFromBigQuery(string $value): Timestamp
398+
{
399+
$microSeconds = (int) $value;
400+
$seconds = floor($microSeconds / 1000000);
401+
$remainderMicroSeconds = abs($microSeconds % 1000000);
402+
403+
// Handle negative timestamps
404+
if ($microSeconds < 0 && $remainderMicroSeconds > 0) {
405+
$remainderMicroSeconds = 1000000 - $remainderMicroSeconds; // Invert remainder
406+
}
407+
$remainderMicroSeconds = str_pad($remainderMicroSeconds, 6, '0', STR_PAD_LEFT);
408+
409+
$dateTime = \DateTime::createFromFormat('U u', "{$seconds} {$remainderMicroSeconds}");
410+
return new Timestamp($dateTime);
411+
}
412+
413+
/**
414+
* Checks if the input is a valid integer.
415+
* An integer is considered any string which consists solely of digits
416+
* and an optional leading '-' or '+'
417+
*
418+
* @param mixed $value
419+
* @return bool true if $value is a valid integer, false otherwise
420+
*/
421+
private function isInt(mixed $value): bool
422+
{
423+
$value = (string) $value;
424+
return ($value[0] === '-' || $value[0] === '+')
425+
? ctype_digit(substr($value, 1))
426+
: ctype_digit($value);
427+
}
384428
}

‎BigQuery/tests/System/LoadDataAndQueryTest.php‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public function setUp(): void
4949
'Spells' => [
5050
[
5151
'Name' => 'Summon Dragon',
52-
'LastUsed' => self::$client->timestamp(new \DateTime('2000-01-01 23:59:56 UTC')),
52+
'LastUsed' => self::$client->timestamp(new \DateTime('2000-01-01 23:59:56.000001 UTC')),
5353
'DiscoveredBy' => 'Bobby',
5454
'Properties' => [
5555
[
@@ -173,6 +173,54 @@ public function testRunQuery($useLegacySql)
173173
}
174174
}
175175

176+
/**
177+
* @depends testInsertRowToTable
178+
*/
179+
public function testRunQueryUseInt64Timestamp()
180+
{
181+
$queryString = sprintf(
182+
'SELECT Spells FROM `%s.%s`',
183+
self::$dataset->id(),
184+
self::$table->id()
185+
);
186+
$floatTimestampResults = self::$client->runQuery(
187+
self::$client->query($queryString),
188+
['formatOptions.useInt64Timestamp' => false, 'returnRawResults' => true]
189+
);
190+
$int64TimestampResults = self::$client->runQuery(
191+
self::$client->query($queryString),
192+
['formatOptions.useInt64Timestamp' => true, 'returnRawResults' => true]
193+
);
194+
$int64TimestampResultsArrayParam = self::$client->runQuery(
195+
self::$client->query($queryString),
196+
['formatOptions' => ['useInt64Timestamp' => true], 'returnRawResults' => true]
197+
);
198+
$floatTimestampResults->waitUntilComplete();
199+
$int64TimestampResults->waitUntilComplete();
200+
$int64TimestampResultsArrayParam->waitUntilComplete();
201+
202+
$int64TimestampRow = iterator_to_array($int64TimestampResults->rows())[0];
203+
$floatTimestampRow = iterator_to_array($floatTimestampResults->rows())[0];
204+
$int64TimestampRowArrayParam = iterator_to_array($int64TimestampResultsArrayParam->rows())[0];
205+
206+
$this->assertEquals(946771196.000001, (float) $floatTimestampRow['Spells'][0]['v']['f'][1]['v']);
207+
$this->assertEquals(946771196000001, (int) $int64TimestampRow['Spells'][0]['v']['f'][1]['v']);
208+
$this->assertEquals(946771196000001, (int) $int64TimestampRowArrayParam['Spells'][0]['v']['f'][1]['v']);
209+
}
210+
211+
/**
212+
* @depends testInsertRowToTable
213+
*/
214+
public function testListTableDataUseInt64Timestamp()
215+
{
216+
$int64TimestampRow = iterator_to_array(self::$table->rows(['formatOptions.useInt64Timestamp' => true]))[0];
217+
$floatTimestampRow = iterator_to_array(self::$table->rows(['formatOptions.useInt64Timestamp' => false]))[0];
218+
219+
$spells = $this->row['Spells'][0];
220+
$this->assertEquals($spells['LastUsed'], $int64TimestampRow['Spells'][0]['LastUsed']);
221+
$this->assertEquals($spells['LastUsed'], $floatTimestampRow['Spells'][0]['LastUsed']);
222+
}
223+
176224
public function testInsertRowToTableWithDefaultValueExpression()
177225
{
178226
$row = $this->row;

‎BigQuery/tests/Unit/QueryResultsTest.php‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/**
34
* Copyright 2016 Google Inc.
45
*
@@ -21,6 +22,7 @@
2122
use Google\Cloud\BigQuery\Job;
2223
use Google\Cloud\BigQuery\Numeric;
2324
use Google\Cloud\BigQuery\QueryResults;
25+
use Google\Cloud\BigQuery\Timestamp;
2426
use Google\Cloud\BigQuery\ValueMapper;
2527
use Google\Cloud\Core\Testing\TestHelpers;
2628
use PHPUnit\Framework\TestCase;
@@ -41,7 +43,7 @@ class QueryResultsTest extends TestCase
4143
'jobComplete' => true,
4244
'jobReference' => ['location' => 123],
4345
'rows' => [
44-
['f' => [['v' => 'Alton'], ['v' => 1]]]
46+
['f' => [['v' => 'Alton'], ['v' => 1], ['v' => '40969200000000']]]
4547
],
4648
'schema' => [
4749
'fields' => [
@@ -53,6 +55,10 @@ class QueryResultsTest extends TestCase
5355
'name' => 'numeric_value',
5456
'type' => 'NUMERIC'
5557
],
58+
[
59+
'name' => 'timestamp',
60+
'type' => 'TIMESTAMP'
61+
]
5662
]
5763
]
5864
];
@@ -96,6 +102,28 @@ public function testGetsRowsWithoutToken()
96102

97103
public function testGetsRowsWithToken()
98104
{
105+
$this->connection->getQueryResults(Argument::allOf(
106+
Argument::withEntry('projectId', $this->projectId),
107+
Argument::withEntry('jobId', $this->jobId),
108+
Argument::withEntry('formatOptions.useInt64Timestamp', true)
109+
))
110+
->willReturn($this->queryData)
111+
->shouldBeCalledTimes(1);
112+
113+
$queryResults = $this->getQueryResults(
114+
$this->connection,
115+
$this->queryData + ['pageToken' => 'abcd']
116+
);
117+
$rows = iterator_to_array($queryResults->rows(['formatOptions.useInt64Timestamp' => true]));
118+
119+
$this->assertEquals('Alton', $rows[1]['first_name']);
120+
$this->assertInstanceOf(Numeric::class, $rows[1]['numeric_value']);
121+
$this->assertEquals(new Timestamp(new \DateTime('1971-04-20T04:20:00.000000Z')), $rows[1]['timestamp']);
122+
}
123+
124+
public function testGetsRowWithFloatTimestamp()
125+
{
126+
$this->queryData['rows'][0]['f'][2]['v'] = '1.438712914E9';
99127
$this->connection->getQueryResults(Argument::allOf(
100128
Argument::withEntry('projectId', $this->projectId),
101129
Argument::withEntry('jobId', $this->jobId)
@@ -111,6 +139,7 @@ public function testGetsRowsWithToken()
111139

112140
$this->assertEquals('Alton', $rows[1]['first_name']);
113141
$this->assertInstanceOf(Numeric::class, $rows[1]['numeric_value']);
142+
$this->assertEquals(new Timestamp(new \DateTime('2015-08-04 18:28:34Z')), $rows[1]['timestamp']);
114143
}
115144

116145
public function testReturnRawResults()
@@ -123,6 +152,7 @@ public function testReturnRawResults()
123152

124153
$this->assertEquals('Alton', $rows[0]['first_name']);
125154
$this->assertEquals(1, $rows[0]['numeric_value']);
155+
$this->assertEquals('40969200000000', $rows[0]['timestamp']);
126156
}
127157

128158
public function testReturnRawResultsIsFalse()

0 commit comments

Comments
 (0)