Skip to content

Commit e693d99

Browse files
pavan-traceablepavan-traceable
authored
Filtering pagination support impl (#286)
* Add generic api's for filtering, pagination in getAllConfigs call
1 parent e8186d0 commit e693d99

8 files changed

Lines changed: 376 additions & 20 deletions

File tree

‎config-service-api/src/main/proto/org/hypertrace/config/service/v1/config_service.proto‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,18 @@ message GetAllConfigsRequest {
8888

8989
// required - namespace with which the config resource is associated
9090
string resource_namespace = 2;
91+
92+
// optional - filtering criteria to narrow down the configs.
93+
// Supports relational and logical operators on config fields.
94+
Filter filter = 3;
95+
96+
// optional - list of sorting conditions to order the results.
97+
// Multiple SortBy entries are applied in the specified order of priority.
98+
repeated SortBy sort_by = 4;
99+
100+
// optional - pagination parameters to limit and offset the result set.
101+
// Useful for retrieving configs in pages when total count is large.
102+
Pagination pagination = 5;
91103
}
92104

93105
message GetAllConfigsResponse {
@@ -201,3 +213,25 @@ enum LogicalOperator {
201213
LOGICAL_OPERATOR_AND = 1;
202214
LOGICAL_OPERATOR_OR = 2;
203215
}
216+
217+
message SortBy {
218+
Selection selection = 1;
219+
SortOrder sort_order = 2;
220+
}
221+
222+
message Pagination {
223+
int32 limit = 1;
224+
int32 offset = 2;
225+
}
226+
227+
message Selection {
228+
oneof type {
229+
string config_json_path = 1;
230+
}
231+
}
232+
233+
enum SortOrder {
234+
SORT_ORDER_UNSPECIFIED = 0;
235+
SORT_ORDER_ASC = 1;
236+
SORT_ORDER_DESC = 2;
237+
}

‎config-service-impl/src/main/java/org/hypertrace/config/service/ConfigServiceGrpcImpl.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,10 @@ public void getAllConfigs(
118118
List<ContextSpecificConfig> contextSpecificConfigList =
119119
configStore.getAllConfigs(
120120
new ConfigResource(
121-
request.getResourceName(), request.getResourceNamespace(), getTenantId()));
121+
request.getResourceName(), request.getResourceNamespace(), getTenantId()),
122+
request.getFilter(),
123+
request.getPagination(),
124+
request.getSortByList());
122125
responseObserver.onNext(
123126
GetAllConfigsResponse.newBuilder()
124127
.addAllContextSpecificConfigs(contextSpecificConfigList)

‎config-service-impl/src/main/java/org/hypertrace/config/service/store/ConfigStore.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
import org.hypertrace.config.service.ConfigResource;
1010
import org.hypertrace.config.service.ConfigResourceContext;
1111
import org.hypertrace.config.service.v1.ContextSpecificConfig;
12+
import org.hypertrace.config.service.v1.Filter;
13+
import org.hypertrace.config.service.v1.Pagination;
14+
import org.hypertrace.config.service.v1.SortBy;
1215
import org.hypertrace.config.service.v1.UpsertAllConfigsResponse.UpsertedConfig;
1316
import org.hypertrace.config.service.v1.UpsertConfigRequest;
1417

@@ -57,10 +60,15 @@ Map<ConfigResourceContext, ContextSpecificConfig> getContextConfigs(
5760
* specified parameters, sorted in the descending order of their creation time.
5861
*
5962
* @param configResource
63+
* @param filter
64+
* @param pagination
65+
* @param sortByList
6066
* @return
6167
* @throws IOException
6268
*/
63-
List<ContextSpecificConfig> getAllConfigs(ConfigResource configResource) throws IOException;
69+
List<ContextSpecificConfig> getAllConfigs(
70+
ConfigResource configResource, Filter filter, Pagination pagination, List<SortBy> sortByList)
71+
throws IOException;
6472

6573
/**
6674
* Write each of the provided config value associated with the specified config resource to the

‎config-service-impl/src/main/java/org/hypertrace/config/service/store/ConstantExpressionConverter.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package org.hypertrace.config.service.store;
2+
3+
import com.google.protobuf.Value;
4+
import io.grpc.Status;
5+
import java.util.List;
6+
import java.util.stream.Collectors;
7+
import org.hypertrace.core.documentstore.expression.impl.ConstantExpression;
8+
9+
public class ConstantExpressionConverter {
10+
11+
public static ConstantExpression fromProtoValue(Value value) {
12+
switch (value.getKindCase()) {
13+
case STRING_VALUE:
14+
return ConstantExpression.of(value.getStringValue());
15+
case NUMBER_VALUE:
16+
return ConstantExpression.of(value.getNumberValue());
17+
case BOOL_VALUE:
18+
return ConstantExpression.of(value.getBoolValue());
19+
case LIST_VALUE:
20+
List<Value> values = value.getListValue().getValuesList();
21+
if (values.isEmpty()) {
22+
// Default to empty string list — or change logic based on expected behavior
23+
return ConstantExpression.ofStrings(List.of());
24+
}
25+
26+
Value.KindCase elementType = values.get(0).getKindCase();
27+
boolean isHomogeneous = values.stream().allMatch(v -> v.getKindCase() == elementType);
28+
29+
if (!isHomogeneous) {
30+
throw Status.INVALID_ARGUMENT
31+
.withDescription("List contains mixed types. All elements must be of the same type.")
32+
.asRuntimeException();
33+
}
34+
switch (elementType) {
35+
case STRING_VALUE:
36+
return ConstantExpression.ofStrings(
37+
values.stream().map(Value::getStringValue).collect(Collectors.toList()));
38+
case NUMBER_VALUE:
39+
return ConstantExpression.ofNumbers(
40+
values.stream().map(Value::getNumberValue).collect(Collectors.toList()));
41+
case BOOL_VALUE:
42+
return ConstantExpression.ofBooleans(
43+
values.stream().map(Value::getBoolValue).collect(Collectors.toList()));
44+
default:
45+
throw Status.UNIMPLEMENTED
46+
.withDescription("Unsupported list element type: " + elementType)
47+
.asRuntimeException();
48+
}
49+
case STRUCT_VALUE:
50+
throw Status.UNIMPLEMENTED
51+
.withDescription("Struct not supported directly in ConstantExpression")
52+
.asRuntimeException();
53+
case NULL_VALUE:
54+
case KIND_NOT_SET:
55+
default:
56+
throw Status.INVALID_ARGUMENT
57+
.withDescription("Unsupported or null value in ConstantExpression")
58+
.asRuntimeException();
59+
}
60+
}
61+
}

‎config-service-impl/src/main/java/org/hypertrace/config/service/store/DocumentConfigStore.java‎

Lines changed: 79 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,14 @@
2424
import java.util.Optional;
2525
import java.util.Set;
2626
import java.util.stream.Collectors;
27+
import lombok.NonNull;
28+
import lombok.SneakyThrows;
2729
import lombok.extern.slf4j.Slf4j;
2830
import org.hypertrace.config.service.ConfigResource;
2931
import org.hypertrace.config.service.ConfigResourceContext;
3032
import org.hypertrace.config.service.ConfigServiceUtils;
3133
import org.hypertrace.config.service.v1.ContextSpecificConfig;
34+
import org.hypertrace.config.service.v1.SortBy;
3235
import org.hypertrace.config.service.v1.UpsertAllConfigsResponse.UpsertedConfig;
3336
import org.hypertrace.config.service.v1.UpsertConfigRequest;
3437
import org.hypertrace.core.documentstore.CloseableIterator;
@@ -58,12 +61,14 @@ public class DocumentConfigStore implements ConfigStore {
5861
private final Datastore datastore;
5962
private final Collection collection;
6063
private final FilterBuilder filterBuilder;
64+
private final FilterExpressionBuilder filterExpressionBuilder;
6165

6266
public DocumentConfigStore(Clock clock, Datastore datastore) {
6367
this.clock = clock;
6468
this.datastore = datastore;
6569
this.collection = this.datastore.getCollection(CONFIGURATIONS_COLLECTION);
6670
this.filterBuilder = new FilterBuilder();
71+
this.filterExpressionBuilder = new FilterExpressionBuilder();
6772
}
6873

6974
@Override
@@ -208,30 +213,88 @@ public Map<ConfigResourceContext, ContextSpecificConfig> getContextConfigs(
208213
}
209214

210215
@Override
211-
public List<ContextSpecificConfig> getAllConfigs(ConfigResource configResource)
216+
public List<ContextSpecificConfig> getAllConfigs(
217+
ConfigResource configResource,
218+
org.hypertrace.config.service.v1.Filter filter,
219+
org.hypertrace.config.service.v1.Pagination pagination,
220+
List<SortBy> sortByList)
212221
throws IOException {
213-
Query query =
214-
Query.builder()
215-
.addSort(IdentifierExpression.of(VERSION_FIELD_NAME), SortOrder.DESC)
216-
.setFilter(getConfigResourceFilterTypeExpression(configResource))
217-
.build();
218-
List<ContextSpecificConfig> contextSpecificConfigList = new ArrayList<>();
222+
223+
Query query = buildQuery(configResource, filter, pagination, sortByList);
224+
List<ContextSpecificConfig> configList = new ArrayList<>();
219225
Set<String> seenContexts = new HashSet<>();
226+
220227
try (CloseableIterator<Document> documentIterator =
221228
collection.query(query, QueryOptions.DEFAULT_QUERY_OPTIONS)) {
222229
while (documentIterator.hasNext()) {
223-
String documentString = documentIterator.next().toJson();
224-
ConfigDocument configDocument = ConfigDocument.fromJson(documentString);
225-
String context = configDocument.getContext();
226-
if (seenContexts.add(context)) {
227-
convertToContextSpecificConfig(configDocument).ifPresent(contextSpecificConfigList::add);
228-
}
230+
processDocument(documentIterator.next(), seenContexts, configList);
229231
}
230232
}
231-
Collections.sort(
232-
contextSpecificConfigList,
233+
234+
configList.sort(
233235
Comparator.comparingLong(ContextSpecificConfig::getCreationTimestamp).reversed());
234-
return contextSpecificConfigList;
236+
return configList;
237+
}
238+
239+
private Query buildQuery(
240+
ConfigResource configResource,
241+
@NonNull org.hypertrace.config.service.v1.Filter filter,
242+
@NonNull org.hypertrace.config.service.v1.Pagination pagination,
243+
List<SortBy> sortByList) {
244+
245+
FilterTypeExpression combinedFilter = getCombinedFilter(configResource, filter);
246+
Query.QueryBuilder queryBuilder = Query.builder().setFilter(combinedFilter);
247+
if (!pagination.equals(org.hypertrace.config.service.v1.Pagination.getDefaultInstance())) {
248+
queryBuilder.setPagination(
249+
Pagination.builder().offset(pagination.getOffset()).limit(pagination.getLimit()).build());
250+
}
251+
252+
if (!sortByList.isEmpty()) {
253+
sortByList.forEach(
254+
sortBy ->
255+
queryBuilder.addSort(
256+
IdentifierExpression.of(sortBy.getSelection().getConfigJsonPath()),
257+
convertSortOrder(sortBy)));
258+
} else {
259+
queryBuilder.addSort(IdentifierExpression.of(VERSION_FIELD_NAME), SortOrder.DESC);
260+
}
261+
return queryBuilder.build();
262+
}
263+
264+
private FilterTypeExpression getCombinedFilter(
265+
ConfigResource configResource,
266+
@NonNull org.hypertrace.config.service.v1.Filter additionalFilter) {
267+
268+
FilterTypeExpression resourceFilter = getConfigResourceFilterTypeExpression(configResource);
269+
if (additionalFilter.equals(org.hypertrace.config.service.v1.Filter.getDefaultInstance())) {
270+
return resourceFilter;
271+
}
272+
273+
FilterTypeExpression docStoreFilter =
274+
filterExpressionBuilder.buildFilterTypeExpression(additionalFilter);
275+
return LogicalExpression.and(resourceFilter, docStoreFilter);
276+
}
277+
278+
@SneakyThrows
279+
private void processDocument(
280+
Document document, Set<String> seenContexts, List<ContextSpecificConfig> configList) {
281+
282+
ConfigDocument configDocument = ConfigDocument.fromJson(document.toJson());
283+
String context = configDocument.getContext();
284+
285+
if (seenContexts.add(context)) {
286+
convertToContextSpecificConfig(configDocument).ifPresent(configList::add);
287+
}
288+
}
289+
290+
private static SortOrder convertSortOrder(SortBy sortBy) {
291+
switch (sortBy.getSortOrder()) {
292+
case SORT_ORDER_DESC:
293+
return SortOrder.DESC;
294+
case SORT_ORDER_ASC:
295+
default:
296+
return SortOrder.ASC;
297+
}
235298
}
236299

237300
@Override

‎config-service-impl/src/main/java/org/hypertrace/config/service/store/FilterExpressionBuilder.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package org.hypertrace.config.service.store;
2+
3+
import static org.hypertrace.config.service.store.ConfigDocument.CONFIG_FIELD_NAME;
4+
5+
import io.grpc.Status;
6+
import java.util.List;
7+
import java.util.stream.Collectors;
8+
import org.hypertrace.config.service.v1.Filter;
9+
import org.hypertrace.config.service.v1.LogicalFilter;
10+
import org.hypertrace.config.service.v1.RelationalFilter;
11+
import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression;
12+
import org.hypertrace.core.documentstore.expression.impl.LogicalExpression;
13+
import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
14+
import org.hypertrace.core.documentstore.expression.operators.LogicalOperator;
15+
import org.hypertrace.core.documentstore.expression.operators.RelationalOperator;
16+
import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression;
17+
18+
public class FilterExpressionBuilder {
19+
20+
public FilterTypeExpression buildFilterTypeExpression(Filter filter) {
21+
switch (filter.getTypeCase()) {
22+
case LOGICAL_FILTER:
23+
return buildLogicalExpression(filter.getLogicalFilter());
24+
case RELATIONAL_FILTER:
25+
return buildRelationalExpression(filter.getRelationalFilter());
26+
case TYPE_NOT_SET:
27+
default:
28+
throw Status.INVALID_ARGUMENT.withDescription("Filter type unset").asRuntimeException();
29+
}
30+
}
31+
32+
private FilterTypeExpression buildLogicalExpression(LogicalFilter logicalFilter) {
33+
List<FilterTypeExpression> childExpressions =
34+
logicalFilter.getOperandsList().stream()
35+
.map(this::buildFilterTypeExpression)
36+
.collect(Collectors.toUnmodifiableList());
37+
38+
LogicalOperator operator;
39+
switch (logicalFilter.getOperator()) {
40+
case LOGICAL_OPERATOR_AND:
41+
operator = LogicalOperator.AND;
42+
break;
43+
case LOGICAL_OPERATOR_OR:
44+
operator = LogicalOperator.OR;
45+
break;
46+
case LOGICAL_OPERATOR_UNSPECIFIED:
47+
default:
48+
throw Status.INVALID_ARGUMENT
49+
.withDescription("Unknown logical operator while building expression")
50+
.asRuntimeException();
51+
}
52+
return LogicalExpression.builder().operator(operator).operands(childExpressions).build();
53+
}
54+
55+
private FilterTypeExpression buildRelationalExpression(RelationalFilter relationalFilter) {
56+
RelationalOperator operator;
57+
switch (relationalFilter.getOperator()) {
58+
case RELATIONAL_OPERATOR_EQ:
59+
operator = RelationalOperator.EQ;
60+
break;
61+
case RELATIONAL_OPERATOR_NEQ:
62+
operator = RelationalOperator.NEQ;
63+
break;
64+
case RELATIONAL_OPERATOR_IN:
65+
operator = RelationalOperator.IN;
66+
break;
67+
case RELATIONAL_OPERATOR_NOT_IN:
68+
operator = RelationalOperator.NOT_IN;
69+
break;
70+
case RELATIONAL_OPERATOR_LT:
71+
operator = RelationalOperator.LT;
72+
break;
73+
case RELATIONAL_OPERATOR_GT:
74+
operator = RelationalOperator.GT;
75+
break;
76+
case RELATIONAL_OPERATOR_LTE:
77+
operator = RelationalOperator.LTE;
78+
break;
79+
case RELATIONAL_OPERATOR_GTE:
80+
operator = RelationalOperator.GTE;
81+
break;
82+
case UNRECOGNIZED:
83+
default:
84+
throw Status.INVALID_ARGUMENT
85+
.withDescription("Unknown relational operator while building expression")
86+
.asRuntimeException();
87+
}
88+
89+
return RelationalExpression.of(
90+
IdentifierExpression.of(buildConfigFieldPath(relationalFilter.getConfigJsonPath())),
91+
operator,
92+
ConstantExpressionConverter.fromProtoValue(relationalFilter.getValue()));
93+
}
94+
95+
private String buildConfigFieldPath(String configJsonPath) {
96+
return String.format("%s.%s", CONFIG_FIELD_NAME, configJsonPath);
97+
}
98+
}

0 commit comments

Comments
 (0)