Skip to content

Commit 6be55ed

Browse files
gearonixxgearonixx
committed
feat scylla: SSL, secdist and speculative execution
1 parent b51705c commit 6be55ed

File tree

14 files changed

+560
-86
lines changed

14 files changed

+560
-86
lines changed

‎samples/scylla_service/main.cpp‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
#include <userver/storages/scylla/component.hpp>
66
#include <userver/storages/scylla/operations.hpp>
77
#include <userver/storages/scylla/session.hpp>
8+
#include <userver/storages/secdist/component.hpp>
9+
#include <userver/storages/secdist/provider_component.hpp>
810
#include <userver/utest/using_namespace_userver.hpp>
911
#include <userver/utils/daemon_run.hpp>
1012

@@ -17,7 +19,7 @@ class InsertHandler final : public server::handlers::HttpHandlerBase {
1719

1820
InsertHandler(const components::ComponentConfig& config, const components::ComponentContext& context)
1921
: HttpHandlerBase(config, context),
20-
session_(context.FindComponent<components::Scylla>("scylla-example").GetSession()) {}
22+
session_(context.FindComponent<components::Scylla>("scylla-dbconnection").GetSession()) {}
2123

2224
std::string HandleRequest(server::http::HttpRequest&, server::request::RequestContext&) const override {
2325
auto table = session_->GetTable("basic");
@@ -44,7 +46,7 @@ class SelectHandler final : public server::handlers::HttpHandlerBase {
4446

4547
SelectHandler(const components::ComponentConfig& config, const components::ComponentContext& context)
4648
: HttpHandlerBase(config, context),
47-
session_(context.FindComponent<components::Scylla>("scylla-example").GetSession()) {}
49+
session_(context.FindComponent<components::Scylla>("scylla-secdist").GetSession()) {}
4850

4951
std::string HandleRequest(server::http::HttpRequest&, server::request::RequestContext&) const override {
5052
auto table = session_->GetTable("basic");
@@ -83,7 +85,10 @@ int main(int argc, char* argv[]) {
8385
const auto component_list =
8486
components::MinimalServerComponentList()
8587
.Append<clients::dns::Component>()
86-
.Append<components::Scylla>("scylla-example")
88+
.Append<components::Secdist>()
89+
.Append<components::DefaultSecdistProvider>()
90+
.Append<components::Scylla>("scylla-dbconnection")
91+
.Append<components::Scylla>("scylla-secdist")
8792
.Append<samples::scylladb::InsertHandler>()
8893
.Append<samples::scylladb::SelectHandler>();
8994
return utils::DaemonMain(argc, argv, component_list);

‎samples/scylla_service/static_config.yaml‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,14 @@ components_manager:
3030
level: debug
3131
overflow_behavior: discard
3232

33-
scylla-example:
34-
dbconnection: scylladb
33+
# Option 1: direct connection string (no secdist needed)
34+
scylla-dbconnection:
35+
dbconnection: 127.0.0.1
3536
consistency: local_quorum
3637
serial_consistency: local_serial
3738
request_timeout: 10s
3839
pool_size: 16
39-
app_name: scylla_app_name_4
40+
app_name: scylla_sample
4041
shard_awareness: true
4142
retry_policy: default
4243
speculative_execution:
@@ -45,3 +46,23 @@ components_manager:
4546
delay: 100ms
4647
default_keyspace: examples
4748

49+
# Option 2: hosts resolved via secdist
50+
scylla-secdist:
51+
dbalias: scylla_example
52+
consistency: local_quorum
53+
serial_consistency: local_serial
54+
request_timeout: 10s
55+
pool_size: 16
56+
app_name: scylla_sample
57+
shard_awareness: true
58+
retry_policy: default
59+
speculative_execution:
60+
enabled: false
61+
max_attempts: 2
62+
delay: 100ms
63+
default_keyspace: examples
64+
65+
secdist:
66+
provider: default-secdist-provider
67+
default-secdist-provider:
68+
config: /etc/scylla_service/secdist.json

‎scylla/include/userver/storages/scylla/component.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
#include <userver/components/component_base.hpp>
44
#include <userver/storages/secdist/secdist.hpp>
5+
#include <userver/utils/statistics/entry.hpp>
56

67
#include <string>
78

@@ -26,6 +27,8 @@ class Scylla : public ComponentBase {
2627
std::string dbalias_;
2728
storages::scylla::SessionPtr session_;
2829

30+
31+
utils::statistics::Entry statistics_entry_;
2932
concurrent::AsyncEventSubscriberScope secdist_subscriber_;
3033
};
3134

‎scylla/include/userver/storages/scylla/session.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include <memory>
4+
#include <optional>
45
#include <string>
56
#include <vector>
67

@@ -56,9 +57,8 @@ class Session {
5657
const std::string& hosts,
5758
const SessionConfig& session_config,
5859
dynamic_config::Source config_source,
59-
// ??
60-
// const std::string& contact_points,
61-
clients::dns::Resolver* dns_resolver
60+
clients::dns::Resolver* dns_resolver,
61+
std::optional<SslSecrets> ssl_secrets = std::nullopt
6262
);
6363

6464
friend void DumpMetric(utils::statistics::Writer& writer, const Session& session);

‎scylla/include/userver/storages/scylla/session_config.hpp‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include <chrono>
44
#include <optional>
55
#include <string>
6+
#include <vector>
67

78
#include <userver/yaml_config/yaml_config.hpp>
89

@@ -26,15 +27,13 @@ enum class Consistency : uint16_t {
2627
enum class SerialConsistency : uint16_t { kSerial = 0x0008, kLocalSerial = 0x0009 };
2728

2829
struct SessionSettings final {
29-
static constexpr std::size_t kDefaultInitialSize = 16;
30-
static constexpr std::size_t kDefaultMaxSize = 128;
31-
static constexpr std::size_t kDefaultIdleLimit = 64;
32-
static constexpr std::size_t kDefaultConnectingLimit = 8;
30+
/// Number of I/O threads in the CassSession (handles async I/O)
31+
static constexpr std::size_t kDefaultNumThreadsIo = 1;
32+
/// Per-host core (minimum) connections kept alive
33+
static constexpr std::size_t kDefaultCoreConnectionsPerHost = 1;
3334

34-
size_t initial_size = kDefaultInitialSize;
35-
size_t max_size = kDefaultMaxSize;
36-
size_t idle_limit = kDefaultIdleLimit;
37-
size_t establishing_limit = kDefaultConnectingLimit;
35+
size_t num_threads_io = kDefaultNumThreadsIo;
36+
size_t core_connections_per_host = kDefaultCoreConnectionsPerHost;
3837

3938
void Validate(const std::string& session_id) const;
4039
};
@@ -57,18 +56,21 @@ struct SessionConfig final {
5756

5857
enum class LoadBalancingPolicy {
5958
kRoundRobin,
60-
kTokenAwareRoundRobin,
61-
kDcAwareRoundRobin,
59+
kDcAware,
6260
};
63-
LoadBalancingPolicy load_balancing_policy = LoadBalancingPolicy::kTokenAwareRoundRobin;
61+
LoadBalancingPolicy load_balancing_policy = LoadBalancingPolicy::kDcAware;
6462
std::string preferred_datacenter;
6563

64+
/// Token-aware routing: sends queries directly to the replica that owns
65+
/// the partition key. In ScyllaDB, this also enables shard-aware routing
66+
// (sends to the exact CPU shard within the node).
67+
/// Sits on top of the base load balancing policy.
68+
bool token_aware_routing = true;
69+
6670
SessionSettings dynamic_settings;
6771

6872
void Validate(const std::string& session_id) const;
6973

70-
bool shard_awareness = true;
71-
7274
enum class RetryPolicyType {
7375
kDefault,
7476
kFallthrough,
@@ -85,9 +87,32 @@ struct SessionConfig final {
8587
std::string default_keyspace;
8688

8789
std::string app_name = kDefaultAppName;
90+
91+
struct SslSettings {
92+
bool enabled = false;
93+
94+
enum class VerifyMode {
95+
kNone,
96+
kPeerCert,
97+
kPeerIdentity,
98+
kPeerIdentityDns,
99+
};
100+
VerifyMode verify = VerifyMode::kPeerCert;
101+
};
102+
SslSettings ssl;
88103
};
89104

90105
SessionConfig Parse(const yaml_config::YamlConfig& config, formats::parse::To<SessionConfig>);
106+
107+
/// TLS certificate material from secdist. Kept alongside SessionConfig
108+
/// because the two are consumed together when building a CassCluster.
109+
struct SslSecrets {
110+
std::vector<std::string> trusted_certs;
111+
std::optional<std::string> client_cert;
112+
std::optional<std::string> client_key;
113+
std::string client_key_password;
114+
};
115+
91116
} // namespace storages::scylla
92117

93118
USERVER_NAMESPACE_END

‎scylla/src/storages/scylla/component.cpp‎

Lines changed: 41 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,24 @@
1-
#include <iostream>
2-
31
#include <userver/storages/scylla/component.hpp>
42

3+
#include <boost/algorithm/string/predicate.hpp>
4+
5+
#include <userver/clients/dns/resolver_utils.hpp>
56
#include <userver/components/component.hpp>
6-
#include <userver/storages/scylla/exception.hpp>
7+
#include <userver/components/statistics_storage.hpp>
78
#include <userver/dynamic_config/storage/component.hpp>
9+
#include <userver/storages/scylla/exception.hpp>
10+
#include <userver/storages/scylla/session.hpp>
11+
#include <userver/storages/scylla/session_config.hpp>
812
#include <userver/yaml_config/merge_schemas.hpp>
913

14+
#include <storages/scylla/scylla_secdist.hpp>
15+
16+
#include <userver/storages/secdist/component.hpp>
17+
1018
#ifndef ARCADIA_ROOT
1119
#include "generated/src/storages/scylla/component.yaml.hpp" // Y_IGNORE
1220
#endif
1321

14-
#include "userver/storages/secdist/component.hpp"
15-
16-
#include <boost/algorithm/string/case_conv.hpp>
17-
#include <boost/algorithm/string/predicate.hpp>
18-
#include <userver/storages/scylla/session.hpp>
19-
#include <userver/storages/scylla/session_config.hpp>
20-
21-
#include <userver/clients/dns/resolver_utils.hpp>
22-
#include <userver/components/statistics_storage.hpp>
23-
2422
USERVER_NAMESPACE_BEGIN
2523

2624
namespace components {
@@ -46,10 +44,8 @@ Scylla::Scylla(const ComponentConfig& config, const ComponentContext& context) :
4644
if (!db_alias.empty()) {
4745
dbalias_ = db_alias;
4846
secdist = &context.FindComponent<Secdist>().GetStorage();
49-
50-
hosts = "???";
47+
hosts = storages::scylla::secdist::GetSecdistHosts(*secdist, dbalias_);
5148
} else {
52-
// TODO: rename to hosts
5349
hosts = config["dbconnection"].As<std::string>();
5450
}
5551

@@ -63,28 +59,46 @@ Scylla::Scylla(const ComponentConfig& config, const ComponentContext& context) :
6359
const auto dynamic_config = context.FindComponent<DynamicConfig>().GetSource();
6460
const auto session_config = ParseSessionConfig(config);
6561

66-
session_ = std::make_shared<
67-
storages::scylla::Session>(config.Name(), hosts, session_config, dynamic_config, dns_resolver);
68-
69-
const auto& statistics_storage = context.FindComponent<components::StatisticsStorage>();
62+
session_ = std::make_shared<storages::scylla::Session>(
63+
config.Name(), hosts, session_config, dynamic_config, dns_resolver);
7064

71-
auto component_name = config.Name();
65+
if (!dbalias_.empty()) {
66+
secdist_subscriber_ = secdist->UpdateAndListen(this, dbalias_, &Scylla::OnSecdistUpdate);
67+
}
7268

73-
const bool has_name_after_prefix = component_name.size() > kStandardScyllaPrefix.size();
74-
const bool has_scylla_prefix = boost::algorithm::starts_with(component_name, kStandardScyllaPrefix);
69+
auto& statistics_storage = context.FindComponent<components::StatisticsStorage>();
7570

76-
if (has_scylla_prefix && has_name_after_prefix) {
77-
component_name = component_name.substr(kStandardScyllaPrefix.size());
71+
auto section_name = config.Name();
72+
if (boost::algorithm::starts_with(section_name, kStandardScyllaPrefix) &&
73+
section_name.size() != kStandardScyllaPrefix.size()) {
74+
section_name = section_name.substr(kStandardScyllaPrefix.size());
7875
}
79-
};
76+
77+
statistics_entry_ = statistics_storage.GetStorage().RegisterWriter(
78+
"scylla",
79+
[this](utils::statistics::Writer& writer) {
80+
UASSERT(session_);
81+
DumpMetric(writer, *session_);
82+
},
83+
{{"scylla_database", std::move(section_name)}}
84+
);
85+
}
8086

8187
storages::scylla::SessionPtr Scylla::GetSession() const { return session_; }
8288

8389
yaml_config::Schema Scylla::GetStaticConfigSchema() {
8490
return yaml_config::MergeSchemasFromResource<ComponentBase>("src/storages/scylla/component.yaml");
8591
}
8692

87-
Scylla::~Scylla() = default;
93+
void Scylla::OnSecdistUpdate(const storages::secdist::SecdistConfig& config) {
94+
auto hosts = storages::scylla::secdist::GetSecdistHosts(config, dbalias_);
95+
session_->SetContactPoints(hosts);
96+
}
97+
98+
Scylla::~Scylla() {
99+
statistics_entry_.Unregister();
100+
secdist_subscriber_.Unsubscribe();
101+
}
88102
} // namespace components
89103

90104
USERVER_NAMESPACE_END

‎scylla/src/storages/scylla/component.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,24 @@ properties:
9090
type: string
9191
description: Default keyspace for queries. If empty, table names must be fully qualified (e.g. keyspace.table).
9292
default: ''
93+
ssl:
94+
type: object
95+
description: TLS/SSL settings
96+
additionalProperties: false
97+
properties:
98+
enabled:
99+
type: boolean
100+
description: enable TLS for cluster connections
101+
default: false
102+
verify:
103+
type: string
104+
description: certificate verification mode
105+
default: peer_cert
106+
enum:
107+
- none
108+
- peer_cert
109+
- peer_identity
110+
- peer_identity_dns
93111
dns_resolver:
94112
type: string
95113
description: server hostname resolver type

0 commit comments

Comments
 (0)