feat postgres: new error fallback strategy in postgres connections auto-limiter

Implements new error fallback strategy for ConnlimitWatchdog. It could be enabled under pg-connlimit-watchdog-new-fallback userver experiment.
commit_hash:ad81a29c7e4008046c24e098af4709c34d9ed164

Author: lexeyo

core/include/userver/utils/impl/userver_experiments.hpp

@@ -59,6 +59,7 @@ extern UserverExperiment kServerSelectionTimeoutExperiment;
 extern UserverExperiment kPgCcExperiment;
 extern UserverExperiment kYdbDeadlinePropagationExperiment;
 extern UserverExperiment kWaitAllCheckedUpgradeExperiment;
+extern UserverExperiment kPgConnlimitWatchdogFallbackExperiment;
 
 }  // namespace utils::impl
 

core/src/utils/impl/userver_experiments.cpp

@@ -95,6 +95,7 @@ UserverExperiment kServerSelectionTimeoutExperiment{"mongo-server-selection-time
 UserverExperiment kPgCcExperiment{"pg-cc"};
 UserverExperiment kYdbDeadlinePropagationExperiment{"ydb-deadline-propagation"};
 UserverExperiment kWaitAllCheckedUpgradeExperiment{"wait-all-checked-upgrade"};
+UserverExperiment kPgConnlimitWatchdogFallbackExperiment{"pg-connlimit-watchdog-new-fallback"};
 
 }  // namespace utils::impl
 

postgresql/src/storages/postgres/connlimit_watchdog.cpp

@@ -1,7 +1,9 @@
+#include <algorithm>
 #include <storages/postgres/connlimit_watchdog.hpp>
 
 #include <storages/postgres/detail/cluster_impl.hpp>
 #include <userver/utils/from_string.hpp>
+#include <userver/utils/impl/userver_experiments.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -39,6 +41,7 @@ ConnlimitWatchdog::ConnlimitWatchdog(
     detail::ClusterImpl& cluster,
     testsuite::TestsuiteTasks& testsuite_tasks,
     int shard_number,
+    std::size_t min_fallback_connections,
     std::function<void()> on_new_connlimit,
     std::string host_name
 )
@@ -47,6 +50,7 @@ ConnlimitWatchdog::ConnlimitWatchdog(
       on_new_connlimit_(std::move(on_new_connlimit)),
       testsuite_tasks_(testsuite_tasks),
       shard_number_(shard_number),
+      min_fallback_connections_(std::max(min_fallback_connections, kDefaultPoolMinSize)),
       host_name_(std::move(host_name))
 {}
 
@@ -158,7 +162,13 @@ void ConnlimitWatchdog::DoStep(
              * connlimit value. The period with "too low max_connections" should be
              * relatively small.
              */
-            connlimit_ = kFallbackConnlimit;
+            if (!USERVER_NAMESPACE::utils::impl::kPgConnlimitWatchdogFallbackExperiment.IsEnabled()) {
+                connlimit_ = kFallbackConnlimit;
+            } else {
+                const auto previous_connlimit = connlimit_.load();
+                connlimit_ = std::max(previous_connlimit / 2, min_fallback_connections_);
+                steps_with_errors_ = 0;
+            }
         }
     }
 

postgresql/src/storages/postgres/connlimit_watchdog.hpp

@@ -23,6 +23,7 @@ class ConnlimitWatchdog final {
         detail::ClusterImpl& cluster,
         testsuite::TestsuiteTasks& testsuite_tasks,
         int shard_number,
+        std::size_t min_fallback_connections,
         std::function<void()> on_new_connlimit,
         std::string host_name = hostinfo::blocking::GetRealHostName()
     );
@@ -55,6 +56,7 @@ class ConnlimitWatchdog final {
     int steps_with_errors_{0};
     USERVER_NAMESPACE::utils::PeriodicTask periodic_;
     int shard_number_;
+    std::size_t min_fallback_connections_;
     std::string host_name_;
 };
 

postgresql/src/storages/postgres/detail/cluster_impl.cpp

@@ -100,7 +100,13 @@ ClusterImpl::ClusterImpl(
       ei_settings_(ei_settings),
       metrics_(std::move(metrics)),
       rr_host_idx_(0),
-      connlimit_watchdog_(*this, testsuite_tasks, shard_number, [this]() { OnConnlimitChanged(); })
+      connlimit_watchdog_(
+          *this,
+          testsuite_tasks,
+          shard_number,
+          cluster_settings.pool_settings.min_size,
+          [this]() { OnConnlimitChanged(); }
+      )
 {
     CreateTopology(dsns);
 

postgresql/src/storages/postgres/tests/connlimit_watchdog_pgtest.cpp

@@ -70,6 +70,8 @@ pg::Transaction GetTransaction(pgd::ClusterImpl& cluster) {
 
 constexpr size_t kReservedConn = 5;
 constexpr size_t kTestsuiteConnlimit = 100 - kReservedConn;
+constexpr size_t kFallbackConnlimit = 17;
+constexpr size_t kMaxStepsWithError = 3;
 
 enum class MigrationVersion { kV1 = 0, kV2 = 1, kCount };
 
@@ -93,20 +95,26 @@ class Watchdog : public PostgreSQLBase {
 
     std::size_t DoStepV1() {
         // This watchdog use the native host like the watchdog in ClusterImpl.
-        pg::ConnlimitWatchdog connlimit_watchdog_v1{cluster_, testsuite_tasks_, kShardNumber, [] {}};
+        auto connlimit_watchdog_v1 = MakeConnlimitWatchdog();
         connlimit_watchdog_v1.StepV1();
         return connlimit_watchdog_v1.GetConnlimit();
     }
 
     std::size_t DoStepV2() {
         // Use different host names to emulate different hosts.
-        pg::ConnlimitWatchdog connlimit_watchdog_v2{cluster_, testsuite_tasks_, kShardNumber, [] {}, "host2"};
+        auto connlimit_watchdog_v2 = MakeConnlimitWatchdog("host2");
         connlimit_watchdog_v2.StepV2();
         return connlimit_watchdog_v2.GetConnlimit();
     }
 
+    pg::ConnlimitWatchdog MakeConnlimitWatchdog(std::string host_name = hostinfo::blocking::GetRealHostName()) {
+        return pg::ConnlimitWatchdog{cluster_, testsuite_tasks_, kShardNumber, kFallbackConnlimit, [] {}, host_name};
+    }
+
     pgd::ClusterImpl& GetCluster() { return cluster_; }
 
+    void SetUp() override { scope_.Set(utils::impl::kPgConnlimitWatchdogFallbackExperiment, true); }
+
 private:
     void ClearTable() {
         auto t = GetTransaction(cluster_);
@@ -116,6 +124,7 @@ class Watchdog : public PostgreSQLBase {
 
     testsuite::TestsuiteTasks testsuite_tasks_{true};
     pgd::ClusterImpl cluster_;
+    utils::impl::UserverExperimentsScope scope_;
 };
 
 UTEST_F(Watchdog, Basic) {
@@ -215,4 +224,23 @@ UTEST_F(Watchdog, MultiUsersWithV2) {
     EXPECT_EQ(kTestsuiteConnlimit / 2, DoStepV2());
 }
 
+UTEST_F(Watchdog, FallbackConnlimit) {
+    auto expected_connlimit = kTestsuiteConnlimit;
+    auto watchdog = MakeConnlimitWatchdog();
+    // Do single step with working connection
+    watchdog.StepV2();
+    // Update connection to a non-working one
+    GetCluster().SetDsnList({GetUnavailableDsn()});
+
+    while (expected_connlimit >= kFallbackConnlimit) {
+        for (size_t i = 0; i <= kMaxStepsWithError; ++i) {
+            ASSERT_EQ(expected_connlimit, watchdog.GetConnlimit());
+            watchdog.StepV2();
+        }
+        expected_connlimit /= 2;
+    }
+
+    ASSERT_EQ(kFallbackConnlimit, watchdog.GetConnlimit());
+}
+
 USERVER_NAMESPACE_END