branch-4.1: [fix](Reliability)Fix Doris query service fails after the file handles on the BE node are used up #62393 (#62540)

Cherry-picked from #62393

Co-authored-by: Guangming Lu <71873108+LuGuangming@users.noreply.github.com>

Author: github-actions[bot]

be/src/common/config.cpp

@@ -1739,6 +1739,7 @@ DEFINE_Validator(concurrency_stats_dump_interval_ms,
 DEFINE_mBool(cloud_mow_sync_rowsets_when_load_txn_begin, "true");
 
 DEFINE_mBool(enable_cloud_make_rs_visible_on_be, "false");
+DEFINE_mInt32(file_handles_deplenish_frequency_times, "3");
 
 // clang-format off
 #ifdef BE_TEST

be/src/common/config.h

@@ -1787,6 +1787,7 @@ DECLARE_mInt32(concurrency_stats_dump_interval_ms);
 DECLARE_mBool(cloud_mow_sync_rowsets_when_load_txn_begin);
 
 DECLARE_mBool(enable_cloud_make_rs_visible_on_be);
+DECLARE_mInt32(file_handles_deplenish_frequency_times);
 
 #ifdef BE_TEST
 // test s3

be/src/common/metrics/system_metrics.cpp

@@ -26,6 +26,7 @@
 #include <utility>
 
 #include "common/cast_set.h"
+#include "common/config.h"
 #include "runtime/memory/jemalloc_control.h"
 #include "util/cgroup_util.h"
 #include "util/perf_counters.h"
@@ -432,6 +433,18 @@ void SystemMetrics::_update_cpu_metrics() {
         char buf[64];
         LOG(WARNING) << "open /proc/stat failed, errno=" << errno
                      << ", message=" << strerror_r(errno, buf, 64);
+        if (errno == 24) {
+            _file_handle_deplenish_counter++;
+        } else {
+            _file_handle_deplenish_counter = 0;
+        }
+        // Threshold of the number of consecutive failures
+        if (_file_handle_deplenish_counter >= config::file_handles_deplenish_frequency_times) {
+            LOG(FATAL) << "The system file handles are insufficient, causing service exceptions"
+                       << ", BE will exit. please check the configs 'soft nofile'"
+                       << " and 'hard nofile' of /etc/security/limits.conf ";
+            exit(-1);
+        }
         return;
     }
 

be/src/common/metrics/system_metrics.h

@@ -121,6 +121,7 @@ class SystemMetrics {
     IntGauge* max_disk_io_util_percent = nullptr;
     IntGauge* max_network_send_bytes_rate = nullptr;
     IntGauge* max_network_receive_bytes_rate = nullptr;
+    int _file_handle_deplenish_counter = 0;
 };
 
 } // namespace doris