Merge pull request #296 from coroot/async_profiler

add async-profiler support for Java applications (disabled by default)

Author: def

.github/workflows/ci.yml

@@ -13,8 +13,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.23.8'
-      - run: sudo apt install -y libsystemd-dev
+          go-version: '1.24.9'
+      - run: sudo apt update && sudo apt install -y libsystemd-dev
       - name: gofmt -l .
         run: files=$(gofmt -l .); if [[ -n "$files" ]]; then echo "$files"; exit 1; fi
       - name: goimports -l .

containers/container.go

@@ -139,6 +139,8 @@ type Container struct {
 	nodejsStats *ebpftracer.NodejsStats
 	pythonStats *ebpftracer.PythonStats
 
+	jvmProfilingStats *JvmProfilingStats
+
 	mounts     map[string]proc.MountInfo
 	seenMounts map[uint64]struct{}
 
@@ -373,7 +375,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 		}
 		switch {
 		case proc.IsJvm(cmdline):
-			jvm, jMetrics := jvmMetrics(pid)
+			jvm, jMetrics := jvmMetrics(pid, c)
 			if len(jMetrics) > 0 && !seenJvms[jvm] {
 				seenJvms[jvm] = true
 				for _, m := range jMetrics {
@@ -846,6 +848,18 @@ func (c *Container) updateDelays() {
 	}
 }
 
+func (c *Container) updateJvmProfilingStats(u *JvmProfilingUpdate) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if c.jvmProfilingStats == nil {
+		c.jvmProfilingStats = &JvmProfilingStats{}
+	}
+	c.jvmProfilingStats.AllocBytes += u.AllocBytes
+	c.jvmProfilingStats.AllocObjects += u.AllocObjects
+	c.jvmProfilingStats.LockContentions += u.LockContentions
+	c.jvmProfilingStats.LockTimeNs += u.LockTimeNs
+}
+
 func (c *Container) updateNodejsStats(s NodejsStatsUpdate) {
 	c.lock.Lock()
 	defer c.lock.Unlock()

containers/jvm.go

@@ -7,13 +7,14 @@ import (
 	"time"
 
 	"github.com/coroot/coroot-node-agent/common"
+	"github.com/coroot/coroot-node-agent/flags"
 	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/xin053/hsperfdata"
 	"k8s.io/klog/v2"
 )
 
-func jvmMetrics(pid uint32) (string, []prometheus.Metric) {
+func jvmMetrics(pid uint32, c *Container) (string, []prometheus.Metric) {
 	nsPid, err := proc.GetNsPid(pid)
 	if err != nil {
 		if !common.IsNotExist(err) {
@@ -42,15 +43,20 @@ func jvmMetrics(pid uint32) (string, []prometheus.Metric) {
 	func() {
 		size := float64(0)
 		used := float64(0)
+		maxSize := float64(0)
 		for _, gen := range []int{0, 1} {
 			spaces := pd.getInt64("sun.gc.generation.%d.spaces", gen)
 			for s := 0; s < int(spaces); s++ {
 				size += float64(pd.getInt64("sun.gc.generation.%d.space.%d.capacity", gen, s))
 				used += float64(pd.getInt64("sun.gc.generation.%d.space.%d.used", gen, s))
+				maxSize += float64(pd.getInt64("sun.gc.generation.%d.space.%d.maxCapacity", gen, s))
 			}
 		}
 		res = append(res, gauge(metrics.JvmHeapSize, size, jvm))
 		res = append(res, gauge(metrics.JvmHeapUsed, used, jvm))
+		if maxSize > 0 {
+			res = append(res, gauge(metrics.JvmHeapMaxSize, maxSize, jvm))
+		}
 	}()
 
 	gc := func(prefix string) {
@@ -66,6 +72,19 @@ func jvmMetrics(pid uint32) (string, []prometheus.Metric) {
 
 	res = append(res, counter(metrics.JvmSafepointTime, time.Duration(pd.getInt64("sun.rt.safepointTime")).Seconds(), jvm))
 	res = append(res, counter(metrics.JvmSafepointSyncTime, time.Duration(pd.getInt64("sun.rt.safepointSyncTime")).Seconds(), jvm))
+
+	if *flags.EnableJavaAsyncProfiler {
+		res = append(res, gauge(metrics.JvmProfilingStatus, 1, jvm))
+		if s := c.jvmProfilingStats; s != nil {
+			res = append(res, counter(metrics.JvmAllocBytes, float64(s.AllocBytes), jvm))
+			res = append(res, counter(metrics.JvmAllocObjects, float64(s.AllocObjects), jvm))
+			res = append(res, counter(metrics.JvmLockContentions, float64(s.LockContentions), jvm))
+			res = append(res, counter(metrics.JvmLockTime, float64(s.LockTimeNs)/1e9, jvm))
+		}
+	} else {
+		res = append(res, gauge(metrics.JvmProfilingStatus, 0, jvm))
+	}
+
 	return jvm, res
 }
 

containers/metrics.go

@@ -49,9 +49,15 @@ var metrics = struct {
 	JvmInfo              *prometheus.Desc
 	JvmHeapSize          *prometheus.Desc
 	JvmHeapUsed          *prometheus.Desc
+	JvmHeapMaxSize       *prometheus.Desc
 	JvmGCTime            *prometheus.Desc
 	JvmSafepointTime     *prometheus.Desc
 	JvmSafepointSyncTime *prometheus.Desc
+	JvmAllocBytes        *prometheus.Desc
+	JvmAllocObjects      *prometheus.Desc
+	JvmLockContentions   *prometheus.Desc
+	JvmLockTime          *prometheus.Desc
+	JvmProfilingStatus   *prometheus.Desc
 
 	PythonThreadLockWaitTime   *prometheus.Desc
 	NodejsEventLoopBlockedTime *prometheus.Desc
@@ -105,9 +111,15 @@ var metrics = struct {
 	JvmInfo:              metric("container_jvm_info", "Meta information about the JVM", "jvm", "java_version"),
 	JvmHeapSize:          metric("container_jvm_heap_size_bytes", "Total heap size in bytes", "jvm"),
 	JvmHeapUsed:          metric("container_jvm_heap_used_bytes", "Used heap size in bytes", "jvm"),
+	JvmHeapMaxSize:       metric("container_jvm_heap_max_size_bytes", "Maximum heap size in bytes (-Xmx)", "jvm"),
 	JvmGCTime:            metric("container_jvm_gc_time_seconds", "Time spent in the given JVM garbage collector in seconds", "jvm", "gc"),
 	JvmSafepointTime:     metric("container_jvm_safepoint_time_seconds", "Time the application has been stopped for safepoint operations in seconds", "jvm"),
 	JvmSafepointSyncTime: metric("container_jvm_safepoint_sync_time_seconds", "Time spent getting to safepoints in seconds", "jvm"),
+	JvmAllocBytes:        metric("container_jvm_alloc_bytes_total", "Total bytes allocated observed by async-profiler", "jvm"),
+	JvmAllocObjects:      metric("container_jvm_alloc_objects_total", "Total objects allocated observed by async-profiler", "jvm"),
+	JvmLockContentions:   metric("container_jvm_lock_contentions_total", "Total number of lock contentions observed by async-profiler", "jvm"),
+	JvmLockTime:          metric("container_jvm_lock_time_seconds_total", "Total time spent waiting for locks observed by async-profiler", "jvm"),
+	JvmProfilingStatus:   metric("container_jvm_profiling_status", "1 if async-profiler is enabled, 0 if disabled", "jvm"),
 
 	Ip2Fqdn: metric("ip_to_fqdn", "Mapping IP addresses to FQDNs based on DNS requests initiated by containers", "ip", "fqdn"),
 

containers/registry.go

@@ -56,7 +56,8 @@ type Registry struct {
 	ip2fqdn                map[netaddr.IP]*common.Domain
 	ip2fqdnLock            sync.RWMutex
 
-	processInfoCh chan<- ProcessInfo
+	processInfoCh        chan<- ProcessInfo
+	jvmProfilingUpdateCh chan *JvmProfilingUpdate
 
 	ebpfStatsLastUpdated time.Time
 	ebpfStatsLock        sync.Mutex
@@ -67,7 +68,7 @@ type Registry struct {
 	gpuProcessUsageSampleChan chan gpu.ProcessUsageSample
 }
 
-func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, gpuProcessUsageSampleChan chan gpu.ProcessUsageSample) (*Registry, error) {
+func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, jvmProfilingUpdateCh chan *JvmProfilingUpdate, gpuProcessUsageSampleChan chan gpu.ProcessUsageSample) (*Registry, error) {
 	ns, err := proc.GetSelfNetNs()
 	if err != nil {
 		return nil, err
@@ -121,6 +122,7 @@ func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, gp
 		trafficStatsUpdateCh: make(chan *TrafficStatsUpdate),
 		nodejsStatsUpdateCh:  make(chan *NodejsStatsUpdate),
 		pythonStatsUpdateCh:  make(chan *PythonStatsUpdate),
+		jvmProfilingUpdateCh: jvmProfilingUpdateCh,
 
 		gpuProcessUsageSampleChan: gpuProcessUsageSampleChan,
 	}
@@ -229,6 +231,10 @@ func (r *Registry) handleEvents(ch <-chan ebpftracer.Event) {
 			if c := r.containersByPid[u.Pid]; c != nil {
 				c.updatePythonStats(*u)
 			}
+		case u := <-r.jvmProfilingUpdateCh:
+			if c := r.containersByPid[u.Pid]; c != nil {
+				c.updateJvmProfilingStats(u)
+			}
 		case sample := <-r.gpuProcessUsageSampleChan:
 			if c := r.containersByPid[sample.Pid]; c != nil {
 				if p := c.processes[sample.Pid]; p != nil {
@@ -597,3 +603,18 @@ type PythonStatsUpdate struct {
 	Pid   uint32
 	Stats ebpftracer.PythonStats
 }
+
+type JvmProfilingUpdate struct {
+	Pid             uint32
+	AllocBytes      int64
+	AllocObjects    int64
+	LockContentions int64
+	LockTimeNs      int64
+}
+
+type JvmProfilingStats struct {
+	AllocBytes      int64
+	AllocObjects    int64
+	LockContentions int64
+	LockTimeNs      int64
+}

flags/flags.go

@@ -9,13 +9,14 @@ import (
 )
 
 var (
-	ListenAddress        = kingpin.Flag("listen", "Listen address - ip:port or :port").Default("0.0.0.0:80").Envar("LISTEN").String()
-	CgroupRoot           = kingpin.Flag("cgroupfs-root", "The mount point of the host cgroupfs root").Default("/sys/fs/cgroup").Envar("CGROUPFS_ROOT").String()
-	DisableLogParsing    = kingpin.Flag("disable-log-parsing", "Disable container log parsing").Default("false").Envar("DISABLE_LOG_PARSING").Bool()
-	DisablePinger        = kingpin.Flag("disable-pinger", "Don't ping upstreams").Default("false").Envar("DISABLE_PINGER").Bool()
-	DisableL7Tracing     = kingpin.Flag("disable-l7-tracing", "Disable L7 tracing").Default("false").Envar("DISABLE_L7_TRACING").Bool()
-	DisableGPUMonitoring = kingpin.Flag("disable-gpu-monitoring", "Disable GPU monitoring (NVML)").Default("false").Envar("DISABLE_GPU_MONITORING").Bool()
-	EnableJavaTls        = kingpin.Flag("enable-java-tls", "Enable Java TLS instrumentation via dynamic agent loading").Default("false").Envar("ENABLE_JAVA_TLS").Bool()
+	ListenAddress           = kingpin.Flag("listen", "Listen address - ip:port or :port").Default("0.0.0.0:80").Envar("LISTEN").String()
+	CgroupRoot              = kingpin.Flag("cgroupfs-root", "The mount point of the host cgroupfs root").Default("/sys/fs/cgroup").Envar("CGROUPFS_ROOT").String()
+	DisableLogParsing       = kingpin.Flag("disable-log-parsing", "Disable container log parsing").Default("false").Envar("DISABLE_LOG_PARSING").Bool()
+	DisablePinger           = kingpin.Flag("disable-pinger", "Don't ping upstreams").Default("false").Envar("DISABLE_PINGER").Bool()
+	DisableL7Tracing        = kingpin.Flag("disable-l7-tracing", "Disable L7 tracing").Default("false").Envar("DISABLE_L7_TRACING").Bool()
+	DisableGPUMonitoring    = kingpin.Flag("disable-gpu-monitoring", "Disable GPU monitoring (NVML)").Default("false").Envar("DISABLE_GPU_MONITORING").Bool()
+	EnableJavaTls           = kingpin.Flag("enable-java-tls", "Enable Java TLS instrumentation via dynamic agent loading").Default("false").Envar("ENABLE_JAVA_TLS").Bool()
+	EnableJavaAsyncProfiler = kingpin.Flag("enable-java-async-profiler", "Enable Java profiling via async-profiler (CPU, memory allocations, lock contention)").Default("false").Envar("ENABLE_JAVA_ASYNC_PROFILER").Bool()
 
 	ContainerAllowlist = kingpin.Flag("container-allowlist", "List of allowed containers (regex patterns)").Envar("CONTAINER_ALLOWLIST").Strings()
 	ContainerDenylist  = kingpin.Flag("container-denylist", "List of denied containers (regex patterns)").Envar("CONTAINER_DENYLIST").Strings()

go.mod

@@ -18,6 +18,8 @@ require (
 	github.com/gobwas/glob v0.2.3
 	github.com/godbus/dbus/v5 v5.2.2
 	github.com/golang/snappy v0.0.4
+	github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef
+	github.com/grafana/jfr-parser v0.15.0
 	github.com/grafana/pyroscope/ebpf v0.4.9
 	github.com/jpillora/backoff v1.0.0
 	github.com/mdlayher/taskstats v0.0.0-20230712191918-387b3d561d14
@@ -105,16 +107,15 @@ require (
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gopacket/gopacket v1.3.1 // indirect
 	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
-	github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect
+	github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
@@ -188,7 +189,7 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect
 	google.golang.org/grpc v1.69.2 // indirect
-	google.golang.org/protobuf v1.36.5 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

go.sum

@@ -208,18 +208,20 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
-github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef h1:xpF9fUHpoIrrjX24DURVKiwHcFpw19ndIs+FwTSMbno=
+github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopacket/gopacket v1.3.1 h1:ZppWyLrOJNZPe5XkdjLbtuTkfQoxQ0xyMJzQCqtqaPU=
 github.com/gopacket/gopacket v1.3.1/go.mod h1:3I13qcqSpB2R9fFQg866OOgzylYkZxLTmkvcXhvf6qg=
+github.com/grafana/jfr-parser v0.15.0 h1:CS78x/oV72Z+q17spr2N6AUFkucAWMKPwHYvj1uemWA=
+github.com/grafana/jfr-parser v0.15.0/go.mod h1:2vR91w+TYF6Jrw+WJMd/uyAiNxE2BUY5xOsvglXXe78=
 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA=
 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
@@ -228,8 +230,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM=
 github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
-github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 h1:KwWnWVWCNtNq/ewIX7HIKnELmEx2nDP42yskD/pi7QE=
-github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
+github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b h1:ogbOPx86mIhFy764gGkqnkFC8m5PJA7sPzlk9ppLVQA=
+github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -591,8 +593,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
-google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=