feat: add connection waiting statistics (#2804)

Co-authored-by: ofekshenawa <104765379+ofekshenawa@users.noreply.github.com>

internal/pool/pool.go

@@ -36,6 +36,8 @@ type Stats struct {
 	Hits           uint32 // number of times free connection was found in the pool
 	Misses         uint32 // number of times free connection was NOT found in the pool
 	Timeouts       uint32 // number of times a wait timeout occurred
+	WaitCount      uint32 // number of times a connection was waited
+	WaitDurationNs int64  // total time spent for waiting a connection in nanoseconds
 
 	TotalConns uint32 // number of total connections in the pool
 	IdleConns  uint32 // number of idle connections in the pool
@@ -91,6 +93,7 @@ type ConnPool struct {
 	idleConnsLen int
 
 	stats          Stats
+	waitDurationNs atomic.Int64
 
 	_closed uint32 // atomic
 }
@@ -320,6 +323,7 @@ func (p *ConnPool) waitTurn(ctx context.Context) error {
 	default:
 	}
 
+	start := time.Now()
 	timer := timers.Get().(*time.Timer)
 	timer.Reset(p.cfg.PoolTimeout)
 
@@ -331,6 +335,8 @@ func (p *ConnPool) waitTurn(ctx context.Context) error {
 		timers.Put(timer)
 		return ctx.Err()
 	case p.queue <- struct{}{}:
+		p.waitDurationNs.Add(time.Since(start).Nanoseconds())
+		atomic.AddUint32(&p.stats.WaitCount, 1)
 		if !timer.Stop() {
 			<-timer.C
 		}
@@ -460,6 +466,8 @@ func (p *ConnPool) Stats() *Stats {
 		Hits:           atomic.LoadUint32(&p.stats.Hits),
 		Misses:         atomic.LoadUint32(&p.stats.Misses),
 		Timeouts:       atomic.LoadUint32(&p.stats.Timeouts),
+		WaitCount:      atomic.LoadUint32(&p.stats.WaitCount),
+		WaitDurationNs: p.waitDurationNs.Load(),
 
 		TotalConns: uint32(p.Len()),
 		IdleConns:  uint32(p.IdleLen()),

internal/pool/pool_test.go

@@ -62,6 +62,8 @@ var _ = Describe("ConnPool", func() {
 			Hits:           0,
 			Misses:         0,
 			Timeouts:       0,
+			WaitCount:      0,
+			WaitDurationNs: 0,
 			TotalConns:     0,
 			IdleConns:      0,
 			StaleConns:     0,
@@ -358,4 +360,31 @@ var _ = Describe("race", func() {
 		Expect(stats.IdleConns).To(Equal(uint32(0)))
 		Expect(stats.TotalConns).To(Equal(uint32(opt.PoolSize)))
 	})
+
+	It("wait", func() {
+		opt := &pool.Options{
+			Dialer: func(ctx context.Context) (net.Conn, error) {
+				return &net.TCPConn{}, nil
+			},
+			PoolSize:    1,
+			PoolTimeout: 3 * time.Second,
+		}
+		p := pool.NewConnPool(opt)
+
+		wait := make(chan struct{})
+		conn, _ := p.Get(ctx)
+		go func() {
+			_, _ = p.Get(ctx)
+			wait <- struct{}{}
+		}()
+		time.Sleep(time.Second)
+		p.Put(ctx, conn)
+		<-wait
+
+		stats := p.Stats()
+		Expect(stats.IdleConns).To(Equal(uint32(0)))
+		Expect(stats.TotalConns).To(Equal(uint32(1)))
+		Expect(stats.WaitCount).To(Equal(uint32(1)))
+		Expect(stats.WaitDurationNs).To(BeNumerically("~", time.Second.Nanoseconds(), 100*time.Millisecond.Nanoseconds()))
+	})
 })