diff --git a/cmd/admin-trace.go b/cmd/admin-trace.go index c00b6da8..2a25d6ab 100644 --- a/cmd/admin-trace.go +++ b/cmd/admin-trace.go @@ -24,17 +24,13 @@ import ( "hash/fnv" "net/http" "net/url" - "os" "path" "sort" - "strconv" "strings" "sync" "time" - "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" - "github.com/charmbracelet/lipgloss" "github.com/dustin/go-humanize" "github.com/fatih/color" "github.com/minio/cli" @@ -42,9 +38,6 @@ import ( "github.com/minio/madmin-go/v3" "github.com/minio/mc/pkg/probe" "github.com/minio/pkg/v3/console" - "github.com/muesli/reflow/truncate" - "github.com/olekukonko/tablewriter" - "golang.org/x/term" ) var adminTraceFlags = []cli.Flag{ @@ -94,12 +87,12 @@ var adminTraceFlags = []cli.Flag{ }, cli.BoolFlag{ Name: "stats", - Usage: "accumulate stats", + Usage: "print statistical summary of all the traced calls", }, cli.IntFlag{ Name: "stats-n", Usage: "maximum number of stat entries", - Value: 15, + Value: 30, Hidden: true, }, cli.BoolFlag{ @@ -979,227 +972,3 @@ func (s *statTrace) add(t madmin.ServiceTraceInfo) { } s.Calls[id] = got } - -func initTraceStatsUI(maxEntries int, traces <-chan madmin.ServiceTraceInfo) *traceStatsUI { - s := spinner.New() - s.Spinner = spinner.Points - s.Spinner.FPS = time.Second / 2 - s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("205")) - console.SetColor("metrics-duration", color.New(color.FgWhite)) - console.SetColor("metrics-size", color.New(color.FgGreen)) - console.SetColor("metrics-dur", color.New(color.FgGreen)) - console.SetColor("metrics-dur-med", color.New(color.FgYellow)) - console.SetColor("metrics-dur-high", color.New(color.FgRed)) - console.SetColor("metrics-error", color.New(color.FgYellow)) - console.SetColor("metrics-title", color.New(color.FgCyan)) - console.SetColor("metrics-top-title", color.New(color.FgHiCyan)) - console.SetColor("metrics-number", color.New(color.FgWhite)) - console.SetColor("metrics-number-secondary", color.New(color.FgBlue)) - console.SetColor("metrics-zero", color.New(color.FgWhite)) - stats := &statTrace{Calls: make(map[string]statItem, 20), Started: time.Now()} - go func() { - for t := range traces { - stats.add(t) - } - }() - return &traceStatsUI{ - started: time.Now(), - spinner: s, - maxEntries: maxEntries, - current: stats, - } -} - -type traceStatsUI struct { - current *statTrace - started time.Time - spinner spinner.Model - quitting bool - maxEntries int -} - -func (m *traceStatsUI) Init() tea.Cmd { - return m.spinner.Tick -} - -func (m *traceStatsUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { - if m.quitting { - return m, tea.Quit - } - switch msg := msg.(type) { - case tea.KeyMsg: - switch msg.String() { - case "q", "esc", "ctrl+c": - m.quitting = true - return m, tea.Quit - default: - return m, nil - } - case spinner.TickMsg: - var cmd tea.Cmd - m.spinner, cmd = m.spinner.Update(msg) - return m, cmd - } - return m, nil -} - -func (m *traceStatsUI) View() string { - var s strings.Builder - - s.WriteString(fmt.Sprintf("%s %s\n", - console.Colorize("metrics-top-title", "Duration: "+time.Since(m.current.Started).Round(time.Second).String()), m.spinner.View())) - - // Set table header - akin to k8s style - // https://github.com/olekukonko/tablewriter#example-10---set-nowhitespace-and-tablepadding-option - table := tablewriter.NewWriter(&s) - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(true) - table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) - table.SetAlignment(tablewriter.ALIGN_LEFT) - table.SetCenterSeparator("") - table.SetColumnSeparator("") - table.SetRowSeparator("") - table.SetHeaderLine(false) - table.SetBorder(false) - table.SetTablePadding(" ") // pad with tabs - table.SetNoWhiteSpace(true) - var entries []statItem - m.current.mu.Lock() - totalCnt := 0 - dur := time.Since(m.current.Started) - for _, v := range m.current.Calls { - totalCnt += v.Count - entries = append(entries, v) - } - m.current.mu.Unlock() - if len(entries) == 0 { - s.WriteString("(waiting for data)") - return s.String() - } - sort.Slice(entries, func(i, j int) bool { - if entries[i].Count == entries[j].Count { - return entries[i].Name < entries[j].Name - } - return entries[i].Count > entries[j].Count - }) - if m.maxEntries > 0 && len(entries) > m.maxEntries { - entries = entries[:m.maxEntries] - } - hasTTFB := false - for _, e := range entries { - if e.TTFB > 0 { - hasTTFB = true - break - } - } - t := []string{ - console.Colorize("metrics-top-title", "Call"), - console.Colorize("metrics-top-title", "Count"), - console.Colorize("metrics-top-title", "RPM"), - console.Colorize("metrics-top-title", "Avg Time"), - console.Colorize("metrics-top-title", "Min Time"), - console.Colorize("metrics-top-title", "Max Time"), - } - if hasTTFB { - t = append(t, - console.Colorize("metrics-top-title", "Avg TTFB"), - console.Colorize("metrics-top-title", "Max TTFB"), - ) - } - t = append(t, - console.Colorize("metrics-top-title", "Avg Size"), - console.Colorize("metrics-top-title", "Rate"), - console.Colorize("metrics-top-title", "Errors"), - ) - - table.Append(t) - for _, v := range entries { - if v.Count <= 0 { - continue - } - errs := "0" - if v.Errors > 0 { - errs = console.Colorize("metrics-error", strconv.Itoa(v.Errors)) - } - avg := v.Duration / time.Duration(v.Count) - avgTTFB := v.TTFB / time.Duration(v.Count) - avgColor := "metrics-dur" - if avg > 10*time.Second { - avgColor = "metrics-dur-high" - } else if avg > time.Second { - avgColor = "metrics-dur-med" - } - - minColor := "metrics-dur" - if v.MinDur > 10*time.Second { - minColor = "metrics-dur-high" - } else if v.MinDur > time.Second { - minColor = "metrics-dur-med" - } - - maxColor := "metrics-dur" - if v.MaxDur > 10*time.Second { - maxColor = "metrics-dur-high" - } else if v.MaxDur > time.Second { - maxColor = "metrics-dur-med" - } - - sz := "-" - rate := "-" - if v.Size > 0 && v.Count > 0 { - sz = humanize.IBytes(uint64(v.Size) / uint64(v.Count)) - rate = fmt.Sprintf("%s/m", humanize.IBytes(uint64(float64(v.Size)/dur.Minutes()))) - } - if v.CallStatsCount > 0 { - var s, r []string - if v.CallStats.Rx > 0 { - s = append(s, fmt.Sprintf("↑ %s", humanize.IBytes(uint64(v.CallStats.Rx/v.CallStatsCount)))) - r = append(r, fmt.Sprintf("↑ %s", humanize.IBytes(uint64(float64(v.CallStats.Rx)/dur.Minutes())))) - } - if v.CallStats.Tx > 0 { - s = append(s, fmt.Sprintf("↓ %s", humanize.IBytes(uint64(v.CallStats.Tx/v.CallStatsCount)))) - r = append(r, fmt.Sprintf("↓ %s", humanize.IBytes(uint64(float64(v.CallStats.Tx)/dur.Minutes())))) - } - if len(s) > 0 { - sz = strings.Join(s, " ") - rate = strings.Join(r, " ") + "/m" - } - } - if sz != "-" { - sz = console.Colorize("metrics-size", sz) - rate = console.Colorize("metrics-size", rate) - } - - t := []string{ - console.Colorize("metrics-title", metricsTitle(v.Name)), - console.Colorize("metrics-number", fmt.Sprintf("%d ", v.Count)) + - console.Colorize("metrics-number-secondary", fmt.Sprintf("(%0.1f%%)", float64(v.Count)/float64(totalCnt)*100)), - console.Colorize("metrics-number", fmt.Sprintf("%0.1f", float64(v.Count)/dur.Minutes())), - console.Colorize(avgColor, fmt.Sprintf("%v", avg.Round(time.Microsecond))), - console.Colorize(minColor, v.MinDur), - console.Colorize(maxColor, v.MaxDur), - } - if hasTTFB { - t = append(t, - console.Colorize(avgColor, fmt.Sprintf("%v", avgTTFB.Round(time.Microsecond))), - console.Colorize(maxColor, v.MaxTTFB)) - } - t = append(t, sz, - rate, - errs) - table.Append(t) - } - table.Render() - if globalTermWidth <= 10 { - return s.String() - } - w := globalTermWidth - if nw, _, e := term.GetSize(int(os.Stdout.Fd())); e == nil { - w = nw - } - split := strings.Split(s.String(), "\n") - for i, line := range split { - split[i] = truncate.StringWithTail(line, uint(w), "»") - } - return strings.Join(split, "\n") -} diff --git a/cmd/support-top-api.go b/cmd/support-top-api.go index 9f1a66e8..52728f84 100644 --- a/cmd/support-top-api.go +++ b/cmd/support-top-api.go @@ -103,27 +103,27 @@ func mainSupportTopAPI(ctx *cli.Context) error { // Start listening on all trace activity. traceCh := client.ServiceTrace(ctxt, opts) - p := tea.NewProgram(initTraceUI()) + filteredTraces := make(chan madmin.ServiceTraceInfo, 1) + ui := tea.NewProgram(initTraceStatsUI(30, filteredTraces)) + var te error go func() { - for apiCallInfo := range traceCh { - if apiCallInfo.Err != nil { - fatalIf(probe.NewError(apiCallInfo.Err), "Unable to fetch top API events") + for t := range traceCh { + if t.Err != nil { + te = t.Err + ui.Kill() + return } - if mopts.matches(apiCallInfo) { - p.Send(topAPIResult{ - apiCallInfo: apiCallInfo, - }) + if mopts.matches(t) { + filteredTraces <- t } - p.Send(topAPIResult{ - apiCallInfo: madmin.ServiceTraceInfo{}, - }) } }() - - if _, e := p.Run(); e != nil { + if _, e := ui.Run(); e != nil { cancel() - fatalIf(probe.NewError(e).Trace(aliasedURL), "Unable to fetch top API events") + if te != nil { + e = te + } + fatalIf(probe.NewError(e).Trace(aliasedURL), "Unable to fetch http trace statistics") } - return nil } diff --git a/cmd/trace-stats-ui.go b/cmd/trace-stats-ui.go new file mode 100644 index 00000000..7bc5c240 --- /dev/null +++ b/cmd/trace-stats-ui.go @@ -0,0 +1,263 @@ +package cmd + +import ( + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + humanize "github.com/dustin/go-humanize" + "github.com/fatih/color" + "github.com/minio/madmin-go/v3" + "github.com/minio/pkg/v3/console" + "github.com/muesli/reflow/truncate" + "github.com/olekukonko/tablewriter" + "golang.org/x/term" +) + +type traceStatsUI struct { + current *statTrace + started time.Time + meter spinner.Model + quitting bool + maxEntries int +} + +func (m *traceStatsUI) Init() tea.Cmd { + return m.meter.Tick +} + +func (m *traceStatsUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + if m.quitting { + return m, tea.Quit + } + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + case "q", "esc", "ctrl+c": + m.quitting = true + return m, tea.Quit + default: + return m, nil + } + case spinner.TickMsg: + var cmd tea.Cmd + m.meter, cmd = m.meter.Update(msg) + return m, cmd + } + return m, nil +} + +func (m *traceStatsUI) View() string { + var s strings.Builder + + s.WriteString(fmt.Sprintf("%s %s\n", + console.Colorize("metrics-top-title", "Duration: "+time.Since(m.current.Started).Round(time.Second).String()), m.meter.View())) + + // Set table header - akin to k8s style + // https://github.com/olekukonko/tablewriter#example-10---set-nowhitespace-and-tablepadding-option + table := tablewriter.NewWriter(&s) + table.SetAutoWrapText(false) + table.SetAutoFormatHeaders(true) + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetCenterSeparator("") + table.SetColumnSeparator("") + table.SetRowSeparator("") + table.SetHeaderLine(false) + table.SetBorder(false) + table.SetTablePadding(" ") // pad with tabs + table.SetNoWhiteSpace(true) + var entries []statItem + + m.current.mu.Lock() + var ( + totalCnt = 0 + totalRX = 0 + totalTX = 0 + ) + dur := time.Since(m.current.Started) + for _, v := range m.current.Calls { + totalCnt += v.Count + totalRX += v.CallStats.Rx + totalTX += v.CallStats.Tx + entries = append(entries, v) + } + m.current.mu.Unlock() + if len(entries) == 0 { + s.WriteString("(waiting for data)") + return s.String() + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Count == entries[j].Count { + return entries[i].Name < entries[j].Name + } + return entries[i].Count > entries[j].Count + }) + if m.maxEntries > 0 && len(entries) > m.maxEntries { + entries = entries[:m.maxEntries] + } + hasTTFB := false + for _, e := range entries { + if e.TTFB > 0 { + hasTTFB = true + break + } + } + + if totalRX > 0 { + s.WriteString(console.Colorize("metrics-top-title", fmt.Sprintf("RX Rate:↑ %s/m\n", + humanize.IBytes(uint64(float64(totalRX)/dur.Minutes()))))) + } + if totalTX > 0 { + s.WriteString(console.Colorize("metrics-top-title", fmt.Sprintf("TX Rate:↓ %s/m\n", + humanize.IBytes(uint64(float64(totalTX)/dur.Minutes()))))) + } + s.WriteString(console.Colorize("metrics-top-title", fmt.Sprintf("RPM : %0.1f\n", float64(totalCnt)/dur.Minutes()))) + s.WriteString("-------------\n") + + t := []string{ + console.Colorize("metrics-top-title", "Call"), + console.Colorize("metrics-top-title", "Count"), + console.Colorize("metrics-top-title", "RPM"), + console.Colorize("metrics-top-title", "Avg Time"), + console.Colorize("metrics-top-title", "Min Time"), + console.Colorize("metrics-top-title", "Max Time"), + } + if hasTTFB { + t = append(t, + console.Colorize("metrics-top-title", "Avg TTFB"), + console.Colorize("metrics-top-title", "Max TTFB"), + ) + } + t = append(t, + console.Colorize("metrics-top-title", "Avg Size"), + console.Colorize("metrics-top-title", "Rate"), + console.Colorize("metrics-top-title", "Errors"), + ) + + table.Append(t) + for _, v := range entries { + if v.Count <= 0 { + continue + } + errs := "0" + if v.Errors > 0 { + errs = console.Colorize("metrics-error", strconv.Itoa(v.Errors)) + } + avg := v.Duration / time.Duration(v.Count) + avgTTFB := v.TTFB / time.Duration(v.Count) + avgColor := "metrics-dur" + if avg > 10*time.Second { + avgColor = "metrics-dur-high" + } else if avg > 2*time.Second { + avgColor = "metrics-dur-med" + } + + minColor := "metrics-dur" + if v.MinDur > 10*time.Second { + minColor = "metrics-dur-high" + } else if v.MinDur > 2*time.Second { + minColor = "metrics-dur-med" + } + + maxColor := "metrics-dur" + if v.MaxDur > 10*time.Second { + maxColor = "metrics-dur-high" + } else if v.MaxDur > time.Second { + maxColor = "metrics-dur-med" + } + + sz := "-" + rate := "-" + if v.Size > 0 && v.Count > 0 { + sz = humanize.IBytes(uint64(v.Size) / uint64(v.Count)) + rate = fmt.Sprintf("%s/m", humanize.IBytes(uint64(float64(v.Size)/dur.Minutes()))) + } + if v.CallStatsCount > 0 { + var s, r []string + if v.CallStats.Rx > 0 { + s = append(s, fmt.Sprintf("↑ %s", humanize.IBytes(uint64(v.CallStats.Rx/v.CallStatsCount)))) + r = append(r, fmt.Sprintf("↑ %s", humanize.IBytes(uint64(float64(v.CallStats.Rx)/dur.Minutes())))) + } + if v.CallStats.Tx > 0 { + s = append(s, fmt.Sprintf("↓ %s", humanize.IBytes(uint64(v.CallStats.Tx/v.CallStatsCount)))) + r = append(r, fmt.Sprintf("↓ %s", humanize.IBytes(uint64(float64(v.CallStats.Tx)/dur.Minutes())))) + } + if len(s) > 0 { + sz = strings.Join(s, " ") + } + if len(r) > 0 { + rate = strings.Join(r, " ") + "/m" + } + } + if sz != "-" { + sz = console.Colorize("metrics-size", sz) + rate = console.Colorize("metrics-size", rate) + } + + t := []string{ + console.Colorize("metrics-title", metricsTitle(v.Name)), + console.Colorize("metrics-number", fmt.Sprintf("%d ", v.Count)) + + console.Colorize("metrics-number-secondary", fmt.Sprintf("(%0.1f%%)", float64(v.Count)/float64(totalCnt)*100)), + console.Colorize("metrics-number", fmt.Sprintf("%0.1f", float64(v.Count)/dur.Minutes())), + console.Colorize(avgColor, fmt.Sprintf("%v", avg.Round(time.Microsecond))), + console.Colorize(minColor, v.MinDur), + console.Colorize(maxColor, v.MaxDur), + } + if hasTTFB { + t = append(t, + console.Colorize(avgColor, fmt.Sprintf("%v", avgTTFB.Round(time.Microsecond))), + console.Colorize(maxColor, v.MaxTTFB)) + } + t = append(t, sz, rate, errs) + table.Append(t) + } + table.Render() + if globalTermWidth <= 10 { + return s.String() + } + w := globalTermWidth + if nw, _, e := term.GetSize(int(os.Stdout.Fd())); e == nil { + w = nw + } + split := strings.Split(s.String(), "\n") + for i, line := range split { + split[i] = truncate.StringWithTail(line, uint(w), "»") + } + return strings.Join(split, "\n") +} + +func initTraceStatsUI(maxEntries int, traces <-chan madmin.ServiceTraceInfo) *traceStatsUI { + meter := spinner.New() + meter.Spinner = spinner.Meter + meter.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("205")) + console.SetColor("metrics-duration", color.New(color.FgWhite)) + console.SetColor("metrics-size", color.New(color.FgGreen)) + console.SetColor("metrics-dur", color.New(color.FgGreen)) + console.SetColor("metrics-dur-med", color.New(color.FgYellow)) + console.SetColor("metrics-dur-high", color.New(color.FgRed)) + console.SetColor("metrics-error", color.New(color.FgYellow)) + console.SetColor("metrics-title", color.New(color.FgCyan)) + console.SetColor("metrics-top-title", color.New(color.FgHiCyan)) + console.SetColor("metrics-number", color.New(color.FgWhite)) + console.SetColor("metrics-number-secondary", color.New(color.FgBlue)) + console.SetColor("metrics-zero", color.New(color.FgWhite)) + stats := &statTrace{Calls: make(map[string]statItem, 20), Started: time.Now()} + go func() { + for t := range traces { + stats.add(t) + } + }() + return &traceStatsUI{ + started: time.Now(), + meter: meter, + maxEntries: maxEntries, + current: stats, + } +}