minio/mc
1
0
Fork 0
mirror of https://github.com/minio/mc.git synced 2025-07-31 18:24:21 +03:00
Code Activity

Change the JSON output of mc support perf (#4198)

Browse Source 
Change the JSON output of mc support perf

The new change will still print multiple JSON lines for each conducted
performance test but now we have a new field called 'type' field to
difference between network, drive and object tests.

This change also adds 'final' field to indicate this is the final json of a
particular performance testing.

Also, errors when conducting those tests will have a field 'err' as well
in the JSON output.
This commit is contained in:
Anis Elleuch
2022-08-10 16:09:05 +01:00
committed by GitHub
parent 73b157e4b1
commit 3e990e3525
5 changed files with 160 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
cmd/speedtest-spinner.go
View File

@ -40,12 +40,33 @@ type speedTestUI struct {
result speedTestResult
}
type speedTestType byte
const (
netSpeedTest speedTestType = 1 << iota
driveSpeedTest
objectSpeedTest
)
func (s speedTestType) Name() string {
switch s {
case netSpeedTest:
return "NetPerf"
case driveSpeedTest:
return "DrivePerf"
case objectSpeedTest:
return "ObjectPerf"
}
return "<unknown>"
}
type speedTestResult struct {
err error
final bool
result *madmin.SpeedTestResult
nresult *madmin.NetperfResult
dresult []madmin.DriveSpeedTestResult
Type speedTestType `json:"type"`
ObjectResult *madmin.SpeedTestResult `json:"object,omitempty"`
NetResult *madmin.NetperfResult `json:"network,omitempty"`
DriveResult []madmin.DriveSpeedTestResult `json:"drive,omitempty"`
Err string `json:"err,omitempty"`
Final bool `json:"final,omitempty"`
}
func initSpeedTestUI() *speedTestUI {
@ -73,7 +94,7 @@ func (m *speedTestUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
}
case speedTestResult:
m.result = msg
if msg.final {
if msg.Final {
m.quitting = true
return m, tea.Quit
}
@ -89,15 +110,8 @@ func (m *speedTestUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
func (m *speedTestUI) View() string {
// Quit when there is an error
if m.result.err != nil {
errMsg := ""
errResp := madmin.ToErrorResponse(m.result.err)
if errResp.Code == "NotImplemented" {
errMsg = "Not implemented"
} else {
errMsg = m.result.err.Error()
}
return fmt.Sprintf("\nNetperf: ✗ (Err: %s)\n", errMsg)
if m.result.Err != "" {
return fmt.Sprintf("\n%s: ✗ (Err: %s)\n", m.result.Type.Name(), m.result.Err)
}
var s strings.Builder
@ -117,9 +131,9 @@ func (m *speedTestUI) View() string {
table.SetTablePadding("\t") // pad with tabs
table.SetNoWhiteSpace(true)
res := m.result.result
nres := m.result.nresult
dres := m.result.dresult
ores := m.result.ObjectResult
nres := m.result.NetResult
dres := m.result.DriveResult
trailerIfGreaterThan := func(in string, max int) string {
if len(in) < max {
@ -128,11 +142,11 @@ func (m *speedTestUI) View() string {
return in[:max] + "..."
}
if res != nil {
if ores != nil {
table.SetHeader([]string{"", "Throughput", "IOPS"})
data := make([][]string, 2)
if res.Version == "" {
if ores.Version == "" {
data[0] = []string{
"PUT",
whiteStyle.Render("-- KiB/sec"),
@ -146,26 +160,25 @@ func (m *speedTestUI) View() string {
} else {
data[0] = []string{
"PUT",
whiteStyle.Render(humanize.IBytes(res.PUTStats.ThroughputPerSec) + "/s"),
whiteStyle.Render(humanize.Comma(int64(res.PUTStats.ObjectsPerSec)) + " objs/s"),
whiteStyle.Render(humanize.IBytes(ores.PUTStats.ThroughputPerSec) + "/s"),
whiteStyle.Render(humanize.Comma(int64(ores.PUTStats.ObjectsPerSec)) + " objs/s"),
}
data[1] = []string{
"GET",
whiteStyle.Render(humanize.IBytes(res.GETStats.ThroughputPerSec) + "/s"),
whiteStyle.Render(humanize.Comma(int64(res.GETStats.ObjectsPerSec)) + " objs/s"),
whiteStyle.Render(humanize.IBytes(ores.GETStats.ThroughputPerSec) + "/s"),
whiteStyle.Render(humanize.Comma(int64(ores.GETStats.ObjectsPerSec)) + " objs/s"),
}
}
table.AppendBulk(data)
table.Render()
if m.quitting {
s.WriteString("\n" + m.result.String())
if vstr := m.result.StringVerbose(); vstr != "" {
s.WriteString(vstr + "\n")
} else {
s.WriteString("\n")
s.WriteString("\n" + objectTestShortResult(ores))
if globalPerfTestVerbose {
s.WriteString("\n\n")
s.WriteString(objectTestVerboseResult(ores))
}
s.WriteString("Objectperf: ✔\n")
s.WriteString("\n")
}
} else if nres != nil {
table.SetHeader([]string{"Node", "RX", "TX", ""})
@ -204,10 +217,6 @@ func (m *speedTestUI) View() string {
table.AppendBulk(data)
table.Render()
if m.quitting {
s.WriteString("\nNetperf: ✔\n")
}
} else if dres != nil {
table.SetHeader([]string{"Node", "Path", "Read", "Write", ""})
data := make([][]string, 0, len(dres))
@ -245,19 +254,13 @@ func (m *speedTestUI) View() string {
}
table.AppendBulk(data)
table.Render()
if m.quitting {
s.WriteString("\nDriveperf: ✔\n")
}
}
// Print the spinner
if !m.quitting {
if nres != nil {
s.WriteString(fmt.Sprintf("\nNetperf: %s", m.spinner.View()))
} else if res != nil {
s.WriteString(fmt.Sprintf("\nObjectperf: %s", m.spinner.View()))
} else if dres != nil {
s.WriteString(fmt.Sprintf("\nDriveperf: %s", m.spinner.View()))
}
s.WriteString(fmt.Sprintf("\n%s: %s", m.result.Type.Name(), m.spinner.View()))
} else {
s.WriteString(fmt.Sprintf("\n%s: ✔\n", m.result.Type.Name()))
}
return s.String()
}

42
cmd/support-perf-drive.go
View File

@ -24,10 +24,8 @@ import (
tea "github.com/charmbracelet/bubbletea"
humanize "github.com/dustin/go-humanize"
"github.com/minio/cli"
json "github.com/minio/colorjson"
"github.com/minio/madmin-go"
"github.com/minio/mc/pkg/probe"
"github.com/minio/pkg/console"
)
func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
@ -62,22 +60,34 @@ func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
serial := ctx.Bool("serial")
resultCh, e := client.DriveSpeedtest(ctxt, madmin.DriveSpeedTestOpts{
resultCh, speedTestErr := client.DriveSpeedtest(ctxt, madmin.DriveSpeedTestOpts{
Serial: serial,
BlockSize: uint64(blocksize),
FileSize: uint64(filesize),
})
fatalIf(probe.NewError(e), "Failed to execute drive speedtest")
if globalJSON {
if speedTestErr != nil {
printMsg(speedTestResult{
Type: driveSpeedTest,
Err: speedTestErr.Error(),
Final: true,
})
return nil
}
var results []madmin.DriveSpeedTestResult
for result := range resultCh {
if result.Version != "" {
jsonBytes, e := json.MarshalIndent(result, "", " ")
fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
console.Println(string(jsonBytes))
results = append(results, result)
}
}
printMsg(speedTestResult{
Type: driveSpeedTest,
DriveResult: results,
Final: true,
})
return nil
}
@ -92,19 +102,29 @@ func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
}()
go func() {
if speedTestErr != nil {
printMsg(speedTestResult{
Type: driveSpeedTest,
Err: speedTestErr.Error(),
})
return
}
var results []madmin.DriveSpeedTestResult
for result := range resultCh {
if result.Version != "" {
results = append(results, result)
} else {
p.Send(speedTestResult{
dresult: []madmin.DriveSpeedTestResult{},
Type: driveSpeedTest,
DriveResult: []madmin.DriveSpeedTestResult{},
})
}
}
p.Send(speedTestResult{
dresult: results,
final: true,
Type: driveSpeedTest,
DriveResult: results,
Final: true,
})
}()

47
cmd/support-perf-net.go
View File

@ -24,24 +24,10 @@ import (
tea "github.com/charmbracelet/bubbletea"
"github.com/minio/cli"
json "github.com/minio/colorjson"
"github.com/minio/madmin-go"
"github.com/minio/mc/pkg/probe"
)
type netperfResult madmin.NetperfResult
func (m netperfResult) String() (msg string) {
// string version is handled by banner.
return ""
}
func (m netperfResult) JSON() string {
JSONBytes, e := json.MarshalIndent(m, "", " ")
fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
return string(JSONBytes)
}
func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
client, perr := newAdminClient(aliasedURL)
if perr != nil {
@ -76,13 +62,21 @@ func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
}()
if globalJSON {
for {
select {
case result := <-resultCh:
printMsg(netperfResult(result))
return nil
}
select {
case err := <-errorCh:
printMsg(speedTestResult{
Type: netSpeedTest,
Err: err.Error(),
Final: true,
})
case result := <-resultCh:
printMsg(speedTestResult{
Type: netSpeedTest,
NetResult: &result,
Final: true,
})
}
return nil
}
done := make(chan struct{})
@ -100,19 +94,22 @@ func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
select {
case err := <-errorCh:
p.Send(speedTestResult{
err: err,
final: true,
Type: netSpeedTest,
Err: err.Error(),
Final: true,
})
return
case result := <-resultCh:
p.Send(speedTestResult{
nresult: &result,
final: true,
Type: netSpeedTest,
NetResult: &result,
Final: true,
})
return
default:
p.Send(speedTestResult{
nresult: &madmin.NetperfResult{},
Type: netSpeedTest,
NetResult: &madmin.NetperfResult{},
})
time.Sleep(100 * time.Millisecond)
}

42
cmd/support-perf-object.go
View File

@ -85,24 +85,41 @@ func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string) error {
// in all other scenarios keep auto-tuning on.
autotune := !ctx.IsSet("concurrent")
resultCh, err := client.Speedtest(ctxt, madmin.SpeedtestOpts{
resultCh, speedTestErr := client.Speedtest(ctxt, madmin.SpeedtestOpts{
Size: int(size),
Duration: duration,
Concurrency: concurrent,
Autotune: autotune,
Bucket: ctx.String("bucket"), // This is a hidden flag.
})
fatalIf(probe.NewError(err), "Failed to execute performance test")
if globalJSON {
for result := range resultCh {
if speedTestErr != nil {
printMsg(speedTestResult{
Type: objectSpeedTest,
Err: speedTestErr.Error(),
Final: true,
})
return nil
}
var result madmin.SpeedTestResult
for result = range resultCh {
if result.Version == "" {
continue
}
printMsg(speedTestResult{
result: &result,
Type: objectSpeedTest,
ObjectResult: &result,
})
}
printMsg(speedTestResult{
Type: objectSpeedTest,
ObjectResult: &result,
Final: true,
})
return nil
}
@ -117,15 +134,26 @@ func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string) error {
}()
go func() {
if speedTestErr != nil {
p.Send(speedTestResult{
Type: objectSpeedTest,
Err: speedTestErr.Error(),
Final: true,
})
return
}
var result madmin.SpeedTestResult
for result = range resultCh {
p.Send(speedTestResult{
result: &result,
Type: objectSpeedTest,
ObjectResult: &result,
})
}
p.Send(speedTestResult{
result: &result,
final: true,
Type: objectSpeedTest,
ObjectResult: &result,
Final: true,
})
}()

48
cmd/support-perf.go
View File

@ -23,6 +23,7 @@ import (
humanize "github.com/dustin/go-humanize"
"github.com/minio/cli"
json "github.com/minio/colorjson"
"github.com/minio/madmin-go"
"github.com/minio/mc/pkg/probe"
)
@ -97,34 +98,29 @@ EXAMPLES:
`,
}
func (s speedTestResult) StringVerbose() (msg string) {
result := s.result
if globalPerfTestVerbose {
msg += "\n\n"
msg += "PUT:\n"
for _, node := range result.PUTStats.Servers {
msg += fmt.Sprintf(" * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
if node.Err != "" {
msg += " Err: " + node.Err
}
msg += "\n"
func objectTestVerboseResult(result *madmin.SpeedTestResult) (msg string) {
msg += "PUT:\n"
for _, node := range result.PUTStats.Servers {
msg += fmt.Sprintf(" * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
if node.Err != "" {
msg += " Err: " + node.Err
}
msg += "GET:\n"
for _, node := range result.GETStats.Servers {
msg += fmt.Sprintf(" * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
if node.Err != "" {
msg += " Err: " + node.Err
}
msg += "\n"
}
msg += "\n"
}
msg += "GET:\n"
for _, node := range result.GETStats.Servers {
msg += fmt.Sprintf(" * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
if node.Err != "" {
msg += " Err: " + node.Err
}
msg += "\n"
}
return msg
}
func (s speedTestResult) String() (msg string) {
result := s.result
func objectTestShortResult(result *madmin.SpeedTestResult) (msg string) {
msg += fmt.Sprintf("MinIO %s, %d servers, %d drives, %s objects, %d threads",
result.Version, result.Servers, result.Disks,
humanize.IBytes(uint64(result.Size)), result.Concurrent)
@ -132,8 +128,12 @@ func (s speedTestResult) String() (msg string) {
return msg
}
func (s speedTestResult) String() string {
return ""
}
func (s speedTestResult) JSON() string {
JSONBytes, e := json.MarshalIndent(s.result, "", " ")
JSONBytes, e := json.MarshalIndent(s, "", " ")
fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
return string(JSONBytes)
}