Change the JSON output of mc support perf (#4198)

Change the JSON output of mc support perf

The new change will still print multiple JSON lines for each conducted
performance test but now we have a new field called 'type' field to
difference between network, drive and object tests.

This change also adds 'final' field to indicate this is the final json of a
particular performance testing.

Also, errors when conducting those tests will have a field 'err' as well
in the JSON output.

cmd/speedtest-spinner.go

@@ -40,12 +40,33 @@ type speedTestUI struct {
 	result   speedTestResult
 }
 
+type speedTestType byte
+
+const (
+	netSpeedTest speedTestType = 1 << iota
+	driveSpeedTest
+	objectSpeedTest
+)
+
+func (s speedTestType) Name() string {
+	switch s {
+	case netSpeedTest:
+		return "NetPerf"
+	case driveSpeedTest:
+		return "DrivePerf"
+	case objectSpeedTest:
+		return "ObjectPerf"
+	}
+	return "<unknown>"
+}
+
 type speedTestResult struct {
-	err     error
+	Type         speedTestType                 `json:"type"`
-	final   bool
+	ObjectResult *madmin.SpeedTestResult       `json:"object,omitempty"`
-	result  *madmin.SpeedTestResult
+	NetResult    *madmin.NetperfResult         `json:"network,omitempty"`
-	nresult *madmin.NetperfResult
+	DriveResult  []madmin.DriveSpeedTestResult `json:"drive,omitempty"`
-	dresult []madmin.DriveSpeedTestResult
+	Err          string                        `json:"err,omitempty"`
+	Final        bool                          `json:"final,omitempty"`
 }
 
 func initSpeedTestUI() *speedTestUI {
@@ -73,7 +94,7 @@ func (m *speedTestUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 		}
 	case speedTestResult:
 		m.result = msg
-		if msg.final {
+		if msg.Final {
 			m.quitting = true
 			return m, tea.Quit
 		}
@@ -89,15 +110,8 @@ func (m *speedTestUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 
 func (m *speedTestUI) View() string {
 	// Quit when there is an error
-	if m.result.err != nil {
+	if m.result.Err != "" {
-		errMsg := ""
+		return fmt.Sprintf("\n%s: ✗ (Err: %s)\n", m.result.Type.Name(), m.result.Err)
-		errResp := madmin.ToErrorResponse(m.result.err)
-		if errResp.Code == "NotImplemented" {
-			errMsg = "Not implemented"
-		} else {
-			errMsg = m.result.err.Error()
-		}
-		return fmt.Sprintf("\nNetperf: ✗ (Err: %s)\n", errMsg)
 	}
 
 	var s strings.Builder
@@ -117,9 +131,9 @@ func (m *speedTestUI) View() string {
 	table.SetTablePadding("\t") // pad with tabs
 	table.SetNoWhiteSpace(true)
 
-	res := m.result.result
+	ores := m.result.ObjectResult
-	nres := m.result.nresult
+	nres := m.result.NetResult
-	dres := m.result.dresult
+	dres := m.result.DriveResult
 
 	trailerIfGreaterThan := func(in string, max int) string {
 		if len(in) < max {
@@ -128,11 +142,11 @@ func (m *speedTestUI) View() string {
 		return in[:max] + "..."
 	}
 
-	if res != nil {
+	if ores != nil {
 		table.SetHeader([]string{"", "Throughput", "IOPS"})
 		data := make([][]string, 2)
 
-		if res.Version == "" {
+		if ores.Version == "" {
 			data[0] = []string{
 				"PUT",
 				whiteStyle.Render("-- KiB/sec"),
@@ -146,26 +160,25 @@ func (m *speedTestUI) View() string {
 		} else {
 			data[0] = []string{
 				"PUT",
-				whiteStyle.Render(humanize.IBytes(res.PUTStats.ThroughputPerSec) + "/s"),
+				whiteStyle.Render(humanize.IBytes(ores.PUTStats.ThroughputPerSec) + "/s"),
-				whiteStyle.Render(humanize.Comma(int64(res.PUTStats.ObjectsPerSec)) + " objs/s"),
+				whiteStyle.Render(humanize.Comma(int64(ores.PUTStats.ObjectsPerSec)) + " objs/s"),
 			}
 			data[1] = []string{
 				"GET",
-				whiteStyle.Render(humanize.IBytes(res.GETStats.ThroughputPerSec) + "/s"),
+				whiteStyle.Render(humanize.IBytes(ores.GETStats.ThroughputPerSec) + "/s"),
-				whiteStyle.Render(humanize.Comma(int64(res.GETStats.ObjectsPerSec)) + " objs/s"),
+				whiteStyle.Render(humanize.Comma(int64(ores.GETStats.ObjectsPerSec)) + " objs/s"),
 			}
 		}
 		table.AppendBulk(data)
 		table.Render()
 
 		if m.quitting {
-			s.WriteString("\n" + m.result.String())
+			s.WriteString("\n" + objectTestShortResult(ores))
-			if vstr := m.result.StringVerbose(); vstr != "" {
+			if globalPerfTestVerbose {
-				s.WriteString(vstr + "\n")
+				s.WriteString("\n\n")
-			} else {
+				s.WriteString(objectTestVerboseResult(ores))
-				s.WriteString("\n")
 			}
-			s.WriteString("Objectperf: ✔\n")
+			s.WriteString("\n")
 		}
 	} else if nres != nil {
 		table.SetHeader([]string{"Node", "RX", "TX", ""})
@@ -204,10 +217,6 @@ func (m *speedTestUI) View() string {
 
 		table.AppendBulk(data)
 		table.Render()
-
-		if m.quitting {
-			s.WriteString("\nNetperf: ✔\n")
-		}
 	} else if dres != nil {
 		table.SetHeader([]string{"Node", "Path", "Read", "Write", ""})
 		data := make([][]string, 0, len(dres))
@@ -245,19 +254,13 @@ func (m *speedTestUI) View() string {
 		}
 		table.AppendBulk(data)
 		table.Render()
-
-		if m.quitting {
-			s.WriteString("\nDriveperf: ✔\n")
-		}
 	}
+
+	// Print the spinner
 	if !m.quitting {
-		if nres != nil {
+		s.WriteString(fmt.Sprintf("\n%s: %s", m.result.Type.Name(), m.spinner.View()))
-			s.WriteString(fmt.Sprintf("\nNetperf: %s", m.spinner.View()))
+	} else {
-		} else if res != nil {
+		s.WriteString(fmt.Sprintf("\n%s: ✔\n", m.result.Type.Name()))
-			s.WriteString(fmt.Sprintf("\nObjectperf: %s", m.spinner.View()))
-		} else if dres != nil {
-			s.WriteString(fmt.Sprintf("\nDriveperf: %s", m.spinner.View()))
-		}
 	}
 	return s.String()
 }

cmd/support-perf-drive.go

@@ -24,10 +24,8 @@ import (
 	tea "github.com/charmbracelet/bubbletea"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
-	json "github.com/minio/colorjson"
 	"github.com/minio/madmin-go"
 	"github.com/minio/mc/pkg/probe"
-	"github.com/minio/pkg/console"
 )
 
 func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
@@ -62,22 +60,34 @@ func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
 
 	serial := ctx.Bool("serial")
 
-	resultCh, e := client.DriveSpeedtest(ctxt, madmin.DriveSpeedTestOpts{
+	resultCh, speedTestErr := client.DriveSpeedtest(ctxt, madmin.DriveSpeedTestOpts{
 		Serial:    serial,
 		BlockSize: uint64(blocksize),
 		FileSize:  uint64(filesize),
 	})
-	fatalIf(probe.NewError(e), "Failed to execute drive speedtest")
 
 	if globalJSON {
+		if speedTestErr != nil {
+			printMsg(speedTestResult{
+				Type:  driveSpeedTest,
+				Err:   speedTestErr.Error(),
+				Final: true,
+			})
+			return nil
+		}
+
+		var results []madmin.DriveSpeedTestResult
 		for result := range resultCh {
 			if result.Version != "" {
-				jsonBytes, e := json.MarshalIndent(result, "", " ")
+				results = append(results, result)
-				fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
-
-				console.Println(string(jsonBytes))
 			}
 		}
+		printMsg(speedTestResult{
+			Type:        driveSpeedTest,
+			DriveResult: results,
+			Final:       true,
+		})
+
 		return nil
 	}
 
@@ -92,19 +102,29 @@ func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
 	}()
 
 	go func() {
+		if speedTestErr != nil {
+			printMsg(speedTestResult{
+				Type: driveSpeedTest,
+				Err:  speedTestErr.Error(),
+			})
+			return
+		}
+
 		var results []madmin.DriveSpeedTestResult
 		for result := range resultCh {
 			if result.Version != "" {
 				results = append(results, result)
 			} else {
 				p.Send(speedTestResult{
-					dresult: []madmin.DriveSpeedTestResult{},
+					Type:        driveSpeedTest,
+					DriveResult: []madmin.DriveSpeedTestResult{},
 				})
 			}
 		}
 		p.Send(speedTestResult{
-			dresult: results,
+			Type:        driveSpeedTest,
-			final:   true,
+			DriveResult: results,
+			Final:       true,
 		})
 	}()
 

cmd/support-perf-net.go

@@ -24,24 +24,10 @@ import (
 
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/minio/cli"
-	json "github.com/minio/colorjson"
 	"github.com/minio/madmin-go"
 	"github.com/minio/mc/pkg/probe"
 )
 
-type netperfResult madmin.NetperfResult
-
-func (m netperfResult) String() (msg string) {
-	// string version is handled by banner.
-	return ""
-}
-
-func (m netperfResult) JSON() string {
-	JSONBytes, e := json.MarshalIndent(m, "", "    ")
-	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
-	return string(JSONBytes)
-}
-
 func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
 	client, perr := newAdminClient(aliasedURL)
 	if perr != nil {
@@ -76,13 +62,21 @@ func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
 	}()
 
 	if globalJSON {
-		for {
+		select {
-			select {
+		case err := <-errorCh:
-			case result := <-resultCh:
+			printMsg(speedTestResult{
-				printMsg(netperfResult(result))
+				Type:  netSpeedTest,
-				return nil
+				Err:   err.Error(),
-			}
+				Final: true,
+			})
+		case result := <-resultCh:
+			printMsg(speedTestResult{
+				Type:      netSpeedTest,
+				NetResult: &result,
+				Final:     true,
+			})
 		}
+		return nil
 	}
 
 	done := make(chan struct{})
@@ -100,19 +94,22 @@ func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
 			select {
 			case err := <-errorCh:
 				p.Send(speedTestResult{
-					err:   err,
+					Type:  netSpeedTest,
-					final: true,
+					Err:   err.Error(),
+					Final: true,
 				})
 				return
 			case result := <-resultCh:
 				p.Send(speedTestResult{
-					nresult: &result,
+					Type:      netSpeedTest,
-					final:   true,
+					NetResult: &result,
+					Final:     true,
 				})
 				return
 			default:
 				p.Send(speedTestResult{
-					nresult: &madmin.NetperfResult{},
+					Type:      netSpeedTest,
+					NetResult: &madmin.NetperfResult{},
 				})
 				time.Sleep(100 * time.Millisecond)
 			}

cmd/support-perf-object.go

@@ -85,24 +85,41 @@ func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string) error {
 	// in all other scenarios keep auto-tuning on.
 	autotune := !ctx.IsSet("concurrent")
 
-	resultCh, err := client.Speedtest(ctxt, madmin.SpeedtestOpts{
+	resultCh, speedTestErr := client.Speedtest(ctxt, madmin.SpeedtestOpts{
 		Size:        int(size),
 		Duration:    duration,
 		Concurrency: concurrent,
 		Autotune:    autotune,
 		Bucket:      ctx.String("bucket"), // This is a hidden flag.
 	})
-	fatalIf(probe.NewError(err), "Failed to execute performance test")
 
 	if globalJSON {
-		for result := range resultCh {
+		if speedTestErr != nil {
+			printMsg(speedTestResult{
+				Type:  objectSpeedTest,
+				Err:   speedTestErr.Error(),
+				Final: true,
+			})
+			return nil
+		}
+
+		var result madmin.SpeedTestResult
+		for result = range resultCh {
 			if result.Version == "" {
 				continue
 			}
 			printMsg(speedTestResult{
-				result: &result,
+				Type:         objectSpeedTest,
+				ObjectResult: &result,
 			})
 		}
+
+		printMsg(speedTestResult{
+			Type:         objectSpeedTest,
+			ObjectResult: &result,
+			Final:        true,
+		})
+
 		return nil
 	}
 
@@ -117,15 +134,26 @@ func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string) error {
 	}()
 
 	go func() {
+		if speedTestErr != nil {
+			p.Send(speedTestResult{
+				Type:  objectSpeedTest,
+				Err:   speedTestErr.Error(),
+				Final: true,
+			})
+			return
+		}
+
 		var result madmin.SpeedTestResult
 		for result = range resultCh {
 			p.Send(speedTestResult{
-				result: &result,
+				Type:         objectSpeedTest,
+				ObjectResult: &result,
 			})
 		}
 		p.Send(speedTestResult{
-			result: &result,
+			Type:         objectSpeedTest,
-			final:  true,
+			ObjectResult: &result,
+			Final:        true,
 		})
 	}()
 

cmd/support-perf.go

@@ -23,6 +23,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	json "github.com/minio/colorjson"
+	"github.com/minio/madmin-go"
 	"github.com/minio/mc/pkg/probe"
 )
 
@@ -97,34 +98,29 @@ EXAMPLES:
 `,
 }
 
-func (s speedTestResult) StringVerbose() (msg string) {
+func objectTestVerboseResult(result *madmin.SpeedTestResult) (msg string) {
-	result := s.result
+	msg += "PUT:\n"
-	if globalPerfTestVerbose {
+	for _, node := range result.PUTStats.Servers {
-		msg += "\n\n"
+		msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
-		msg += "PUT:\n"
+		if node.Err != "" {
-		for _, node := range result.PUTStats.Servers {
+			msg += " Err: " + node.Err
-			msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
-			if node.Err != "" {
-				msg += " Err: " + node.Err
-			}
-			msg += "\n"
 		}
-
+		msg += "\n"
-		msg += "GET:\n"
-		for _, node := range result.GETStats.Servers {
-			msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
-			if node.Err != "" {
-				msg += " Err: " + node.Err
-			}
-			msg += "\n"
-		}
-
 	}
+
+	msg += "GET:\n"
+	for _, node := range result.GETStats.Servers {
+		msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
+		if node.Err != "" {
+			msg += " Err: " + node.Err
+		}
+		msg += "\n"
+	}
+
 	return msg
 }
 
-func (s speedTestResult) String() (msg string) {
+func objectTestShortResult(result *madmin.SpeedTestResult) (msg string) {
-	result := s.result
 	msg += fmt.Sprintf("MinIO %s, %d servers, %d drives, %s objects, %d threads",
 		result.Version, result.Servers, result.Disks,
 		humanize.IBytes(uint64(result.Size)), result.Concurrent)
@@ -132,8 +128,12 @@ func (s speedTestResult) String() (msg string) {
 	return msg
 }
 
+func (s speedTestResult) String() string {
+	return ""
+}
+
 func (s speedTestResult) JSON() string {
-	JSONBytes, e := json.MarshalIndent(s.result, "", "    ")
+	JSONBytes, e := json.MarshalIndent(s, "", "    ")
 	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
 	return string(JSONBytes)
 }