minio/mc
1
0
Fork 0
mirror of https://github.com/minio/mc.git synced 2025-07-30 07:23:03 +03:00
Code Activity

Upload results of perf tests to SUBNET (#4254)

Browse Source
This commit is contained in:
Shireesh Anjal
2022-09-27 15:26:43 +05:30
committed by GitHub
parent 698e9ad094
commit ba5f9ac373
6 changed files with 189 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/subnet-utils.go
View File

@ -494,15 +494,15 @@ func getSubnetCreds(alias string) (string, string, error) {
// getSubnetAPIKey - returns the SUBNET API key.
// Returns error if the cluster is not registered with SUBNET.
func getSubnetAPIKey(alias string) (string, error) {
apiKey, _, e := getSubnetCreds(alias)
apiKey, lic, e := getSubnetCreds(alias)
if e != nil {
return "", e
}
if len(apiKey) > 0 {
return apiKey, nil
}
e = fmt.Errorf("Please register the cluster first by running 'mc support register %s', or use --airgap flag", alias)
if len(apiKey) == 0 && len(lic) == 0 {
e = fmt.Errorf("Please register the cluster first by running 'mc license register %s'", alias)
return "", e
}
return apiKey, nil
}
func getSubnetAPIKeyUsingLicense(lic string) (string, error) {

19
cmd/support-perf-drive.go
View File

@ -28,7 +28,7 @@ import (
"github.com/minio/mc/pkg/probe"
)
func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string, outCh chan<- PerfTestResult) error {
client, perr := newAdminClient(aliasedURL)
if perr != nil {
fatalIf(perr.Trace(aliasedURL), "Unable to initialize admin client.")
@ -103,10 +103,15 @@ func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
go func() {
if e != nil {
printMsg(PerfTestResult{
r := PerfTestResult{
Type: DrivePerfTest,
Err: e.Error(),
})
Final: true,
}
p.Send(r)
if outCh != nil {
outCh <- r
}
return
}
@ -121,11 +126,15 @@ func mainAdminSpeedTestDrive(ctx *cli.Context, aliasedURL string) error {
})
}
}
p.Send(PerfTestResult{
r := PerfTestResult{
Type: DrivePerfTest,
DriveResult: results,
Final: true,
})
}
p.Send(r)
if outCh != nil {
outCh <- r
}
}()
<-done

18
cmd/support-perf-net.go
View File

@ -28,7 +28,7 @@ import (
"github.com/minio/mc/pkg/probe"
)
func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string, outCh chan<- PerfTestResult) error {
client, perr := newAdminClient(aliasedURL)
if perr != nil {
fatalIf(perr.Trace(aliasedURL), "Unable to initialize admin client.")
@ -93,18 +93,26 @@ func mainAdminSpeedTestNetperf(ctx *cli.Context, aliasedURL string) error {
for {
select {
case e := <-errorCh:
p.Send(PerfTestResult{
r := PerfTestResult{
Type: NetPerfTest,
Err: e.Error(),
Final: true,
})
}
p.Send(r)
if outCh != nil {
outCh <- r
}
return
case result := <-resultCh:
p.Send(PerfTestResult{
r := PerfTestResult{
Type: NetPerfTest,
NetResult: &result,
Final: true,
})
}
p.Send(r)
if outCh != nil {
outCh <- r
}
return
default:
p.Send(PerfTestResult{

18
cmd/support-perf-object.go
View File

@ -46,7 +46,7 @@ func mainAdminSpeedtest(ctx *cli.Context) error {
return nil
}
func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string) error {
func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string, outCh chan<- PerfTestResult) error {
client, perr := newAdminClient(aliasedURL)
if perr != nil {
fatalIf(perr.Trace(aliasedURL), "Unable to initialize admin client.")
@ -135,11 +135,15 @@ func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string) error {
go func() {
if e != nil {
p.Send(PerfTestResult{
r := PerfTestResult{
Type: ObjectPerfTest,
Err: e.Error(),
Final: true,
})
}
p.Send(r)
if outCh != nil {
outCh <- r
}
return
}
@ -150,11 +154,15 @@ func mainAdminSpeedTestObject(ctx *cli.Context, aliasedURL string) error {
ObjectResult: &result,
})
}
p.Send(PerfTestResult{
r := PerfTestResult{
Type: ObjectPerfTest,
ObjectResult: &result,
Final: true,
})
}
p.Send(r)
if outCh != nil {
outCh <- r
}
}()
<-done

158
cmd/support-perf.go
View File

@ -18,16 +18,22 @@
package cmd
import (
"archive/zip"
gojson "encoding/json"
"fmt"
"os"
"path/filepath"
humanize "github.com/dustin/go-humanize"
"github.com/fatih/color"
"github.com/minio/cli"
json "github.com/minio/colorjson"
"github.com/minio/madmin-go"
"github.com/minio/mc/pkg/probe"
"github.com/minio/pkg/console"
)
var supportPerfFlags = []cli.Flag{
var supportPerfFlags = append([]cli.Flag{
cli.StringFlag{
Name: "duration",
Usage: "duration the entire perf tests are run",
@ -72,7 +78,7 @@ var supportPerfFlags = []cli.Flag{
Usage: "run tests on drive(s) one-by-one",
Hidden: true,
},
}
}, subnetCommonFlags...)
var supportPerfCmd = cli.Command{
Name: "perf",
@ -146,6 +152,7 @@ func mainSupportPerf(ctx *cli.Context) error {
// the alias parameter from cli
aliasedURL := ""
perfType := ""
switch len(args) {
case 1:
// cannot use alias by the name 'drive' or 'net'
@ -154,24 +161,143 @@ func mainSupportPerf(ctx *cli.Context) error {
}
aliasedURL = args[0]
mainAdminSpeedTestNetperf(ctx, aliasedURL)
mainAdminSpeedTestDrive(ctx, aliasedURL)
mainAdminSpeedTestObject(ctx, aliasedURL)
case 2:
aliasedURL := args[1]
switch args[0] {
case "drive":
return mainAdminSpeedTestDrive(ctx, aliasedURL)
case "object":
return mainAdminSpeedTestObject(ctx, aliasedURL)
case "net":
return mainAdminSpeedTestNetperf(ctx, aliasedURL)
default:
cli.ShowCommandHelpAndExit(ctx, "perf", 1) // last argument is exit code
}
perfType = args[0]
aliasedURL = args[1]
default:
cli.ShowCommandHelpAndExit(ctx, "perf", 1) // last argument is exit code
}
// Main execution
execSupportPerf(ctx, aliasedURL, perfType)
return nil
}
func execSupportPerf(ctx *cli.Context, aliasedURL string, perfType string) {
alias, apiKey := initSubnetConnectivity(ctx, aliasedURL)
if len(apiKey) == 0 {
// api key not passed as flag. check if it's available in the config
var e error
apiKey, e = getSubnetAPIKey(alias)
// Non-registered execution allowed only in debug+airgapped mode
if !(globalDebug && globalAirgapped) {
fatalIf(probe.NewError(e), "Unable to retrieve SUBNET API key")
}
}
results := runPerfTests(ctx, aliasedURL, perfType)
if globalJSON {
// No file to be saved or uploaded to SUBNET in case of `--json`
return
}
resultFileNamePfx := fmt.Sprintf("%s-perf_%s", filepath.Clean(alias), UTCNow().Format("20060102150405"))
resultFileName := resultFileNamePfx + ".json"
regInfo := getClusterRegInfo(getAdminInfo(aliasedURL), alias)
tmpFileName, e := zipPerfResult(results, resultFileName, regInfo)
fatalIf(probe.NewError(e), "Error creating zip from perf test results:")
if globalAirgapped {
savePerfResultFile(tmpFileName, resultFileNamePfx, alias)
return
}
uploadURL := subnetUploadURL("perf", tmpFileName)
reqURL, headers := prepareSubnetUploadURL(uploadURL, alias, tmpFileName, apiKey)
_, e = uploadFileToSubnet(alias, tmpFileName, reqURL, headers)
if e != nil {
console.Errorln("Unable to upload perf test results to SUBNET portal: " + e.Error())
savePerfResultFile(tmpFileName, resultFileNamePfx, alias)
return
}
clr := color.New(color.FgGreen, color.Bold)
clr.Println("uploaded successfully to SUBNET.")
if len(apiKey) > 0 {
setSubnetAPIKey(alias, apiKey)
}
}
func savePerfResultFile(tmpFileName string, resultFileNamePfx string, alias string) {
zipFileName := resultFileNamePfx + ".zip"
e := moveFile(tmpFileName, zipFileName)
fatalIf(probe.NewError(e), fmt.Sprintf("Error moving temp file %s to %s:", tmpFileName, zipFileName))
console.Infoln("MinIO performance report saved at", zipFileName)
}
func runPerfTests(ctx *cli.Context, aliasedURL string, perfType string) []PerfTestResult {
resultCh := make(chan PerfTestResult)
results := []PerfTestResult{}
defer close(resultCh)
tests := []string{perfType}
if len(perfType) == 0 {
// by default run all tests
tests = []string{"net", "drive", "object"}
}
for _, t := range tests {
switch t {
case "drive":
mainAdminSpeedTestDrive(ctx, aliasedURL, resultCh)
case "object":
mainAdminSpeedTestObject(ctx, aliasedURL, resultCh)
case "net":
mainAdminSpeedTestNetperf(ctx, aliasedURL, resultCh)
default:
cli.ShowCommandHelpAndExit(ctx, "perf", 1) // last argument is exit code
}
if !globalJSON {
results = append(results, <-resultCh)
}
}
return results
}
func writeJSONObjToZip(zipWriter *zip.Writer, obj interface{}, filename string) error {
writer, e := zipWriter.Create(filename)
if e != nil {
return e
}
enc := gojson.NewEncoder(writer)
if e = enc.Encode(obj); e != nil {
return e
}
return nil
}
// compress MinIO performance output
func zipPerfResult(perfResult []PerfTestResult, resultFilename string, regInfo ClusterRegistrationInfo) (string, error) {
// Create profile zip file
tmpArchive, e := os.CreateTemp("", "mc-perf-")
if e != nil {
return "", e
}
defer tmpArchive.Close()
zipWriter := zip.NewWriter(tmpArchive)
defer zipWriter.Close()
e = writeJSONObjToZip(zipWriter, perfResult, resultFilename)
if e != nil {
return "", e
}
e = writeJSONObjToZip(zipWriter, regInfo, "cluster.info")
if e != nil {
return "", e
}
return tmpArchive.Name(), nil
}

5
cmd/support-profile.go
View File

@ -170,11 +170,6 @@ func mainSupportProfile(ctx *cli.Context) error {
aliasedURL := ctx.Args().Get(0)
alias, apiKey := initSubnetConnectivity(ctx, aliasedURL)
// if `--airgap` is provided do not try to upload to SUBNET.
if !globalAirgapped {
fatalIf(checkURLReachable(subnetBaseURL()).Trace(aliasedURL), "Unable to reach %s to upload MinIO profile file, please use --airgap to upload manually", subnetBaseURL())
}
// Create a new MinIO Admin Client
client := getClient(aliasedURL)