Add tier-info command for tier-specific stats (#3827)

cmd/admin-tier-info.go

@@ -0,0 +1,224 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"errors"
+	"strconv"
+
+	"github.com/dustin/go-humanize"
+	"github.com/fatih/color"
+	"github.com/minio/cli"
+	json "github.com/minio/colorjson"
+	"github.com/minio/madmin-go"
+	"github.com/minio/mc/pkg/probe"
+	"github.com/minio/pkg/console"
+)
+
+var adminTierInfoCmd = cli.Command{
+	Name:         "info",
+	Usage:        "Displays per-tier statistics of all tier targets",
+	Action:       mainAdminTierInfo,
+	OnUsageError: onUsageError,
+	Before:       setGlobalsFromContext,
+	Flags:        globalFlags,
+	CustomHelpTemplate: `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} TARGET
+
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}
+
+EXAMPLES:
+  1. Prints per-tier statistics of all remote tier targets configured in myminio
+     {{.Prompt}} {{.HelpName}} myminio
+`,
+}
+
+// checkAdminTierInfoSyntax - validate all the passed arguments
+func checkAdminTierInfoSyntax(ctx *cli.Context) {
+	argsNr := len(ctx.Args())
+	if argsNr < 1 {
+		cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1) // last argument is exit code
+	}
+	if argsNr > 1 {
+		fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...),
+			"Incorrect number of arguments for tier-info subcommand.")
+	}
+}
+
+type tierInfoRowHdr int
+
+const (
+	tierInfoNameHdr tierInfoRowHdr = iota
+	tierInfoAPIHdr
+	tierInfoTypeHdr
+	tierInfoUsageHdr
+	tierInfoObjectsHdr
+	tierInfoVersionsHdr
+)
+
+var tierInfoRowNames = []string{
+	"Tier Name",
+	"API",
+	"Type",
+	"Usage",
+	"Objects",
+	"Versions",
+}
+
+var tierInfoColorScheme = []*color.Color{
+	color.New(color.FgYellow),
+	color.New(color.FgCyan),
+	color.New(color.FgCyan),
+	color.New(color.FgHiWhite),
+	color.New(color.FgHiWhite),
+	color.New(color.FgHiWhite),
+}
+
+type tierInfos []madmin.TierInfo
+
+func (t tierInfos) NumRows() int {
+	return len([]madmin.TierInfo(t))
+}
+
+func (t tierInfos) NumCols() int {
+	return len(tierInfoRowNames)
+}
+func (t tierInfos) EmptyMessage() string {
+	return "No remote tiers configured."
+}
+
+func (t tierInfos) MarshalJSON() ([]byte, error) {
+	type tierInfo struct {
+		Name  string
+		API   string
+		Type  string
+		Stats madmin.TierStats
+	}
+	ts := make([]tierInfo, 0, len(t))
+	for _, tInfo := range t {
+		ts = append(ts, tierInfo{
+			Name:  tInfo.Name,
+			API:   tierInfoAPI(tInfo.Type),
+			Type:  tierInfoType(tInfo.Type),
+			Stats: tInfo.Stats,
+		})
+	}
+	return json.Marshal(ts)
+}
+
+func tierInfoAPI(tierType string) string {
+	switch tierType {
+	case madmin.S3.String(), madmin.GCS.String():
+		return tierType
+	case madmin.Azure.String():
+		return "blob"
+	case "internal":
+		return madmin.S3.String()
+	default:
+		return "unknown"
+	}
+}
+
+func tierInfoType(tierType string) string {
+	if tierType == "internal" {
+		return "hot"
+	}
+	return "warm"
+}
+
+func (t tierInfos) ToRow(i int, ls []int) []string {
+	row := make([]string, len(tierInfoRowNames))
+	if i == -1 {
+		copy(row, tierInfoRowNames)
+	} else {
+		tierInfo := t[i]
+		row[tierInfoNameHdr] = tierInfo.Name
+		row[tierInfoAPIHdr] = tierInfoAPI(tierInfo.Type)
+		row[tierInfoTypeHdr] = tierInfoType(tierInfo.Type)
+		row[tierInfoUsageHdr] = humanize.IBytes(tierInfo.Stats.TotalSize)
+		row[tierInfoObjectsHdr] = strconv.Itoa(tierInfo.Stats.NumObjects)
+		row[tierInfoVersionsHdr] = strconv.Itoa(tierInfo.Stats.NumVersions)
+	}
+
+	// update ls to accommodate this row's values
+	for i := range tierInfoRowNames {
+		if ls[i] < len(row[i]) {
+			ls[i] = len(row[i])
+		}
+	}
+	return row
+}
+
+func mainAdminTierInfo(ctx *cli.Context) error {
+	checkAdminTierInfoSyntax(ctx)
+
+	for i, color := range tierInfoColorScheme {
+		console.SetColor(tierInfoRowNames[i], color)
+	}
+
+	args := ctx.Args()
+	aliasedURL := args.Get(0)
+
+	// Create a new MinIO Admin Client
+	client, cerr := newAdminClient(aliasedURL)
+	fatalIf(cerr, "Unable to initialize admin connection.")
+
+	var msg tierInfoMessage
+	tInfos, err := client.TierStats(globalContext)
+	if err != nil {
+		msg = tierInfoMessage{
+			Status:  "error",
+			Context: ctx,
+			Error:   err.Error(),
+		}
+	} else {
+		msg = tierInfoMessage{
+			Status:    "success",
+			Context:   ctx,
+			TierInfos: tierInfos(tInfos),
+		}
+	}
+	printMsg(&msg)
+	return nil
+}
+
+type tierInfoMessage struct {
+	Status    string       `json:"status"`
+	Context   *cli.Context `json:"-"`
+	TierInfos tierInfos    `json:"tiers,omitempty"`
+	Error     string       `json:"error,omitempty"`
+}
+
+// String method returns a tabular listing of remote tier configurations.
+func (msg *tierInfoMessage) String() string {
+	if msg.Status == "error" {
+		fatal(probe.NewError(errors.New(msg.Error)), "Unable to get tier statistics")
+	}
+	return toTable(tierInfos(msg.TierInfos))
+}
+
+// JSON method returns JSON encoding of msg.
+func (msg *tierInfoMessage) JSON() string {
+	b, _ := json.Marshal(msg)
+	return string(b)
+}

cmd/admin-tier-listing.go

@@ -0,0 +1,42 @@
+package cmd
+
+import "fmt"
+
+type tabulator interface {
+	ToRow(i int, lengths []int) []string // returns row representation i-th element in collection, modifies lengths s.t it contains maximum column widths incl this row.
+	NumRows() int
+	NumCols() int
+	EmptyMessage() string
+}
+
+func toTable(tbl tabulator) string {
+	if tbl.NumRows() == 0 {
+		return tbl.EmptyMessage()
+	}
+
+	const tableSeparator = "|"
+	rows, cols := getRowsAndCols(tbl)
+	table := newPrettyTable(tableSeparator, cols...)
+	var contents string
+	for _, row := range rows {
+		contents += fmt.Sprintf("%s\n", table.buildRow(row...))
+	}
+	return contents
+}
+
+func getRowsAndCols(tbl tabulator) ([][]string, []Field) {
+	rows := make([][]string, 0, tbl.NumRows()+1)
+	lengths := make([]int, tbl.NumCols())
+	rows = append(rows, tbl.ToRow(-1, lengths))
+	for i := 0; i < tbl.NumRows(); i++ {
+		rows = append(rows, tbl.ToRow(i, lengths))
+	}
+	cols := make([]Field, tbl.NumCols())
+	for i, hdr := range rows[0] {
+		cols[i] = Field{
+			colorTheme: hdr,
+			maxLen:     lengths[i] + 2,
+		}
+	}
+	return rows, cols
+}

cmd/admin-tier-ls.go

@@ -18,8 +18,6 @@
 package cmd
 
 import (
-	"fmt"
-
 	"github.com/fatih/color"
 	"github.com/minio/cli"
 	json "github.com/minio/colorjson"
@@ -63,19 +61,19 @@ func checkAdminTierListSyntax(ctx *cli.Context) {
 	}
 }
 
-type tierRowHdr int
+type tierLSRowHdr int
 
 const (
-	tierNameHdr tierRowHdr = iota
-	tierTypeHdr
-	tierEndpointHdr
-	tierBucketHdr
-	tierPrefixHdr
-	tierRegionHdr
-	tierStorageClassHdr
+	tierLSNameHdr tierLSRowHdr = iota
+	tierLSTypeHdr
+	tierLSEndpointHdr
+	tierLSBucketHdr
+	tierLSPrefixHdr
+	tierLSRegionHdr
+	tierLSStorageClassHdr
 )
 
-var tierRowNames = []string{
+var tierLSRowNames = []string{
 	"Name",
 	"Type",
 	"Endpoint",
@@ -85,7 +83,7 @@ var tierRowNames = []string{
 	"Storage-Class",
 }
 
-var tierColorScheme = []*color.Color{
+var tierLSColorScheme = []*color.Color{
 	color.New(color.FgYellow),
 	color.New(color.FgCyan),
 	color.New(color.FgGreen),
@@ -108,52 +106,45 @@ func storageClass(t *madmin.TierConfig) string {
 	}
 }
 
-type tierCfg struct {
-	*madmin.TierConfig
+type tierLS []*madmin.TierConfig
+
+func (t tierLS) NumRows() int {
+	return len(([]*madmin.TierConfig)(t))
 }
 
-func (tc *tierCfg) toRow(lengths []int) []string {
-	row := make([]string, len(tierRowNames))
-	row[tierNameHdr] = tc.Name
-	row[tierTypeHdr] = tc.Type.String()
-	row[tierEndpointHdr] = tc.Endpoint()
-	row[tierBucketHdr] = tc.Bucket()
-	row[tierPrefixHdr] = tc.Prefix()
-	row[tierRegionHdr] = tc.Region()
-	row[tierStorageClassHdr] = storageClass(tc.TierConfig)
-	for i := range tierRowNames {
-		if lengths[i] < len(row[i]) {
-			lengths[i] = len(row[i])
+func (t tierLS) NumCols() int {
+	return len(tierLSRowNames)
+}
+func (t tierLS) EmptyMessage() string {
+	return "No remote tier has been configured"
+}
+
+func (t tierLS) ToRow(i int, ls []int) []string {
+	row := make([]string, len(tierLSRowNames))
+	if i == -1 {
+		copy(row, tierLSRowNames)
+	} else {
+		tc := t[i]
+		row[tierLSNameHdr] = tc.Name
+		row[tierLSTypeHdr] = tc.Type.String()
+		row[tierLSEndpointHdr] = tc.Endpoint()
+		row[tierLSBucketHdr] = tc.Bucket()
+		row[tierLSPrefixHdr] = tc.Prefix()
+		row[tierLSRegionHdr] = tc.Region()
+		row[tierLSStorageClassHdr] = storageClass(tc)
+
+	}
+
+	// update ls to accommodate this row's values
+	for i := range tierLSRowNames {
+		if ls[i] < len(row[i]) {
+			ls[i] = len(row[i])
 		}
 	}
 	return row
 }
 
-// getTierListRowsAndCols returns a list of rows and a list of column header
-// metadata like color theme and max cell length, given a list of tiers. Each
-// row is represented by a list of cells in that row.
-func getTierListRowsAndCols(tiers []*madmin.TierConfig) ([][]string, []Field) {
-	rows := make([][]string, len(tiers))
-	rows[0] = tierRowNames
-	lengths := make([]int, len(rows[0]))
-	for i := range lengths {
-		lengths[i] = len(rows[0][i])
-	}
-	for _, tier := range tiers {
-		tierCfg := tierCfg{tier}
-		rows = append(rows, tierCfg.toRow(lengths))
-	}
-	// add 2 spaces to each column's max length to improve readability of
-	// each cell
-	cols := make([]Field, len(tierRowNames))
-	for i, hdr := range rows[0] {
-		cols[i] = Field{
-			colorTheme: hdr,
-			maxLen:     lengths[i] + 2,
-		}
-	}
-	return rows, cols
-}
+var _ tabulator = (tierLS)(nil)
 
 type tierListMessage struct {
 	Status  string               `json:"status"`
@@ -163,18 +154,7 @@ type tierListMessage struct {
 
 // String method returns a tabular listing of remote tier configurations.
 func (msg *tierListMessage) String() string {
-	if len(msg.Tiers) == 0 {
-		return "No remote tier has been configured"
-	}
-
-	const tableSeparator = "|"
-	rows, cols := getTierListRowsAndCols(msg.Tiers)
-	tbl := newPrettyTable(tableSeparator, cols...)
-	var contents string
-	for _, row := range rows {
-		contents += fmt.Sprintf("%s\n", tbl.buildRow(row...))
-	}
-	return contents
+	return toTable(tierLS(msg.Tiers))
 }
 
 // JSON method returns JSON encoding of msg.
@@ -186,8 +166,8 @@ func (msg *tierListMessage) JSON() string {
 func mainAdminTierList(ctx *cli.Context) error {
 	checkAdminTierListSyntax(ctx)
 
-	for i, color := range tierColorScheme {
-		console.SetColor(tierRowNames[i], color)
+	for i, color := range tierLSColorScheme {
+		console.SetColor(tierLSRowNames[i], color)
 	}
 
 	args := ctx.Args()

cmd/admin-tier-main.go

@@ -23,11 +23,12 @@ var adminTierSubCommands = []cli.Command{
 	adminTierAddCmd,
 	adminTierListCmd,
 	adminTierEditCmd,
+	adminTierInfoCmd,
 }
 
 var adminTierCmd = cli.Command{
 	Name:            "tier",
-	Usage:           "configure remote tier targets for ILM transition",
+	Usage:           "manage remote tier targets for ILM transition",
 	Action:          mainAdminTier,
 	Before:          setGlobalsFromContext,
 	Flags:           globalFlags,

cmd/auto-complete.go

@@ -376,6 +376,7 @@ var completeCmds = map[string]complete.Predictor{
 	"/admin/tier/add":  nil,
 	"/admin/tier/edit": nil,
 	"/admin/tier/ls":   nil,
+	"/admin/tier/info": nil,
 
 	"/admin/replicate/add":  aliasCompleter,
 	"/admin/replicate/info": aliasCompleter,

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0 // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fatih/color v1.13.0
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/goccy/go-json v0.7.9 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/uuid v1.3.0 // indirect
@@ -51,8 +52,8 @@ require (
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b
-	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
+	golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7
 	google.golang.org/genproto v0.0.0-20210928142010-c7af6a1a74c9 // indirect

go.sum

@@ -110,8 +110,9 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
-github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/goccy/go-json v0.4.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.7.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
@@ -509,8 +510,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b h1:eB48h3HiRycXNy8E0Gf5e0hv7YT6Kt14L/D73G1fuwo=
-golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -580,8 +581,8 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk=
+golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=