profile: Add MC_PROFILER env supporting cpu, mem, block profiling. (#1714)

All profiled outputs are written inside `~/.mc/profile`

globals.go

@@ -37,6 +37,9 @@ const (
 	// session config and shared urls related constants
 	globalSessionDir        = "session"
 	globalSharedURLsDataDir = "share"
+
+	// Profile directory for dumping profiler outputs.
+	globalProfileDir = "profile"
 )
 
 var (

main.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
 	"sort"
 	"strconv"
@@ -28,6 +29,7 @@ import (
 	"github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/pkg/probe"
 	"github.com/minio/pb"
+	"github.com/pkg/profile"
 )
 
 var (
@@ -228,7 +230,23 @@ func registerApp() *cli.App {
 	return app
 }
 
+// mustGetProfilePath must get location that the profile will be written to.
+func mustGetProfileDir() string {
+	return filepath.Join(mustGetMcConfigDir(), globalProfileDir)
+}
+
 func main() {
+	// Enable profiling supported modes are [cpu, mem, block].
+	// ``MC_PROFILER`` supported options are [cpu, mem, block].
+	switch os.Getenv("MC_PROFILER") {
+	case "cpu":
+		defer profile.Start(profile.CPUProfile, profile.ProfilePath(mustGetProfileDir())).Stop()
+	case "mem":
+		defer profile.Start(profile.MemProfile, profile.ProfilePath(mustGetProfileDir())).Stop()
+	case "block":
+		defer profile.Start(profile.BlockProfile, profile.ProfilePath(mustGetProfileDir())).Stop()
+	}
+
 	probe.Init() // Set project's root source path.
 	probe.SetAppInfo("Release-Tag", mcReleaseTag)
 	probe.SetAppInfo("Commit", mcShortCommitID)

vendor/github.com/pkg/profile/AUTHORS

@@ -0,0 +1 @@
+Dave Cheney <dave@cheney.net>

vendor/github.com/pkg/profile/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2013 Dave Cheney. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/pkg/profile/README.md

@@ -0,0 +1,44 @@
+profile
+=======
+
+Simple profiling support package for Go
+
+installation
+------------
+
+    go get github.com/pkg/profile
+
+usage
+-----
+
+Enabling profiling in your application is as simple as one line at the top of your main function
+
+```go
+import "github.com/pkg/profile"
+
+func main() {
+    defer profile.Start().Stop()
+    ...
+}
+```
+
+options
+-------
+
+What to profile is controlled by config value passed to profile.Start. 
+By default CPU profiling is enabled.
+
+```go
+import "github.com/pkg/profile"
+
+func main() {
+    // p.Stop() must be called before the program exits to
+    // ensure profiling information is written to disk.
+    p := profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.NoShutdownHook)
+    ...
+}
+```
+
+Several convenience package level values are provided for cpu, memory, and block (contention) profiling.
+
+For more complex options, consult the [documentation](http://godoc.org/github.com/pkg/profile).

vendor/github.com/pkg/profile/profile.go

@@ -0,0 +1,199 @@
+// Package profile provides a simple way to manage runtime/pprof
+// profiling of your Go application.
+package profile
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"sync/atomic"
+)
+
+// started counts the number of times Start has been called
+var started uint32
+
+const (
+	cpuMode = iota
+	memMode
+	blockMode
+)
+
+type profile struct {
+	// quiet suppresses informational messages during profiling.
+	quiet bool
+
+	// noShutdownHook controls whether the profiling package should
+	// hook SIGINT to write profiles cleanly.
+	noShutdownHook bool
+
+	// mode holds the type of profiling that will be made
+	mode int
+
+	// path holds the base path where various profiling files are  written.
+	// If blank, the base path will be generated by ioutil.TempDir.
+	path string
+
+	// memProfileRate holds the rate for the memory profile.
+	memProfileRate int
+
+	// closers holds the cleanup functions that run after each profile
+	closers []func()
+
+	// stopped records if a call to profile.Stop has been made
+	stopped uint32
+}
+
+// NoShutdownHook controls whether the profiling package should
+// hook SIGINT to write profiles cleanly.
+// Programs with more sophisticated signal handling should set
+// this to true and ensure the Stop() function returned from Start()
+// is called during shutdown.
+func NoShutdownHook(p *profile) { p.noShutdownHook = true }
+
+// Quiet suppresses informational messages during profiling.
+func Quiet(p *profile) { p.quiet = true }
+
+// CPUProfile controls if cpu profiling will be enabled. It disables any previous profiling settings.
+func CPUProfile(p *profile) { p.mode = cpuMode }
+
+// DefaultMemProfileRate is the default memory profiling rate.
+// See also http://golang.org/pkg/runtime/#pkg-variables
+const DefaultMemProfileRate = 4096
+
+// MemProfile controls if memory profiling will be enabled. It disables any previous profiling settings.
+func MemProfile(p *profile) {
+	p.memProfileRate = DefaultMemProfileRate
+	p.mode = memMode
+}
+
+// MemProfileRate controls if memory profiling will be enabled. Additionally, it takes a parameter which
+// allows the setting of the memory profile rate.
+func MemProfileRate(rate int) func(*profile) {
+	return func(p *profile) {
+		p.memProfileRate = rate
+		p.mode = memMode
+	}
+}
+
+// BlockProfile controls if block (contention) profiling will be enabled. It disables any previous profiling settings.
+func BlockProfile(p *profile) { p.mode = blockMode }
+
+// ProfilePath controls the base path where various profiling
+// files are written. If blank, the base path will be generated
+// by ioutil.TempDir.
+func ProfilePath(path string) func(*profile) {
+	return func(p *profile) {
+		p.path = path
+	}
+}
+
+// Stop stops the profile and flushes any unwritten data.
+func (p *profile) Stop() {
+	if !atomic.CompareAndSwapUint32(&p.stopped, 0, 1) {
+		// someone has already called close
+		return
+	}
+	for _, c := range p.closers {
+		c()
+	}
+}
+
+// Start starts a new profiling session.
+// The caller should call the Stop method on the value returned
+// to cleanly stop profiling.
+func Start(options ...func(*profile)) interface {
+	Stop()
+} {
+	if !atomic.CompareAndSwapUint32(&started, 0, 1) {
+		log.Fatal("profile: Start() already called")
+	}
+
+	var prof profile
+	for _, option := range options {
+		option(&prof)
+	}
+
+	path, err := func() (string, error) {
+		if p := prof.path; p != "" {
+			return p, os.MkdirAll(p, 0777)
+		}
+		return ioutil.TempDir("", "profile")
+	}()
+
+	if err != nil {
+		log.Fatalf("profile: could not create initial output directory: %v", err)
+	}
+
+	switch prof.mode {
+	case cpuMode:
+		fn := filepath.Join(path, "cpu.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create cpu profile %q: %v", fn, err)
+		}
+		if !prof.quiet {
+			log.Printf("profile: cpu profiling enabled, %s", fn)
+		}
+		pprof.StartCPUProfile(f)
+		prof.closers = append(prof.closers, func() {
+			pprof.StopCPUProfile()
+			f.Close()
+		})
+
+	case memMode:
+		fn := filepath.Join(path, "mem.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create memory profile %q: %v", fn, err)
+		}
+		old := runtime.MemProfileRate
+		runtime.MemProfileRate = prof.memProfileRate
+		if !prof.quiet {
+			log.Printf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, fn)
+		}
+		prof.closers = append(prof.closers, func() {
+			pprof.Lookup("heap").WriteTo(f, 0)
+			f.Close()
+			runtime.MemProfileRate = old
+		})
+
+	case blockMode:
+		fn := filepath.Join(path, "block.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create block profile %q: %v", fn, err)
+		}
+		runtime.SetBlockProfileRate(1)
+		if !prof.quiet {
+			log.Printf("profile: block profiling enabled, %s", fn)
+		}
+		prof.closers = append(prof.closers, func() {
+			pprof.Lookup("block").WriteTo(f, 0)
+			f.Close()
+			runtime.SetBlockProfileRate(0)
+		})
+	}
+
+	if !prof.noShutdownHook {
+		go func() {
+			c := make(chan os.Signal, 1)
+			signal.Notify(c, os.Interrupt)
+			<-c
+
+			log.Println("profile: caught interrupt, stopping profiles")
+			prof.Stop()
+
+			os.Exit(0)
+		}()
+	}
+
+	prof.closers = append(prof.closers, func() {
+		atomic.SwapUint32(&started, 0)
+	})
+
+	return &prof
+}

vendor/github.com/pkg/profile/wercker.yml

@@ -0,0 +1 @@
+box: wercker/golang

vendor/vendor.json

@@ -77,6 +77,11 @@
 			"revision": "ecf753e7c962639ab5a1fb46f7da627d4c0a04b8",
 			"revisionTime": "2014-04-12T15:01:45-07:00"
 		},
+		{
+			"path": "github.com/pkg/profile",
+			"revision": "19396ecfda37c1c9ddaa52019adca59d582e4832",
+			"revisionTime": "2016-03-15T12:21:13+11:00"
+		},
 		{
 			"path": "golang.org/x/sys/unix",
 			"revision": "50c6bc5e4292a1d4e65c6e9be5f53be28bcbe28e",