diff --git a/common.go b/common.go index 2dec1a50..75e77bb3 100644 --- a/common.go +++ b/common.go @@ -18,6 +18,7 @@ package main import ( "encoding/json" + "net/http" "os" "os/user" "path" @@ -33,6 +34,13 @@ func getAuthFilePath() (string, error) { return path.Join(u.HomeDir, Auth), nil } +func getNewClient(auth *s3.Auth) (*s3.Client, error) { + return &s3.Client{ + Auth: auth, + Transport: http.DefaultTransport, + }, nil +} + func getAWSEnvironment() (auth *s3.Auth, err error) { var s3Auth *os.File var accessKey, secretKey, endpoint string diff --git a/configure.go b/configure.go index 381457d8..58fc17f7 100644 --- a/configure.go +++ b/configure.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "log" "os" "path" @@ -17,16 +18,22 @@ func parseConfigureInput(c *cli.Context) (auth *s3.Auth, err error) { pathstyle := c.Bool("pathstyle") if accessKey == "" { - return nil, configAccessErr + return nil, errAccess } if secretKey == "" { - return nil, configSecretErr + return nil, errSecret } if endpoint == "" { - return nil, configEndpointErr + return nil, errEndpoint + } + + auth = &s3.Auth{ + AccessKey: accessKey, + SecretAccessKey: secretKey, + Endpoint: endpoint, + S3ForcePathStyle: pathstyle, } - auth = s3.NewAuth(accessKey, secretKey, endpoint, pathstyle) return auth, nil } @@ -39,7 +46,7 @@ func doConfigure(c *cli.Context) { log.Fatal(err) } - jAuth, err = json.Marshal(auth) + jAuth, err = json.MarshalIndent(auth, "", " ") if err != nil { log.Fatal(err) } @@ -51,11 +58,10 @@ func doConfigure(c *cli.Context) { if err != nil { log.Fatal(err) } - _, err = s3File.Write(jAuth) if err != nil { log.Fatal(err) } - log.Println("Written!", path.Join(home, Auth)) - log.Println("Now run ``mc --help`` to read on other options") + fmt.Println("") + fmt.Println("Configuration written to", path.Join(home, Auth)) } diff --git a/pkg/s3/auth.go b/pkg/s3/auth.go index d87c5f02..69adf38e 100644 --- a/pkg/s3/auth.go +++ b/pkg/s3/auth.go @@ -56,8 +56,7 @@ import ( "time" ) -// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html - +// Auth - see http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html type Auth struct { AccessKey string SecretAccessKey string @@ -73,7 +72,8 @@ type Auth struct { S3ForcePathStyle bool } -type TlsConfig struct { +// TLSConfig - TLS cert and key configuration +type TLSConfig struct { CertPEMBlock []byte KeyPEMBlock []byte } @@ -85,14 +85,13 @@ func (a *Auth) endpoint() string { if a.Endpoint != "" { if strings.HasSuffix(a.Endpoint, "amazonaws.com") { return "https://" + a.Endpoint - } else { - return "http://" + a.Endpoint } + return "http://" + a.Endpoint } return "https://" + standardUSRegionAWS } -func (a *Auth) loadKeys(cert string, key string) (*TlsConfig, error) { +func (a *Auth) loadKeys(cert string, key string) (*TLSConfig, error) { certBlock, err := ioutil.ReadFile(cert) if err != nil { return nil, err @@ -101,13 +100,13 @@ func (a *Auth) loadKeys(cert string, key string) (*TlsConfig, error) { if err != nil { return nil, err } - t := &TlsConfig{} + t := &TLSConfig{} t.CertPEMBlock = certBlock t.KeyPEMBlock = keyBlock return t, nil } -func (a *Auth) getTlsTransport() (*http.Transport, error) { +func (a *Auth) getTLSTransport() (*http.Transport, error) { if a.CertPEM == "" || a.KeyPEM == "" { return &http.Transport{ Dial: (&net.Dialer{ @@ -139,7 +138,7 @@ func (a *Auth) getTlsTransport() (*http.Transport, error) { return transport, nil } -func (a *Auth) SignRequest(req *http.Request) { +func (a *Auth) signRequest(req *http.Request) { if date := req.Header.Get("Date"); date == "" { req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) } @@ -203,7 +202,7 @@ func hasPrefixCaseInsensitive(s, pfx string) bool { } func (a *Auth) writeCanonicalizedAmzHeaders(buf *bytes.Buffer, req *http.Request) { - amzHeaders := make([]string, 0) + var amzHeaders []string vals := make(map[string][]string) for k, vv := range req.Header { if hasPrefixCaseInsensitive(k, "x-amz-") { diff --git a/pkg/s3/auth_test.go b/pkg/s3/auth_test.go index 22057d52..9ed94547 100644 --- a/pkg/s3/auth_test.go +++ b/pkg/s3/auth_test.go @@ -134,15 +134,15 @@ func TestBucketFromHostname(t *testing.T) { } } -func TestSignRequest(t *testing.T) { +func TestsignRequest(t *testing.T) { r := req("GET /foo HTTP/1.1\n\n") auth := &Auth{AccessKey: "key", SecretAccessKey: "secretkey"} - auth.SignRequest(r) + auth.signRequest(r) if r.Header.Get("Date") == "" { t.Error("expected a Date set") } r.Header.Set("Date", "Sat, 02 Apr 2011 04:23:52 GMT") - auth.SignRequest(r) + auth.signRequest(r) if e, g := r.Header.Get("Authorization"), "AWS key:kHpCR/N7Rw3PwRlDd8+5X40CFVc="; e != g { t.Errorf("got header %q; expected %q", g, e) } diff --git a/pkg/s3/client.go b/pkg/s3/client.go index 6913d4b7..b92cbac1 100644 --- a/pkg/s3/client.go +++ b/pkg/s3/client.go @@ -59,8 +59,9 @@ import ( "time" ) +// Total max object list const ( - MAX_OBJECT_LIST = 1000 + MaxKeys = 1000 ) // Client is an Amazon S3 client. @@ -69,6 +70,7 @@ type Client struct { Transport http.RoundTripper // or nil for the default } +// Bucket - carries s3 bucket reply header type Bucket struct { Name string CreationDate string // 2006-02-03T16:45:09.000Z @@ -83,26 +85,26 @@ func (c *Client) transport() http.RoundTripper { // bucketURL returns the URL prefix of the bucket, with trailing slash func (c *Client) bucketURL(bucket string) string { - var url_ string + var url string if IsValidBucket(bucket) && !strings.Contains(bucket, ".") { // if localhost forcePathStyle if strings.Contains(c.endpoint(), "localhost") || strings.Contains(bucket, "127.0.0.1") { - url_ = fmt.Sprintf("%s/%s", c.endpoint(), bucket) + url = fmt.Sprintf("%s/%s", c.endpoint(), bucket) goto ret } if !c.S3ForcePathStyle { if strings.Contains(c.endpoint(), "amazonaws.com") { - url_ = fmt.Sprintf("https://%s.%s/", bucket, strings.TrimPrefix(c.endpoint(), "https://")) + url = fmt.Sprintf("https://%s.%s/", bucket, strings.TrimPrefix(c.endpoint(), "https://")) } else { - url_ = fmt.Sprintf("http://%s.%s/", bucket, strings.TrimPrefix(c.endpoint(), "http://")) + url = fmt.Sprintf("http://%s.%s/", bucket, strings.TrimPrefix(c.endpoint(), "http://")) } } else { - url_ = fmt.Sprintf("%s/%s", c.endpoint(), bucket) + url = fmt.Sprintf("%s/%s", c.endpoint(), bucket) } } ret: - return url_ + return url } func (c *Client) keyURL(bucket, key string) string { @@ -113,8 +115,8 @@ func (c *Client) keyURL(bucket, key string) string { return c.bucketURL(bucket) + key } -func newReq(url_ string) *http.Request { - req, err := http.NewRequest("GET", url_, nil) +func newReq(url string) *http.Request { + req, err := http.NewRequest("GET", url, nil) if err != nil { panic(fmt.Sprintf("s3 client; invalid URL: %v", err)) } @@ -122,20 +124,6 @@ func newReq(url_ string) *http.Request { return req } -func (c *Client) Buckets() ([]*Bucket, error) { - req := newReq(c.endpoint() + "/") - c.Auth.SignRequest(req) - res, err := c.transport().RoundTrip(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("s3: Unexpected status code %d fetching bucket list", res.StatusCode) - } - return parseListAllMyBuckets(res.Body) -} - func parseListAllMyBuckets(r io.Reader) ([]*Bucket, error) { type allMyBuckets struct { Buckets struct { @@ -149,11 +137,28 @@ func parseListAllMyBuckets(r io.Reader) ([]*Bucket, error) { return res.Buckets.Bucket, nil } -// Returns 0, "", os.ErrNotExist if not on S3, otherwise reterr is real. +/// Object API operations + +// Buckets - Get list of buckets +func (c *Client) Buckets() ([]*Bucket, error) { + req := newReq(c.endpoint() + "/") + c.Auth.signRequest(req) + res, err := c.transport().RoundTrip(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("s3: Unexpected status code %d fetching bucket list", res.StatusCode) + } + return parseListAllMyBuckets(res.Body) +} + +// Stat - returns 0, "", os.ErrNotExist if not on S3 func (c *Client) Stat(key, bucket string) (size int64, date string, reterr error) { req := newReq(c.keyURL(bucket, key)) req.Method = "HEAD" - c.Auth.SignRequest(req) + c.Auth.signRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return 0, "", err @@ -175,14 +180,15 @@ func (c *Client) Stat(key, bucket string) (size int64, date string, reterr error return 0, "", fmt.Errorf("s3: Unexpected status code %d statting object %v", res.StatusCode, key) } +// PutBucket - create new bucket func (c *Client) PutBucket(bucket string) error { - var url_ string + var url string if IsValidBucket(bucket) && !strings.Contains(bucket, ".") { - url_ = fmt.Sprintf("%s/%s", c.endpoint(), bucket) + url = fmt.Sprintf("%s/%s", c.endpoint(), bucket) } - req := newReq(url_) + req := newReq(url) req.Method = "PUT" - c.Auth.SignRequest(req) + c.Auth.signRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return err @@ -193,13 +199,13 @@ func (c *Client) PutBucket(bucket string) error { } if res.StatusCode != http.StatusOK { - // res.Write(os.Stderr) return fmt.Errorf("Got response code %d from s3", res.StatusCode) } return nil } +// Put - upload new object to bucket func (c *Client) Put(bucket, key string, md5 hash.Hash, size int64, body io.Reader) error { req := newReq(c.keyURL(bucket, key)) req.Method = "PUT" @@ -211,7 +217,7 @@ func (c *Client) Put(bucket, key string, md5 hash.Hash, size int64, body io.Read encoder.Close() req.Header.Set("Content-MD5", b64.String()) } - c.Auth.SignRequest(req) + c.Auth.signRequest(req) req.Body = ioutil.NopCloser(body) res, err := c.transport().RoundTrip(req) @@ -229,12 +235,14 @@ func (c *Client) Put(bucket, key string, md5 hash.Hash, size int64, body io.Read return nil } +// Item - object item list type Item struct { Key string LastModified string Size int64 } +// Prefix - common prefix type Prefix struct { Prefix string } @@ -258,14 +266,14 @@ type listBucketResults struct { CommonPrefixes []*Prefix } -// BucketLocation returns the S3 endpoint to be used with the given bucket. +// BucketLocation - returns the S3 endpoint to be used with the given bucket. func (c *Client) BucketLocation(bucket string) (location string, err error) { if !strings.HasSuffix(c.endpoint(), "amazonaws.com") { return "", errors.New("BucketLocation not implemented for non-Amazon S3 endpoints") } - url_ := fmt.Sprintf("%s/%s/?location", c.endpoint(), url.QueryEscape(bucket)) - req := newReq(url_) - c.Auth.SignRequest(req) + urlReq := fmt.Sprintf("%s/%s/?location", c.endpoint(), url.QueryEscape(bucket)) + req := newReq(urlReq) + c.Auth.signRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return @@ -285,7 +293,7 @@ func (c *Client) BucketLocation(bucket string) (location string, err error) { // 'marker' value). If the length of the returned items is equal to // maxKeys, there is no indication whether or not the returned list is truncated. func (c *Client) GetBucket(bucket string, startAt, prefix, delimiter string, maxKeys int) (items []*Item, prefixes []*Prefix, err error) { - var url_ string + var urlReq string var buffer bytes.Buffer if maxKeys <= 0 { @@ -295,8 +303,8 @@ func (c *Client) GetBucket(bucket string, startAt, prefix, delimiter string, max marker := startAt for len(items) < maxKeys { fetchN := maxKeys - len(items) - if fetchN > MAX_OBJECT_LIST { - fetchN = MAX_OBJECT_LIST + if fetchN > MaxKeys { + fetchN = MaxKeys } var bres listBucketResults buffer.WriteString(fmt.Sprintf("%s?max-keys=%d", c.bucketURL(bucket), fetchN)) @@ -311,15 +319,15 @@ func (c *Client) GetBucket(bucket string, startAt, prefix, delimiter string, max buffer.WriteString(fmt.Sprintf("&delimiter=%s", url.QueryEscape(delimiter))) } - url_ = buffer.String() + urlReq = buffer.String() // Try the enumerate three times, since Amazon likes to close // https connections a lot, and Go sucks at dealing with it: // https://code.google.com/p/go/issues/detail?id=3514 const maxTries = 5 for try := 1; try <= maxTries; try++ { time.Sleep(time.Duration(try-1) * 100 * time.Millisecond) - req := newReq(url_) - c.Auth.SignRequest(req) + req := newReq(urlReq) + c.Auth.signRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { if try < maxTries { @@ -380,7 +388,6 @@ func (c *Client) GetBucket(bucket string, startAt, prefix, delimiter string, max } if !bres.IsTruncated { - // log.Printf("Not truncated. so breaking. items = %d; len Contents = %d, url = %s", len(items), len(bres.Contents), url_) break } if len(items) == 0 { @@ -390,9 +397,10 @@ func (c *Client) GetBucket(bucket string, startAt, prefix, delimiter string, max return items, prefixes, nil } +// Get - download a requested object from a given bucket func (c *Client) Get(bucket, key string) (body io.ReadCloser, size int64, err error) { req := newReq(c.keyURL(bucket, key)) - c.Auth.SignRequest(req) + c.Auth.signRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return @@ -423,7 +431,7 @@ func (c *Client) GetPartial(bucket, key string, offset, length int64) (rc io.Rea } else { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } - c.Auth.SignRequest(req) + c.Auth.signRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { @@ -445,7 +453,7 @@ func (c *Client) GetPartial(bucket, key string, offset, length int64) (rc io.Rea func (c *Client) Delete(bucket, key string) error { req := newReq(c.keyURL(bucket, key)) req.Method = "DELETE" - c.Auth.SignRequest(req) + c.Auth.signRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { return err @@ -461,23 +469,13 @@ func (c *Client) Delete(bucket, key string) error { } */ -func NewAuth(accessKey, secretKey, endpoint string, style bool) (auth *Auth) { - auth = &Auth{ - AccessKey: accessKey, - SecretAccessKey: secretKey, - Endpoint: endpoint, - S3ForcePathStyle: style, - } - return -} - -func NewS3Client(auth *Auth) (client *Client) { +// NewClient - get new client +func NewClient(auth *Auth) (client *Client) { client = &Client{auth, http.DefaultTransport} return } -// IsValid reports whether bucket is a valid bucket name, per Amazon's naming restrictions. -// +// IsValidBucket reports whether bucket is a valid bucket name, per Amazon's naming restrictions. // See http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html func IsValidBucket(bucket string) bool { if len(bucket) < 3 || len(bucket) > 63 { @@ -533,7 +531,7 @@ type xmlError struct { XMLName xml.Name `xml:"Error"` Code string Message string - RequestId string + RequestID string Bucket string Endpoint string StringToSignBytes string diff --git a/s3-errors.go b/s3-errors.go index 2e8a4f90..5ecc61ea 100644 --- a/s3-errors.go +++ b/s3-errors.go @@ -19,17 +19,16 @@ package main import "errors" // fs -var fsPathErr = errors.New("Arguments missing or ") -var fsUriErr = errors.New("Invalid URI scheme") -var fsKeyErr = errors.New("Key is needed to get the file") +var errFspath = errors.New("Arguments missing or ") +var errFsuri = errors.New("Invalid uri scheme") +var errFskey = errors.New("Key is needed to get the file") // configure -var configAccessErr = errors.New("accesskey is mandatory") -var configSecretErr = errors.New("secretkey is mandatory") -var configEndpointErr = errors.New("endpoint is missing") +var errAccess = errors.New("accesskey is mandatory") +var errSecret = errors.New("secretkey is mandatory") +var errEndpoint = errors.New("endpoint is mandatory") // common -var missingAccessSecretErr = errors.New("You can configure your credentials by running `mc configure`") -var missingAccessErr = errors.New("Partial credentials found in the env, missing : AWS_ACCESS_KEY_ID") -var missingSecretErr = errors.New("Partial credentials found in the env, missing : AWS_SECRET_ACCESS_KEY") -var invalidBucketErr = errors.New("Invalid bucket name") +var errMissingaccess = errors.New("Partial credentials found in the env, missing : AWS_ACCESS_KEY_ID") +var errMissingsecret = errors.New("Partial credentials found in the env, missing : AWS_SECRET_ACCESS_KEY") +var errInvalidbucket = errors.New("Invalid bucket name") diff --git a/s3-fs-cp.go b/s3-fs-cp.go index b4ea19c3..1e3dd25a 100644 --- a/s3-fs-cp.go +++ b/s3-fs-cp.go @@ -17,6 +17,7 @@ package main import ( + "fmt" "io" "log" "os" @@ -37,16 +38,16 @@ import ( func parseCpOptions(c *cli.Context) (fsoptions fsOptions, err error) { switch len(c.Args()) { case 1: - return fsOptions{}, fsPathErr + return fsOptions{}, errFspath case 2: if strings.HasPrefix(c.Args().Get(0), "s3://") { uri := uri.ParseURI(c.Args().Get(0)) if uri.Scheme == "" { - return fsOptions{}, fsUriErr + return fsOptions{}, errFsuri } fsoptions.bucket = uri.Server if uri.Path == "" { - return fsOptions{}, fsKeyErr + return fsOptions{}, errFskey } fsoptions.key = strings.TrimPrefix(uri.Path, "/") if c.Args().Get(1) == "." { @@ -59,7 +60,7 @@ func parseCpOptions(c *cli.Context) (fsoptions fsOptions, err error) { } else if strings.HasPrefix(c.Args().Get(1), "s3://") { uri := uri.ParseURI(c.Args().Get(1)) if uri.Scheme == "" { - return fsOptions{}, fsUriErr + return fsOptions{}, errFsuri } fsoptions.bucket = uri.Server if uri.Path == "" { @@ -72,7 +73,7 @@ func parseCpOptions(c *cli.Context) (fsoptions fsOptions, err error) { fsoptions.isput = true } default: - return fsOptions{}, fsPathErr + return fsOptions{}, errFspath } return } @@ -87,13 +88,18 @@ func startBar(size int64) *pb.ProgressBar { func doFsCopy(c *cli.Context) { var auth *s3.Auth + var s3c *s3.Client var err error var bodyFile *os.File auth, err = getAWSEnvironment() if err != nil { log.Fatal(err) } - s3c := s3.NewS3Client(auth) + s3c, err = getNewClient(auth) + if err != nil { + log.Fatal(err) + } + var fsoptions fsOptions fsoptions, err = parseCpOptions(c) if err != nil { @@ -119,6 +125,7 @@ func doFsCopy(c *cli.Context) { if err != nil { log.Fatal(err) } + fmt.Printf("%s uploaded -- to bucket:%s", fsoptions.key, fsoptions.bucket) } else if fsoptions.isget { var objectReader io.ReadCloser var objectSize int64 diff --git a/s3-fs-ls.go b/s3-fs-ls.go index a4e20b36..c573dbbd 100644 --- a/s3-fs-ls.go +++ b/s3-fs-ls.go @@ -93,6 +93,7 @@ func getBucketAndObject(p string) (bucket, object string) { func doFsList(c *cli.Context) { var err error var auth *s3.Auth + var s3c *s3.Client var items []*s3.Item var prefixes []*s3.Prefix @@ -102,7 +103,10 @@ func doFsList(c *cli.Context) { } var buckets []*s3.Bucket - s3c := s3.NewS3Client(auth) + s3c, err = getNewClient(auth) + if err != nil { + log.Fatal(err) + } switch len(c.Args()) { case 1: input := c.Args().Get(0) @@ -111,7 +115,7 @@ func doFsList(c *cli.Context) { } bucket, object := getBucketAndObject(input) if object == "" { - items, prefixes, err = s3c.GetBucket(bucket, "", "", string(delimiter), s3.MAX_OBJECT_LIST) + items, prefixes, err = s3c.GetBucket(bucket, "", "", string(delimiter), s3.MaxKeys) if err != nil { log.Fatal(err) } @@ -122,7 +126,7 @@ func doFsList(c *cli.Context) { var size int64 size, date, err = s3c.Stat(object, bucket) if err != nil { - items, prefixes, err = s3c.GetBucket(bucket, "", object, string(delimiter), s3.MAX_OBJECT_LIST) + items, prefixes, err = s3c.GetBucket(bucket, "", object, string(delimiter), s3.MaxKeys) if err != nil { log.Fatal(err) } diff --git a/s3-fs-mb.go b/s3-fs-mb.go index 16caa6b2..debcb964 100644 --- a/s3-fs-mb.go +++ b/s3-fs-mb.go @@ -27,7 +27,7 @@ func doFsMb(c *cli.Context) { switch len(c.Args()) { case 1: if !s3.IsValidBucket(c.Args().Get(0)) { - log.Fatalf("%s: (%s)", invalidBucketErr, c.Args().Get(0)) + log.Fatalf("%s: (%s)", errInvalidbucket, c.Args().Get(0)) } default: log.Fatal("Needs an argument ") @@ -35,11 +35,12 @@ func doFsMb(c *cli.Context) { bucketName := c.Args().Get(0) var err error var auth *s3.Auth + var s3c *s3.Client auth, err = getAWSEnvironment() if err != nil { log.Fatal(err) } - s3c := s3.NewS3Client(auth) + s3c, err = getNewClient(auth) err = s3c.PutBucket(bucketName) if err != nil { log.Fatal(err) diff --git a/s3-fs-options.go b/s3-fs-options.go index 4ac2dd34..ebff3278 100644 --- a/s3-fs-options.go +++ b/s3-fs-options.go @@ -20,35 +20,35 @@ import ( "github.com/codegangsta/cli" ) -var Cp = cli.Command{ +var cp = cli.Command{ Name: "cp", Usage: "", Description: `Copies a local file or Object to another location locally or in S3.`, Action: doFsCopy, } -var Ls = cli.Command{ +var ls = cli.Command{ Name: "ls", Usage: "", Description: `List Objects and common prefixes under a prefix or all Buckets`, Action: doFsList, } -var Mb = cli.Command{ +var mb = cli.Command{ Name: "mb", Usage: "", Description: "Creates an S3 bucket", Action: doFsMb, } -var Sync = cli.Command{ +var sync = cli.Command{ Name: "sync", Usage: "", Description: "Syncs directories and S3 prefixes", Action: doFsSync, } -var Configure = cli.Command{ +var configure = cli.Command{ Name: "configure", Usage: "", Description: `Configure minio client configuration data. If your config @@ -88,14 +88,15 @@ type fsOptions struct { isput bool } -const ( - Auth = ".auth" -) - var options = []cli.Command{ - Cp, - Ls, - Mb, - Sync, - Configure, + cp, + ls, + mb, + sync, + configure, } + +// Common authentication file +const ( + Auth = ".minio/auth.json" +) diff --git a/s3-fs-sync.go b/s3-fs-sync.go index 6339363b..af56e7fb 100644 --- a/s3-fs-sync.go +++ b/s3-fs-sync.go @@ -17,6 +17,7 @@ package main import ( + "fmt" "log" "os" "path" @@ -27,18 +28,6 @@ import ( "github.com/minio-io/mc/pkg/s3" ) -var s3c *s3.Client - -func init() { - var auth *s3.Auth - var err error - auth, err = getAWSEnvironment() - if err != nil { - log.Fatal(err) - } - s3c = s3.NewS3Client(auth) -} - func isValidBucketName(p string) { if path.IsAbs(p) { log.Fatal("directory bucketname cannot be absolute") @@ -47,11 +36,15 @@ func isValidBucketName(p string) { log.Fatal("Relative directory references not supported") } if !s3.IsValidBucket(p) { - log.Fatal(invalidBucketErr) + log.Fatal(errInvalidbucket) } } -func putWalk(p string, info os.FileInfo, err error) error { +type walk struct { + s3 *s3.Client +} + +func (w *walk) putWalk(p string, info os.FileInfo, err error) error { if info.IsDir() { return nil } @@ -64,15 +57,28 @@ func putWalk(p string, info os.FileInfo, err error) error { bodyFile, err := os.Open(p) defer bodyFile.Close() - err = s3c.Put(bucketname, key, nil, info.Size(), bodyFile) + err = w.s3.Put(bucketname, key, nil, info.Size(), bodyFile) if err != nil { return err } - log.Printf("%s uploaded -- to bucket:%s", key, bucketname) + fmt.Printf("%s uploaded -- to bucket:%s\n", key, bucketname) return nil } func doFsSync(c *cli.Context) { + var auth *s3.Auth + var s3c *s3.Client + var err error + auth, err = getAWSEnvironment() + if err != nil { + log.Fatal(err) + } + s3c, err = getNewClient(auth) + if err != nil { + log.Fatal(err) + } + p := &walk{s3c} + switch len(c.Args()) { case 1: input := path.Clean(c.Args().Get(0)) @@ -90,7 +96,7 @@ func doFsSync(c *cli.Context) { if err != nil { log.Fatal(err) } - err = filepath.Walk(input, putWalk) + err = filepath.Walk(input, p.putWalk) if err != nil { log.Fatal(err) }