mc/pkg/s3/bucket.go

// Original license //
// ---------------- //

/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// All other modifications and improvements //
// ---------------------------------------- //

/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"sort"
	"strings"
	"time"

	"encoding/xml"
	"net/http"
	"net/url"

	"github.com/minio-io/mc/pkg/client"
)

// bySize implements sort.Interface for []Item based on the Size field.
type bySize []*client.Item

func (a bySize) Len() int           { return len(a) }
func (a bySize) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a bySize) Less(i, j int) bool { return a[i].Size < a[j].Size }

/// Bucket API operations

// ListBuckets - Get list of buckets
func (c *s3Client) ListBuckets() ([]*client.Bucket, error) {
	url := fmt.Sprintf("%s://%s/", c.URL.Scheme, c.URL.Host)
	req := newReq(url)
	c.signRequest(req, c.URL.Host)

	res, err := c.Transport.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, NewError(res)
	}

	return parseListAllMyBuckets(res.Body)
}

// PutBucket - create new bucket
func (c *s3Client) PutBucket(bucket string) error {
	var url string
	if IsValidBucket(bucket) && !strings.Contains(bucket, ".") {
		url = fmt.Sprintf("%s://%s/%s", c.URL.Scheme, c.URL.Host, bucket)
	}
	req := newReq(url)
	req.Method = "PUT"
	c.signRequest(req, c.URL.Host)
	res, err := c.Transport.RoundTrip(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return NewError(res)
	}

	return nil
}

// ListObjects returns 0 to maxKeys (inclusive) items from the
// provided bucket. Keys before startAt will be skipped. (This is the S3
// 'marker' value). If the length of the returned items is equal to
// maxKeys, there is no indication whether or not the returned list is truncated.
func (c *s3Client) ListObjects(bucket string, startAt, prefix, delimiter string, maxKeys int) (items []*client.Item, prefixes []*client.Prefix, err error) {
	var urlReq string
	var buffer bytes.Buffer

	if maxKeys <= 0 {
		return nil, nil, errors.New("negative maxKeys are invalid")
	}

	marker := startAt
	for len(items) < maxKeys {
		fetchN := maxKeys - len(items)
		if fetchN > MaxKeys {
			fetchN = MaxKeys
		}
		var bres listBucketResults
		buffer.WriteString(fmt.Sprintf("%s?max-keys=%d", c.bucketURL(bucket), fetchN))
		switch true {
		case marker != "":
			buffer.WriteString(fmt.Sprintf("&marker=%s", url.QueryEscape(marker)))
			fallthrough
		case prefix != "":
			buffer.WriteString(fmt.Sprintf("&prefix=%s", url.QueryEscape(prefix)))
			fallthrough
		case delimiter != "":
			buffer.WriteString(fmt.Sprintf("&delimiter=%s", url.QueryEscape(delimiter)))
		}

		urlReq = buffer.String()
		// Try the enumerate three times, since Amazon likes to close
		// https connections a lot, and Go sucks at dealing with it:
		// https://code.google.com/p/go/issues/detail?id=3514
		const maxTries = 5
		for try := 1; try <= maxTries; try++ {
			time.Sleep(time.Duration(try-1) * 100 * time.Millisecond)
			req := newReq(urlReq)
			c.signRequest(req, c.URL.Host)
			res, err := c.Transport.RoundTrip(req)
			if err != nil {
				if try < maxTries {
					continue
				}
				return nil, nil, err
			}
			defer res.Body.Close()

			if res.StatusCode != http.StatusOK {
				return nil, nil, NewError(res)
			}

			bres = listBucketResults{}
			var logbuf bytes.Buffer
			err = xml.NewDecoder(io.TeeReader(res.Body, &logbuf)).Decode(&bres)
			if err != nil {
				fmt.Printf("Error parsing s3 XML response: %v for %q", err, logbuf.Bytes())
			} else if bres.MaxKeys != fetchN || bres.Name != bucket || bres.Marker != marker {
				msg := fmt.Sprintf("Unexpected parse from server: %#v from: %s", bres, logbuf.Bytes())
				err = errors.New(msg)
				fmt.Print(err)
			}
			if err != nil {
				if try < maxTries-1 {
					continue
				}
				fmt.Print(err)
				return nil, nil, err
			}
			break
		}
		for _, it := range bres.Contents {
			if it.Key == marker && it.Key != startAt {
				// Skip first dup on pages 2 and higher.
				continue
			}
			if it.Key < startAt {
				msg := fmt.Sprintf("Unexpected response from Amazon: item key %q but wanted greater than %q", it.Key, startAt)
				return nil, nil, errors.New(msg)
			}
			items = append(items, it)
			marker = it.Key
		}

		for _, pre := range bres.CommonPrefixes {
			if pre.Prefix != "" {
				prefixes = append(prefixes, pre)
			}
		}

		if !bres.IsTruncated {
			break
		}

		if len(items) == 0 {
			return nil, nil, errors.New("No items replied")
		}
	}
	sort.Sort(bySize(items))
	return items, prefixes, nil
}