diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ea0efa94cf..61ecaa3624 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -158,6 +158,14 @@ { "ImportPath": "google.golang.org/api/googleapi", "Rev": "a09229c13c2f13bbdedf7b31b506cad4c83ef3bf" + }, + { + "ImportPath": "golang.org/x/oauth2", + "Rev": "038cb4adce85ed41e285c2e7cc6221a92bfa44aa" + }, + { + "ImportPath": "google.golang.org/cloud/compute", + "Rev": "2400193c85c3561d13880d34e0e10c4315bb02af" } ] } diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/.travis.yml b/Godeps/_workspace/src/golang.org/x/oauth2/.travis.yml new file mode 100644 index 0000000000..a035125c35 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/.travis.yml @@ -0,0 +1,14 @@ +language: go + +go: + - 1.3 + - 1.4 + +install: + - export GOPATH="$HOME/gopath" + - mkdir -p "$GOPATH/src/golang.org/x" + - mv "$TRAVIS_BUILD_DIR" "$GOPATH/src/golang.org/x/oauth2" + - go get -v -t -d golang.org/x/oauth2/... + +script: + - go test -v golang.org/x/oauth2/... diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/AUTHORS b/Godeps/_workspace/src/golang.org/x/oauth2/AUTHORS new file mode 100644 index 0000000000..15167cd746 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/CONTRIBUTING.md b/Godeps/_workspace/src/golang.org/x/oauth2/CONTRIBUTING.md new file mode 100644 index 0000000000..46aa2b12dd --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing to Go + +Go is an open source project. + +It is the work of hundreds of contributors. We appreciate your help! + + +## Filing issues + +When [filing an issue](https://github.com/golang/oauth2/issues), make sure to answer these five questions: + +1. What version of Go are you using (`go version`)? +2. What operating system and processor architecture are you using? +3. What did you do? +4. What did you expect to see? +5. What did you see instead? + +General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker. +The gophers there will answer or ask you to file an issue if you've tripped over a bug. + +## Contributing code + +Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html) +before sending patches. + +**We do not accept GitHub pull requests** +(we use [Gerrit](https://code.google.com/p/gerrit/) instead for code review). + +Unless otherwise noted, the Go source files are distributed under +the BSD-style license found in the LICENSE file. + diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/CONTRIBUTORS b/Godeps/_workspace/src/golang.org/x/oauth2/CONTRIBUTORS new file mode 100644 index 0000000000..1c4577e968 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/LICENSE b/Godeps/_workspace/src/golang.org/x/oauth2/LICENSE new file mode 100644 index 0000000000..d02f24fd52 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The oauth2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/README.md b/Godeps/_workspace/src/golang.org/x/oauth2/README.md new file mode 100644 index 0000000000..0d5141733f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/README.md @@ -0,0 +1,64 @@ +# OAuth2 for Go + +[![Build Status](https://travis-ci.org/golang/oauth2.svg?branch=master)](https://travis-ci.org/golang/oauth2) + +oauth2 package contains a client implementation for OAuth 2.0 spec. + +## Installation + +~~~~ +go get golang.org/x/oauth2 +~~~~ + +See godoc for further documentation and examples. + +* [godoc.org/golang.org/x/oauth2](http://godoc.org/golang.org/x/oauth2) +* [godoc.org/golang.org/x/oauth2/google](http://godoc.org/golang.org/x/oauth2/google) + + +## App Engine + +In change 96e89be (March 2015) we removed the `oauth2.Context2` type in favor +of the [`context.Context`](https://golang.org/x/net/context#Context) type from +the `golang.org/x/net/context` package + +This means its no longer possible to use the "Classic App Engine" +`appengine.Context` type with the `oauth2` package. (You're using +Classic App Engine if you import the package `"appengine"`.) + +To work around this, you may use the new `"google.golang.org/appengine"` +package. This package has almost the same API as the `"appengine"` package, +but it can be fetched with `go get` and used on "Managed VMs" and well as +Classic App Engine. + +See the [new `appengine` package's readme](https://github.com/golang/appengine#updating-a-go-app-engine-app) +for information on updating your app. + +If you don't want to update your entire app to use the new App Engine packages, +you may use both sets of packages in parallel, using only the new packages +with the `oauth2` package. + + import ( + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + newappengine "google.golang.org/appengine" + newurlfetch "google.golang.org/appengine/urlfetch" + + "appengine" + ) + + func handler(w http.ResponseWriter, r *http.Request) { + var c appengine.Context = appengine.NewContext(r) + c.Infof("Logging a message with the old package") + + var ctx context.Context = newappengine.NewContext(r) + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: google.AppEngineTokenSource(ctx, "scope"), + Base: &newurlfetch.Transport{Context: ctx}, + }, + } + client.Get("...") + } + diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/bitbucket/bitbucket.go b/Godeps/_workspace/src/golang.org/x/oauth2/bitbucket/bitbucket.go new file mode 100644 index 0000000000..44af1f1a9c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/bitbucket/bitbucket.go @@ -0,0 +1,16 @@ +// Copyright 2015 The oauth2 Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bitbucket provides constants for using OAuth2 to access Bitbucket. +package bitbucket + +import ( + "golang.org/x/oauth2" +) + +// Endpoint is Bitbucket's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://bitbucket.org/site/oauth2/authorize", + TokenURL: "https://bitbucket.org/site/oauth2/access_token", +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/client_appengine.go b/Godeps/_workspace/src/golang.org/x/oauth2/client_appengine.go new file mode 100644 index 0000000000..8962c49d1d --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/client_appengine.go @@ -0,0 +1,25 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +// App Engine hooks. + +package oauth2 + +import ( + "net/http" + + "golang.org/x/net/context" + "golang.org/x/oauth2/internal" + "google.golang.org/appengine/urlfetch" +) + +func init() { + internal.RegisterContextClientFunc(contextClientAppEngine) +} + +func contextClientAppEngine(ctx context.Context) (*http.Client, error) { + return urlfetch.Client(ctx), nil +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/clientcredentials/clientcredentials.go b/Godeps/_workspace/src/golang.org/x/oauth2/clientcredentials/clientcredentials.go new file mode 100644 index 0000000000..38be112604 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/clientcredentials/clientcredentials.go @@ -0,0 +1,112 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package clientcredentials implements the OAuth2.0 "client credentials" token flow, +// also known as the "two-legged OAuth 2.0". +// +// This should be used when the client is acting on its own behalf or when the client +// is the resource owner. It may also be used when requesting access to protected +// resources based on an authorization previously arranged with the authorization +// server. +// +// See http://tools.ietf.org/html/draft-ietf-oauth-v2-31#section-4.4 +package clientcredentials // import "golang.org/x/oauth2/clientcredentials" + +import ( + "net/http" + "net/url" + "strings" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/internal" +) + +// tokenFromInternal maps an *internal.Token struct into +// an *oauth2.Token struct. +func tokenFromInternal(t *internal.Token) *oauth2.Token { + if t == nil { + return nil + } + tk := &oauth2.Token{ + AccessToken: t.AccessToken, + TokenType: t.TokenType, + RefreshToken: t.RefreshToken, + Expiry: t.Expiry, + } + return tk.WithExtra(t.Raw) +} + +// retrieveToken takes a *Config and uses that to retrieve an *internal.Token. +// This token is then mapped from *internal.Token into an *oauth2.Token which is +// returned along with an error. +func retrieveToken(ctx context.Context, c *Config, v url.Values) (*oauth2.Token, error) { + tk, err := internal.RetrieveToken(ctx, c.ClientID, c.ClientSecret, c.TokenURL, v) + if err != nil { + return nil, err + } + return tokenFromInternal(tk), nil +} + +// Client Credentials Config describes a 2-legged OAuth2 flow, with both the +// client application information and the server's endpoint URLs. +type Config struct { + // ClientID is the application's ID. + ClientID string + + // ClientSecret is the application's secret. + ClientSecret string + + // TokenURL is the resource server's token endpoint + // URL. This is a constant specific to each server. + TokenURL string + + // Scope specifies optional requested permissions. + Scopes []string +} + +// Token uses client credentials to retrieve a token. +// The HTTP client to use is derived from the context. +// If nil, http.DefaultClient is used. +func (c *Config) Token(ctx context.Context) (*oauth2.Token, error) { + return retrieveToken(ctx, c, url.Values{ + "grant_type": {"client_credentials"}, + "scope": internal.CondVal(strings.Join(c.Scopes, " ")), + }) +} + +// Client returns an HTTP client using the provided token. +// The token will auto-refresh as necessary. The underlying +// HTTP transport will be obtained using the provided context. +// The returned client and its Transport should not be modified. +func (c *Config) Client(ctx context.Context) *http.Client { + return oauth2.NewClient(ctx, c.TokenSource(ctx)) +} + +// TokenSource returns a TokenSource that returns t until t expires, +// automatically refreshing it as necessary using the provided context and the +// client ID and client secret. +// +// Most users will use Config.Client instead. +func (c *Config) TokenSource(ctx context.Context) oauth2.TokenSource { + source := &tokenSource{ + ctx: ctx, + conf: c, + } + return oauth2.ReuseTokenSource(nil, source) +} + +type tokenSource struct { + ctx context.Context + conf *Config +} + +// Token refreshes the token by using a new client credentials request. +// tokens received this way do not include a refresh token +func (c *tokenSource) Token() (*oauth2.Token, error) { + return retrieveToken(c.ctx, c.conf, url.Values{ + "grant_type": {"client_credentials"}, + "scope": internal.CondVal(strings.Join(c.conf.Scopes, " ")), + }) +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/clientcredentials/clientcredentials_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/clientcredentials/clientcredentials_test.go new file mode 100644 index 0000000000..5a0170a95f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/clientcredentials/clientcredentials_test.go @@ -0,0 +1,96 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package clientcredentials + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "golang.org/x/oauth2" +) + +func newConf(url string) *Config { + return &Config{ + ClientID: "CLIENT_ID", + ClientSecret: "CLIENT_SECRET", + Scopes: []string{"scope1", "scope2"}, + TokenURL: url + "/token", + } +} + +type mockTransport struct { + rt func(req *http.Request) (resp *http.Response, err error) +} + +func (t *mockTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) { + return t.rt(req) +} + +func TestTokenRequest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() != "/token" { + t.Errorf("authenticate client request URL = %q; want %q", r.URL, "/token") + } + headerAuth := r.Header.Get("Authorization") + if headerAuth != "Basic Q0xJRU5UX0lEOkNMSUVOVF9TRUNSRVQ=" { + t.Errorf("Unexpected authorization header, %v is found.", headerAuth) + } + if got, want := r.Header.Get("Content-Type"), "application/x-www-form-urlencoded"; got != want { + t.Errorf("Content-Type header = %q; want %q", got, want) + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + r.Body.Close() + } + if err != nil { + t.Errorf("failed reading request body: %s.", err) + } + if string(body) != "client_id=CLIENT_ID&grant_type=client_credentials&scope=scope1+scope2" { + t.Errorf("payload = %q; want %q", string(body), "client_id=CLIENT_ID&grant_type=client_credentials&scope=scope1+scope2") + } + w.Header().Set("Content-Type", "application/x-www-form-urlencoded") + w.Write([]byte("access_token=90d64460d14870c08c81352a05dedd3465940a7c&token_type=bearer")) + })) + defer ts.Close() + conf := newConf(ts.URL) + tok, err := conf.Token(oauth2.NoContext) + if err != nil { + t.Error(err) + } + if !tok.Valid() { + t.Fatalf("token invalid. got: %#v", tok) + } + if tok.AccessToken != "90d64460d14870c08c81352a05dedd3465940a7c" { + t.Errorf("Access token = %q; want %q", tok.AccessToken, "90d64460d14870c08c81352a05dedd3465940a7c") + } + if tok.TokenType != "bearer" { + t.Errorf("token type = %q; want %q", tok.TokenType, "bearer") + } +} + +func TestTokenRefreshRequest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() == "/somethingelse" { + return + } + if r.URL.String() != "/token" { + t.Errorf("Unexpected token refresh request URL, %v is found.", r.URL) + } + headerContentType := r.Header.Get("Content-Type") + if headerContentType != "application/x-www-form-urlencoded" { + t.Errorf("Unexpected Content-Type header, %v is found.", headerContentType) + } + body, _ := ioutil.ReadAll(r.Body) + if string(body) != "client_id=CLIENT_ID&grant_type=client_credentials&scope=scope1+scope2" { + t.Errorf("Unexpected refresh token payload, %v is found.", string(body)) + } + })) + defer ts.Close() + conf := newConf(ts.URL) + c := conf.Client(oauth2.NoContext) + c.Get(ts.URL + "/somethingelse") +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/example_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/example_test.go new file mode 100644 index 0000000000..33b305c628 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/example_test.go @@ -0,0 +1,45 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package oauth2_test + +import ( + "fmt" + "log" + + "golang.org/x/oauth2" +) + +func ExampleConfig() { + conf := &oauth2.Config{ + ClientID: "YOUR_CLIENT_ID", + ClientSecret: "YOUR_CLIENT_SECRET", + Scopes: []string{"SCOPE1", "SCOPE2"}, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://provider.com/o/oauth2/auth", + TokenURL: "https://provider.com/o/oauth2/token", + }, + } + + // Redirect user to consent page to ask for permission + // for the scopes specified above. + url := conf.AuthCodeURL("state", oauth2.AccessTypeOffline) + fmt.Printf("Visit the URL for the auth dialog: %v", url) + + // Use the authorization code that is pushed to the redirect URL. + // NewTransportWithCode will do the handshake to retrieve + // an access token and initiate a Transport that is + // authorized and authenticated by the retrieved token. + var code string + if _, err := fmt.Scan(&code); err != nil { + log.Fatal(err) + } + tok, err := conf.Exchange(oauth2.NoContext, code) + if err != nil { + log.Fatal(err) + } + + client := conf.Client(oauth2.NoContext, tok) + client.Get("...") +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/facebook/facebook.go b/Godeps/_workspace/src/golang.org/x/oauth2/facebook/facebook.go new file mode 100644 index 0000000000..14c801a2a1 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/facebook/facebook.go @@ -0,0 +1,16 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package facebook provides constants for using OAuth2 to access Facebook. +package facebook // import "golang.org/x/oauth2/facebook" + +import ( + "golang.org/x/oauth2" +) + +// Endpoint is Facebook's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://www.facebook.com/dialog/oauth", + TokenURL: "https://graph.facebook.com/oauth/access_token", +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/github/github.go b/Godeps/_workspace/src/golang.org/x/oauth2/github/github.go new file mode 100644 index 0000000000..f2978015b0 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/github/github.go @@ -0,0 +1,16 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package github provides constants for using OAuth2 to access Github. +package github // import "golang.org/x/oauth2/github" + +import ( + "golang.org/x/oauth2" +) + +// Endpoint is Github's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://github.com/login/oauth/authorize", + TokenURL: "https://github.com/login/oauth/access_token", +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/appengine.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/appengine.go new file mode 100644 index 0000000000..dc993efb5e --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/appengine.go @@ -0,0 +1,86 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "sort" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2" +) + +// Set at init time by appenginevm_hook.go. If true, we are on App Engine Managed VMs. +var appengineVM bool + +// Set at init time by appengine_hook.go. If nil, we're not on App Engine. +var appengineTokenFunc func(c context.Context, scopes ...string) (token string, expiry time.Time, err error) + +// AppEngineTokenSource returns a token source that fetches tokens +// issued to the current App Engine application's service account. +// If you are implementing a 3-legged OAuth 2.0 flow on App Engine +// that involves user accounts, see oauth2.Config instead. +// +// The provided context must have come from appengine.NewContext. +func AppEngineTokenSource(ctx context.Context, scope ...string) oauth2.TokenSource { + if appengineTokenFunc == nil { + panic("google: AppEngineTokenSource can only be used on App Engine.") + } + scopes := append([]string{}, scope...) + sort.Strings(scopes) + return &appEngineTokenSource{ + ctx: ctx, + scopes: scopes, + key: strings.Join(scopes, " "), + } +} + +// aeTokens helps the fetched tokens to be reused until their expiration. +var ( + aeTokensMu sync.Mutex + aeTokens = make(map[string]*tokenLock) // key is space-separated scopes +) + +type tokenLock struct { + mu sync.Mutex // guards t; held while fetching or updating t + t *oauth2.Token +} + +type appEngineTokenSource struct { + ctx context.Context + scopes []string + key string // to aeTokens map; space-separated scopes +} + +func (ts *appEngineTokenSource) Token() (*oauth2.Token, error) { + if appengineTokenFunc == nil { + panic("google: AppEngineTokenSource can only be used on App Engine.") + } + + aeTokensMu.Lock() + tok, ok := aeTokens[ts.key] + if !ok { + tok = &tokenLock{} + aeTokens[ts.key] = tok + } + aeTokensMu.Unlock() + + tok.mu.Lock() + defer tok.mu.Unlock() + if tok.t.Valid() { + return tok.t, nil + } + access, exp, err := appengineTokenFunc(ts.ctx, ts.scopes...) + if err != nil { + return nil, err + } + tok.t = &oauth2.Token{ + AccessToken: access, + Expiry: exp, + } + return tok.t, nil +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/appengine_hook.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/appengine_hook.go new file mode 100644 index 0000000000..4f42c8b343 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/appengine_hook.go @@ -0,0 +1,13 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package google + +import "google.golang.org/appengine" + +func init() { + appengineTokenFunc = appengine.AccessToken +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/appenginevm_hook.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/appenginevm_hook.go new file mode 100644 index 0000000000..633611cc3a --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/appenginevm_hook.go @@ -0,0 +1,14 @@ +// Copyright 2015 The oauth2 Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appenginevm + +package google + +import "google.golang.org/appengine" + +func init() { + appengineVM = true + appengineTokenFunc = appengine.AccessToken +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/default.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/default.go new file mode 100644 index 0000000000..b952362977 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/default.go @@ -0,0 +1,155 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/jwt" + "google.golang.org/cloud/compute/metadata" +) + +// DefaultClient returns an HTTP Client that uses the +// DefaultTokenSource to obtain authentication credentials. +// +// This client should be used when developing services +// that run on Google App Engine or Google Compute Engine +// and use "Application Default Credentials." +// +// For more details, see: +// https://developers.google.com/accounts/docs/application-default-credentials +// +func DefaultClient(ctx context.Context, scope ...string) (*http.Client, error) { + ts, err := DefaultTokenSource(ctx, scope...) + if err != nil { + return nil, err + } + return oauth2.NewClient(ctx, ts), nil +} + +// DefaultTokenSource is a token source that uses +// "Application Default Credentials". +// +// It looks for credentials in the following places, +// preferring the first location found: +// +// 1. A JSON file whose path is specified by the +// GOOGLE_APPLICATION_CREDENTIALS environment variable. +// 2. A JSON file in a location known to the gcloud command-line tool. +// On Windows, this is %APPDATA%/gcloud/application_default_credentials.json. +// On other systems, $HOME/.config/gcloud/application_default_credentials.json. +// 3. On Google App Engine it uses the appengine.AccessToken function. +// 4. On Google Compute Engine and Google App Engine Managed VMs, it fetches +// credentials from the metadata server. +// (In this final case any provided scopes are ignored.) +// +// For more details, see: +// https://developers.google.com/accounts/docs/application-default-credentials +// +func DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) { + // First, try the environment variable. + const envVar = "GOOGLE_APPLICATION_CREDENTIALS" + if filename := os.Getenv(envVar); filename != "" { + ts, err := tokenSourceFromFile(ctx, filename, scope) + if err != nil { + return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err) + } + return ts, nil + } + + // Second, try a well-known file. + filename := wellKnownFile() + _, err := os.Stat(filename) + if err == nil { + ts, err2 := tokenSourceFromFile(ctx, filename, scope) + if err2 == nil { + return ts, nil + } + err = err2 + } else if os.IsNotExist(err) { + err = nil // ignore this error + } + if err != nil { + return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err) + } + + // Third, if we're on Google App Engine use those credentials. + if appengineTokenFunc != nil && !appengineVM { + return AppEngineTokenSource(ctx, scope...), nil + } + + // Fourth, if we're on Google Compute Engine use the metadata server. + if metadata.OnGCE() { + return ComputeTokenSource(""), nil + } + + // None are found; return helpful error. + const url = "https://developers.google.com/accounts/docs/application-default-credentials" + return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url) +} + +func wellKnownFile() string { + const f = "application_default_credentials.json" + if runtime.GOOS == "windows" { + return filepath.Join(os.Getenv("APPDATA"), "gcloud", f) + } + return filepath.Join(guessUnixHomeDir(), ".config", "gcloud", f) +} + +func tokenSourceFromFile(ctx context.Context, filename string, scopes []string) (oauth2.TokenSource, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + var d struct { + // Common fields + Type string + ClientID string `json:"client_id"` + + // User Credential fields + ClientSecret string `json:"client_secret"` + RefreshToken string `json:"refresh_token"` + + // Service Account fields + ClientEmail string `json:"client_email"` + PrivateKeyID string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + } + if err := json.Unmarshal(b, &d); err != nil { + return nil, err + } + switch d.Type { + case "authorized_user": + cfg := &oauth2.Config{ + ClientID: d.ClientID, + ClientSecret: d.ClientSecret, + Scopes: append([]string{}, scopes...), // copy + Endpoint: Endpoint, + } + tok := &oauth2.Token{RefreshToken: d.RefreshToken} + return cfg.TokenSource(ctx, tok), nil + case "service_account": + cfg := &jwt.Config{ + Email: d.ClientEmail, + PrivateKey: []byte(d.PrivateKey), + Scopes: append([]string{}, scopes...), // copy + TokenURL: JWTTokenURL, + } + return cfg.TokenSource(ctx), nil + case "": + return nil, errors.New("missing 'type' field in credentials") + default: + return nil, fmt.Errorf("unknown credential type: %q", d.Type) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/example_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/example_test.go new file mode 100644 index 0000000000..9745be1929 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/example_test.go @@ -0,0 +1,150 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appenginevm !appengine + +package google_test + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/appengine" + "google.golang.org/appengine/urlfetch" +) + +func ExampleDefaultClient() { + client, err := google.DefaultClient(oauth2.NoContext, + "https://www.googleapis.com/auth/devstorage.full_control") + if err != nil { + log.Fatal(err) + } + client.Get("...") +} + +func Example_webServer() { + // Your credentials should be obtained from the Google + // Developer Console (https://console.developers.google.com). + conf := &oauth2.Config{ + ClientID: "YOUR_CLIENT_ID", + ClientSecret: "YOUR_CLIENT_SECRET", + RedirectURL: "YOUR_REDIRECT_URL", + Scopes: []string{ + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/blogger", + }, + Endpoint: google.Endpoint, + } + // Redirect user to Google's consent page to ask for permission + // for the scopes specified above. + url := conf.AuthCodeURL("state") + fmt.Printf("Visit the URL for the auth dialog: %v", url) + + // Handle the exchange code to initiate a transport. + tok, err := conf.Exchange(oauth2.NoContext, "authorization-code") + if err != nil { + log.Fatal(err) + } + client := conf.Client(oauth2.NoContext, tok) + client.Get("...") +} + +func ExampleJWTConfigFromJSON() { + // Your credentials should be obtained from the Google + // Developer Console (https://console.developers.google.com). + // Navigate to your project, then see the "Credentials" page + // under "APIs & Auth". + // To create a service account client, click "Create new Client ID", + // select "Service Account", and click "Create Client ID". A JSON + // key file will then be downloaded to your computer. + data, err := ioutil.ReadFile("/path/to/your-project-key.json") + if err != nil { + log.Fatal(err) + } + conf, err := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/bigquery") + if err != nil { + log.Fatal(err) + } + // Initiate an http.Client. The following GET request will be + // authorized and authenticated on the behalf of + // your service account. + client := conf.Client(oauth2.NoContext) + client.Get("...") +} + +func ExampleSDKConfig() { + // The credentials will be obtained from the first account that + // has been authorized with `gcloud auth login`. + conf, err := google.NewSDKConfig("") + if err != nil { + log.Fatal(err) + } + // Initiate an http.Client. The following GET request will be + // authorized and authenticated on the behalf of the SDK user. + client := conf.Client(oauth2.NoContext) + client.Get("...") +} + +func Example_serviceAccount() { + // Your credentials should be obtained from the Google + // Developer Console (https://console.developers.google.com). + conf := &jwt.Config{ + Email: "xxx@developer.gserviceaccount.com", + // The contents of your RSA private key or your PEM file + // that contains a private key. + // If you have a p12 file instead, you + // can use `openssl` to export the private key into a pem file. + // + // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes + // + // The field only supports PEM containers with no passphrase. + // The openssl command will convert p12 keys to passphrase-less PEM containers. + PrivateKey: []byte("-----BEGIN RSA PRIVATE KEY-----..."), + Scopes: []string{ + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/blogger", + }, + TokenURL: google.JWTTokenURL, + // If you would like to impersonate a user, you can + // create a transport with a subject. The following GET + // request will be made on the behalf of user@example.com. + // Optional. + Subject: "user@example.com", + } + // Initiate an http.Client, the following GET request will be + // authorized and authenticated on the behalf of user@example.com. + client := conf.Client(oauth2.NoContext) + client.Get("...") +} + +func ExampleAppEngineTokenSource() { + var req *http.Request // from the ServeHTTP handler + ctx := appengine.NewContext(req) + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: google.AppEngineTokenSource(ctx, "https://www.googleapis.com/auth/bigquery"), + Base: &urlfetch.Transport{ + Context: ctx, + }, + }, + } + client.Get("...") +} + +func ExampleComputeTokenSource() { + client := &http.Client{ + Transport: &oauth2.Transport{ + // Fetch from Google Compute Engine's metadata server to retrieve + // an access token for the provided account. + // If no account is specified, "default" is used. + Source: google.ComputeTokenSource(""), + }, + } + client.Get("...") +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/google.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/google.go new file mode 100644 index 0000000000..9a3d5feb1b --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/google.go @@ -0,0 +1,145 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package google provides support for making OAuth2 authorized and +// authenticated HTTP requests to Google APIs. +// It supports the Web server flow, client-side credentials, service accounts, +// Google Compute Engine service accounts, and Google App Engine service +// accounts. +// +// For more information, please read +// https://developers.google.com/accounts/docs/OAuth2 +// and +// https://developers.google.com/accounts/docs/application-default-credentials. +package google // import "golang.org/x/oauth2/google" + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/jwt" + "google.golang.org/cloud/compute/metadata" +) + +// Endpoint is Google's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://accounts.google.com/o/oauth2/auth", + TokenURL: "https://accounts.google.com/o/oauth2/token", +} + +// JWTTokenURL is Google's OAuth 2.0 token URL to use with the JWT flow. +const JWTTokenURL = "https://accounts.google.com/o/oauth2/token" + +// ConfigFromJSON uses a Google Developers Console client_credentials.json +// file to construct a config. +// client_credentials.json can be downloadable from https://console.developers.google.com, +// under "APIs & Auth" > "Credentials". Download the Web application credentials in the +// JSON format and provide the contents of the file as jsonKey. +func ConfigFromJSON(jsonKey []byte, scope ...string) (*oauth2.Config, error) { + type cred struct { + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + RedirectURIs []string `json:"redirect_uris"` + AuthURI string `json:"auth_uri"` + TokenURI string `json:"token_uri"` + } + var j struct { + Web *cred `json:"web"` + Installed *cred `json:"installed"` + } + if err := json.Unmarshal(jsonKey, &j); err != nil { + return nil, err + } + var c *cred + switch { + case j.Web != nil: + c = j.Web + case j.Installed != nil: + c = j.Installed + default: + return nil, fmt.Errorf("oauth2/google: no credentials found") + } + if len(c.RedirectURIs) < 1 { + return nil, errors.New("oauth2/google: missing redirect URL in the client_credentials.json") + } + return &oauth2.Config{ + ClientID: c.ClientID, + ClientSecret: c.ClientSecret, + RedirectURL: c.RedirectURIs[0], + Scopes: scope, + Endpoint: oauth2.Endpoint{ + AuthURL: c.AuthURI, + TokenURL: c.TokenURI, + }, + }, nil +} + +// JWTConfigFromJSON uses a Google Developers service account JSON key file to read +// the credentials that authorize and authenticate the requests. +// Create a service account on "Credentials" page under "APIs & Auth" for your +// project at https://console.developers.google.com to download a JSON key file. +func JWTConfigFromJSON(jsonKey []byte, scope ...string) (*jwt.Config, error) { + var key struct { + Email string `json:"client_email"` + PrivateKey string `json:"private_key"` + } + if err := json.Unmarshal(jsonKey, &key); err != nil { + return nil, err + } + return &jwt.Config{ + Email: key.Email, + PrivateKey: []byte(key.PrivateKey), + Scopes: scope, + TokenURL: JWTTokenURL, + }, nil +} + +// ComputeTokenSource returns a token source that fetches access tokens +// from Google Compute Engine (GCE)'s metadata server. It's only valid to use +// this token source if your program is running on a GCE instance. +// If no account is specified, "default" is used. +// Further information about retrieving access tokens from the GCE metadata +// server can be found at https://cloud.google.com/compute/docs/authentication. +func ComputeTokenSource(account string) oauth2.TokenSource { + return oauth2.ReuseTokenSource(nil, computeSource{account: account}) +} + +type computeSource struct { + account string +} + +func (cs computeSource) Token() (*oauth2.Token, error) { + if !metadata.OnGCE() { + return nil, errors.New("oauth2/google: can't get a token from the metadata service; not running on GCE") + } + acct := cs.account + if acct == "" { + acct = "default" + } + tokenJSON, err := metadata.Get("instance/service-accounts/" + acct + "/token") + if err != nil { + return nil, err + } + var res struct { + AccessToken string `json:"access_token"` + ExpiresInSec int `json:"expires_in"` + TokenType string `json:"token_type"` + } + err = json.NewDecoder(strings.NewReader(tokenJSON)).Decode(&res) + if err != nil { + return nil, fmt.Errorf("oauth2/google: invalid token JSON from metadata: %v", err) + } + if res.ExpiresInSec == 0 || res.AccessToken == "" { + return nil, fmt.Errorf("oauth2/google: incomplete token received from metadata") + } + return &oauth2.Token{ + AccessToken: res.AccessToken, + TokenType: res.TokenType, + Expiry: time.Now().Add(time.Duration(res.ExpiresInSec) * time.Second), + }, nil +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/google_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/google_test.go new file mode 100644 index 0000000000..74080edea3 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/google_test.go @@ -0,0 +1,67 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "strings" + "testing" +) + +var webJSONKey = []byte(` +{ + "web": { + "auth_uri": "https://google.com/o/oauth2/auth", + "client_secret": "3Oknc4jS_wA2r9i", + "token_uri": "https://google.com/o/oauth2/token", + "client_email": "222-nprqovg5k43uum874cs9osjt2koe97g8@developer.gserviceaccount.com", + "redirect_uris": ["https://www.example.com/oauth2callback"], + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/222-nprqovg5k43uum874cs9osjt2koe97g8@developer.gserviceaccount.com", + "client_id": "222-nprqovg5k43uum874cs9osjt2koe97g8.apps.googleusercontent.com", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "javascript_origins": ["https://www.example.com"] + } +}`) + +var installedJSONKey = []byte(`{ + "installed": { + "client_id": "222-installed.apps.googleusercontent.com", + "redirect_uris": ["https://www.example.com/oauth2callback"] + } +}`) + +func TestConfigFromJSON(t *testing.T) { + conf, err := ConfigFromJSON(webJSONKey, "scope1", "scope2") + if err != nil { + t.Error(err) + } + if got, want := conf.ClientID, "222-nprqovg5k43uum874cs9osjt2koe97g8.apps.googleusercontent.com"; got != want { + t.Errorf("ClientID = %q; want %q", got, want) + } + if got, want := conf.ClientSecret, "3Oknc4jS_wA2r9i"; got != want { + t.Errorf("ClientSecret = %q; want %q", got, want) + } + if got, want := conf.RedirectURL, "https://www.example.com/oauth2callback"; got != want { + t.Errorf("RedictURL = %q; want %q", got, want) + } + if got, want := strings.Join(conf.Scopes, ","), "scope1,scope2"; got != want { + t.Errorf("Scopes = %q; want %q", got, want) + } + if got, want := conf.Endpoint.AuthURL, "https://google.com/o/oauth2/auth"; got != want { + t.Errorf("AuthURL = %q; want %q", got, want) + } + if got, want := conf.Endpoint.TokenURL, "https://google.com/o/oauth2/token"; got != want { + t.Errorf("TokenURL = %q; want %q", got, want) + } +} + +func TestConfigFromJSON_Installed(t *testing.T) { + conf, err := ConfigFromJSON(installedJSONKey) + if err != nil { + t.Error(err) + } + if got, want := conf.ClientID, "222-installed.apps.googleusercontent.com"; got != want { + t.Errorf("ClientID = %q; want %q", got, want) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/jwt.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/jwt.go new file mode 100644 index 0000000000..b91991786f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/jwt.go @@ -0,0 +1,71 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "crypto/rsa" + "fmt" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/internal" + "golang.org/x/oauth2/jws" +) + +// JWTAccessTokenSourceFromJSON uses a Google Developers service account JSON +// key file to read the credentials that authorize and authenticate the +// requests, and returns a TokenSource that does not use any OAuth2 flow but +// instead creates a JWT and sends that as the access token. +// The audience is typically a URL that specifies the scope of the credentials. +// +// Note that this is not a standard OAuth flow, but rather an +// optimization supported by a few Google services. +// Unless you know otherwise, you should use JWTConfigFromJSON instead. +func JWTAccessTokenSourceFromJSON(jsonKey []byte, audience string) (oauth2.TokenSource, error) { + cfg, err := JWTConfigFromJSON(jsonKey) + if err != nil { + return nil, fmt.Errorf("google: could not parse JSON key: %v", err) + } + pk, err := internal.ParseKey(cfg.PrivateKey) + if err != nil { + return nil, fmt.Errorf("google: could not parse key: %v", err) + } + ts := &jwtAccessTokenSource{ + email: cfg.Email, + audience: audience, + pk: pk, + } + tok, err := ts.Token() + if err != nil { + return nil, err + } + return oauth2.ReuseTokenSource(tok, ts), nil +} + +type jwtAccessTokenSource struct { + email, audience string + pk *rsa.PrivateKey +} + +func (ts *jwtAccessTokenSource) Token() (*oauth2.Token, error) { + iat := time.Now() + exp := iat.Add(time.Hour) + cs := &jws.ClaimSet{ + Iss: ts.email, + Sub: ts.email, + Aud: ts.audience, + Iat: iat.Unix(), + Exp: exp.Unix(), + } + hdr := &jws.Header{ + Algorithm: "RS256", + Typ: "JWT", + } + msg, err := jws.Encode(hdr, cs, ts.pk) + if err != nil { + return nil, fmt.Errorf("google: could not encode JWT: %v", err) + } + return &oauth2.Token{AccessToken: msg, TokenType: "Bearer", Expiry: exp}, nil +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/sdk.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/sdk.go new file mode 100644 index 0000000000..d29a3bb9bb --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/sdk.go @@ -0,0 +1,168 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "os/user" + "path/filepath" + "runtime" + "strings" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/internal" +) + +type sdkCredentials struct { + Data []struct { + Credential struct { + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenExpiry *time.Time `json:"token_expiry"` + } `json:"credential"` + Key struct { + Account string `json:"account"` + Scope string `json:"scope"` + } `json:"key"` + } +} + +// An SDKConfig provides access to tokens from an account already +// authorized via the Google Cloud SDK. +type SDKConfig struct { + conf oauth2.Config + initialToken *oauth2.Token +} + +// NewSDKConfig creates an SDKConfig for the given Google Cloud SDK +// account. If account is empty, the account currently active in +// Google Cloud SDK properties is used. +// Google Cloud SDK credentials must be created by running `gcloud auth` +// before using this function. +// The Google Cloud SDK is available at https://cloud.google.com/sdk/. +func NewSDKConfig(account string) (*SDKConfig, error) { + configPath, err := sdkConfigPath() + if err != nil { + return nil, fmt.Errorf("oauth2/google: error getting SDK config path: %v", err) + } + credentialsPath := filepath.Join(configPath, "credentials") + f, err := os.Open(credentialsPath) + if err != nil { + return nil, fmt.Errorf("oauth2/google: failed to load SDK credentials: %v", err) + } + defer f.Close() + + var c sdkCredentials + if err := json.NewDecoder(f).Decode(&c); err != nil { + return nil, fmt.Errorf("oauth2/google: failed to decode SDK credentials from %q: %v", credentialsPath, err) + } + if len(c.Data) == 0 { + return nil, fmt.Errorf("oauth2/google: no credentials found in %q, run `gcloud auth login` to create one", credentialsPath) + } + if account == "" { + propertiesPath := filepath.Join(configPath, "properties") + f, err := os.Open(propertiesPath) + if err != nil { + return nil, fmt.Errorf("oauth2/google: failed to load SDK properties: %v", err) + } + defer f.Close() + ini, err := internal.ParseINI(f) + if err != nil { + return nil, fmt.Errorf("oauth2/google: failed to parse SDK properties %q: %v", propertiesPath, err) + } + core, ok := ini["core"] + if !ok { + return nil, fmt.Errorf("oauth2/google: failed to find [core] section in %v", ini) + } + active, ok := core["account"] + if !ok { + return nil, fmt.Errorf("oauth2/google: failed to find %q attribute in %v", "account", core) + } + account = active + } + + for _, d := range c.Data { + if account == "" || d.Key.Account == account { + if d.Credential.AccessToken == "" && d.Credential.RefreshToken == "" { + return nil, fmt.Errorf("oauth2/google: no token available for account %q", account) + } + var expiry time.Time + if d.Credential.TokenExpiry != nil { + expiry = *d.Credential.TokenExpiry + } + return &SDKConfig{ + conf: oauth2.Config{ + ClientID: d.Credential.ClientID, + ClientSecret: d.Credential.ClientSecret, + Scopes: strings.Split(d.Key.Scope, " "), + Endpoint: Endpoint, + RedirectURL: "oob", + }, + initialToken: &oauth2.Token{ + AccessToken: d.Credential.AccessToken, + RefreshToken: d.Credential.RefreshToken, + Expiry: expiry, + }, + }, nil + } + } + return nil, fmt.Errorf("oauth2/google: no such credentials for account %q", account) +} + +// Client returns an HTTP client using Google Cloud SDK credentials to +// authorize requests. The token will auto-refresh as necessary. The +// underlying http.RoundTripper will be obtained using the provided +// context. The returned client and its Transport should not be +// modified. +func (c *SDKConfig) Client(ctx context.Context) *http.Client { + return &http.Client{ + Transport: &oauth2.Transport{ + Source: c.TokenSource(ctx), + }, + } +} + +// TokenSource returns an oauth2.TokenSource that retrieve tokens from +// Google Cloud SDK credentials using the provided context. +// It will returns the current access token stored in the credentials, +// and refresh it when it expires, but it won't update the credentials +// with the new access token. +func (c *SDKConfig) TokenSource(ctx context.Context) oauth2.TokenSource { + return c.conf.TokenSource(ctx, c.initialToken) +} + +// Scopes are the OAuth 2.0 scopes the current account is authorized for. +func (c *SDKConfig) Scopes() []string { + return c.conf.Scopes +} + +// sdkConfigPath tries to guess where the gcloud config is located. +// It can be overridden during tests. +var sdkConfigPath = func() (string, error) { + if runtime.GOOS == "windows" { + return filepath.Join(os.Getenv("APPDATA"), "gcloud"), nil + } + homeDir := guessUnixHomeDir() + if homeDir == "" { + return "", errors.New("unable to get current user home directory: os/user lookup failed; $HOME is empty") + } + return filepath.Join(homeDir, ".config", "gcloud"), nil +} + +func guessUnixHomeDir() string { + usr, err := user.Current() + if err == nil { + return usr.HomeDir + } + return os.Getenv("HOME") +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/sdk_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/google/sdk_test.go new file mode 100644 index 0000000000..a5aa2a6466 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/sdk_test.go @@ -0,0 +1,46 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import "testing" + +func TestSDKConfig(t *testing.T) { + sdkConfigPath = func() (string, error) { + return "testdata/gcloud", nil + } + + tests := []struct { + account string + accessToken string + err bool + }{ + {"", "bar_access_token", false}, + {"foo@example.com", "foo_access_token", false}, + {"bar@example.com", "bar_access_token", false}, + {"baz@serviceaccount.example.com", "", true}, + } + for _, tt := range tests { + c, err := NewSDKConfig(tt.account) + if got, want := err != nil, tt.err; got != want { + if !tt.err { + t.Errorf("expected no error, got error: %v", tt.err, err) + } else { + t.Errorf("expected error, got none") + } + continue + } + if err != nil { + continue + } + tok := c.initialToken + if tok == nil { + t.Errorf("expected token %q, got: nil", tt.accessToken) + continue + } + if tok.AccessToken != tt.accessToken { + t.Errorf("expected token %q, got: %q", tt.accessToken, tok.AccessToken) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/testdata/gcloud/credentials b/Godeps/_workspace/src/golang.org/x/oauth2/google/testdata/gcloud/credentials new file mode 100644 index 0000000000..ff5eefbd0a --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/testdata/gcloud/credentials @@ -0,0 +1,122 @@ +{ + "data": [ + { + "credential": { + "_class": "OAuth2Credentials", + "_module": "oauth2client.client", + "access_token": "foo_access_token", + "client_id": "foo_client_id", + "client_secret": "foo_client_secret", + "id_token": { + "at_hash": "foo_at_hash", + "aud": "foo_aud", + "azp": "foo_azp", + "cid": "foo_cid", + "email": "foo@example.com", + "email_verified": true, + "exp": 1420573614, + "iat": 1420569714, + "id": "1337", + "iss": "accounts.google.com", + "sub": "1337", + "token_hash": "foo_token_hash", + "verified_email": true + }, + "invalid": false, + "refresh_token": "foo_refresh_token", + "revoke_uri": "https://accounts.google.com/o/oauth2/revoke", + "token_expiry": "2015-01-09T00:51:51Z", + "token_response": { + "access_token": "foo_access_token", + "expires_in": 3600, + "id_token": "foo_id_token", + "token_type": "Bearer" + }, + "token_uri": "https://accounts.google.com/o/oauth2/token", + "user_agent": "Cloud SDK Command Line Tool" + }, + "key": { + "account": "foo@example.com", + "clientId": "foo_client_id", + "scope": "https://www.googleapis.com/auth/appengine.admin https://www.googleapis.com/auth/bigquery https://www.googleapis.com/auth/compute https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/ndev.cloudman https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/sqlservice.admin https://www.googleapis.com/auth/prediction https://www.googleapis.com/auth/projecthosting", + "type": "google-cloud-sdk" + } + }, + { + "credential": { + "_class": "OAuth2Credentials", + "_module": "oauth2client.client", + "access_token": "bar_access_token", + "client_id": "bar_client_id", + "client_secret": "bar_client_secret", + "id_token": { + "at_hash": "bar_at_hash", + "aud": "bar_aud", + "azp": "bar_azp", + "cid": "bar_cid", + "email": "bar@example.com", + "email_verified": true, + "exp": 1420573614, + "iat": 1420569714, + "id": "1337", + "iss": "accounts.google.com", + "sub": "1337", + "token_hash": "bar_token_hash", + "verified_email": true + }, + "invalid": false, + "refresh_token": "bar_refresh_token", + "revoke_uri": "https://accounts.google.com/o/oauth2/revoke", + "token_expiry": "2015-01-09T00:51:51Z", + "token_response": { + "access_token": "bar_access_token", + "expires_in": 3600, + "id_token": "bar_id_token", + "token_type": "Bearer" + }, + "token_uri": "https://accounts.google.com/o/oauth2/token", + "user_agent": "Cloud SDK Command Line Tool" + }, + "key": { + "account": "bar@example.com", + "clientId": "bar_client_id", + "scope": "https://www.googleapis.com/auth/appengine.admin https://www.googleapis.com/auth/bigquery https://www.googleapis.com/auth/compute https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/ndev.cloudman https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/sqlservice.admin https://www.googleapis.com/auth/prediction https://www.googleapis.com/auth/projecthosting", + "type": "google-cloud-sdk" + } + }, + { + "credential": { + "_class": "ServiceAccountCredentials", + "_kwargs": {}, + "_module": "oauth2client.client", + "_private_key_id": "00000000000000000000000000000000", + "_private_key_pkcs8_text": "-----BEGIN RSA PRIVATE KEY-----\nMIICWwIBAAKBgQCt3fpiynPSaUhWSIKMGV331zudwJ6GkGmvQtwsoK2S2LbvnSwU\nNxgj4fp08kIDR5p26wF4+t/HrKydMwzftXBfZ9UmLVJgRdSswmS5SmChCrfDS5OE\nvFFcN5+6w1w8/Nu657PF/dse8T0bV95YrqyoR0Osy8WHrUOMSIIbC3hRuwIDAQAB\nAoGAJrGE/KFjn0sQ7yrZ6sXmdLawrM3mObo/2uI9T60+k7SpGbBX0/Pi6nFrJMWZ\nTVONG7P3Mu5aCPzzuVRYJB0j8aldSfzABTY3HKoWCczqw1OztJiEseXGiYz4QOyr\nYU3qDyEpdhS6q6wcoLKGH+hqRmz6pcSEsc8XzOOu7s4xW8kCQQDkc75HjhbarCnd\nJJGMe3U76+6UGmdK67ltZj6k6xoB5WbTNChY9TAyI2JC+ppYV89zv3ssj4L+02u3\nHIHFGxsHAkEAwtU1qYb1tScpchPobnYUFiVKJ7KA8EZaHVaJJODW/cghTCV7BxcJ\nbgVvlmk4lFKn3lPKAgWw7PdQsBTVBUcCrQJATPwoIirizrv3u5soJUQxZIkENAqV\nxmybZx9uetrzP7JTrVbFRf0SScMcyN90hdLJiQL8+i4+gaszgFht7sNMnwJAAbfj\nq0UXcauQwALQ7/h2oONfTg5S+MuGC/AxcXPSMZbMRGGoPh3D5YaCv27aIuS/ukQ+\n6dmm/9AGlCb64fsIWQJAPaokbjIifo+LwC5gyK73Mc4t8nAOSZDenzd/2f6TCq76\nS1dcnKiPxaED7W/y6LJiuBT2rbZiQ2L93NJpFZD/UA==\n-----END RSA PRIVATE KEY-----\n", + "_revoke_uri": "https://accounts.google.com/o/oauth2/revoke", + "_scopes": "https://www.googleapis.com/auth/appengine.admin https://www.googleapis.com/auth/bigquery https://www.googleapis.com/auth/compute https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/ndev.cloudman https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/sqlservice.admin https://www.googleapis.com/auth/prediction https://www.googleapis.com/auth/projecthosting", + "_service_account_email": "baz@serviceaccount.example.com", + "_service_account_id": "baz.serviceaccount.example.com", + "_token_uri": "https://accounts.google.com/o/oauth2/token", + "_user_agent": "Cloud SDK Command Line Tool", + "access_token": null, + "assertion_type": null, + "client_id": null, + "client_secret": null, + "id_token": null, + "invalid": false, + "refresh_token": null, + "revoke_uri": "https://accounts.google.com/o/oauth2/revoke", + "service_account_name": "baz@serviceaccount.example.com", + "token_expiry": null, + "token_response": null, + "user_agent": "Cloud SDK Command Line Tool" + }, + "key": { + "account": "baz@serviceaccount.example.com", + "clientId": "baz_client_id", + "scope": "https://www.googleapis.com/auth/appengine.admin https://www.googleapis.com/auth/bigquery https://www.googleapis.com/auth/compute https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/ndev.cloudman https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/sqlservice.admin https://www.googleapis.com/auth/prediction https://www.googleapis.com/auth/projecthosting", + "type": "google-cloud-sdk" + } + } + ], + "file_version": 1 +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/google/testdata/gcloud/properties b/Godeps/_workspace/src/golang.org/x/oauth2/google/testdata/gcloud/properties new file mode 100644 index 0000000000..025de886cf --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/google/testdata/gcloud/properties @@ -0,0 +1,2 @@ +[core] +account = bar@example.com \ No newline at end of file diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/internal/oauth2.go b/Godeps/_workspace/src/golang.org/x/oauth2/internal/oauth2.go new file mode 100644 index 0000000000..fbe1028d64 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/internal/oauth2.go @@ -0,0 +1,76 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "bufio" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io" + "strings" +) + +// ParseKey converts the binary contents of a private key file +// to an *rsa.PrivateKey. It detects whether the private key is in a +// PEM container or not. If so, it extracts the the private key +// from PEM container before conversion. It only supports PEM +// containers with no passphrase. +func ParseKey(key []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(key) + if block != nil { + key = block.Bytes + } + parsedKey, err := x509.ParsePKCS8PrivateKey(key) + if err != nil { + parsedKey, err = x509.ParsePKCS1PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("private key should be a PEM or plain PKSC1 or PKCS8; parse error: %v", err) + } + } + parsed, ok := parsedKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("private key is invalid") + } + return parsed, nil +} + +func ParseINI(ini io.Reader) (map[string]map[string]string, error) { + result := map[string]map[string]string{ + "": map[string]string{}, // root section + } + scanner := bufio.NewScanner(ini) + currentSection := "" + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, ";") { + // comment. + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + currentSection = strings.TrimSpace(line[1 : len(line)-1]) + result[currentSection] = map[string]string{} + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 && parts[0] != "" { + result[currentSection][strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning ini: %v", err) + } + return result, nil +} + +func CondVal(v string) []string { + if v == "" { + return nil + } + return []string{v} +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/internal/oauth2_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/internal/oauth2_test.go new file mode 100644 index 0000000000..c61585542d --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/internal/oauth2_test.go @@ -0,0 +1,62 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseINI(t *testing.T) { + tests := []struct { + ini string + want map[string]map[string]string + }{ + { + `root = toor +[foo] +bar = hop +ini = nin +`, + map[string]map[string]string{ + "": map[string]string{"root": "toor"}, + "foo": map[string]string{"bar": "hop", "ini": "nin"}, + }, + }, + { + `[empty] +[section] +empty= +`, + map[string]map[string]string{ + "": map[string]string{}, + "empty": map[string]string{}, + "section": map[string]string{"empty": ""}, + }, + }, + { + `ignore +[invalid +=stuff +;comment=true +`, + map[string]map[string]string{ + "": map[string]string{}, + }, + }, + } + for _, tt := range tests { + result, err := ParseINI(strings.NewReader(tt.ini)) + if err != nil { + t.Errorf("ParseINI(%q) error %v, want: no error", tt.ini, err) + continue + } + if !reflect.DeepEqual(result, tt.want) { + t.Errorf("ParseINI(%q) = %#v, want: %#v", tt.ini, result, tt.want) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/internal/token.go b/Godeps/_workspace/src/golang.org/x/oauth2/internal/token.go new file mode 100644 index 0000000000..909d52a128 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/internal/token.go @@ -0,0 +1,214 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "golang.org/x/net/context" +) + +// Token represents the crendentials used to authorize +// the requests to access protected resources on the OAuth 2.0 +// provider's backend. +// +// This type is a mirror of oauth2.Token and exists to break +// an otherwise-circular dependency. Other internal packages +// should convert this Token into an oauth2.Token before use. +type Token struct { + // AccessToken is the token that authorizes and authenticates + // the requests. + AccessToken string + + // TokenType is the type of token. + // The Type method returns either this or "Bearer", the default. + TokenType string + + // RefreshToken is a token that's used by the application + // (as opposed to the user) to refresh the access token + // if it expires. + RefreshToken string + + // Expiry is the optional expiration time of the access token. + // + // If zero, TokenSource implementations will reuse the same + // token forever and RefreshToken or equivalent + // mechanisms for that TokenSource will not be used. + Expiry time.Time + + // Raw optionally contains extra metadata from the server + // when updating a token. + Raw interface{} +} + +// tokenJSON is the struct representing the HTTP response from OAuth2 +// providers returning a token in JSON form. +type tokenJSON struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + RefreshToken string `json:"refresh_token"` + ExpiresIn expirationTime `json:"expires_in"` // at least PayPal returns string, while most return number + Expires expirationTime `json:"expires"` // broken Facebook spelling of expires_in +} + +func (e *tokenJSON) expiry() (t time.Time) { + if v := e.ExpiresIn; v != 0 { + return time.Now().Add(time.Duration(v) * time.Second) + } + if v := e.Expires; v != 0 { + return time.Now().Add(time.Duration(v) * time.Second) + } + return +} + +type expirationTime int32 + +func (e *expirationTime) UnmarshalJSON(b []byte) error { + var n json.Number + err := json.Unmarshal(b, &n) + if err != nil { + return err + } + i, err := n.Int64() + if err != nil { + return err + } + *e = expirationTime(i) + return nil +} + +var brokenAuthHeaderProviders = []string{ + "https://accounts.google.com/", + "https://www.googleapis.com/", + "https://api.instagram.com/", + "https://www.douban.com/", + "https://api.dropbox.com/", + "https://api.soundcloud.com/", + "https://www.linkedin.com/", + "https://api.twitch.tv/", + "https://oauth.vk.com/", + "https://api.odnoklassniki.ru/", + "https://connect.stripe.com/", + "https://api.pushbullet.com/", + "https://oauth.sandbox.trainingpeaks.com/", + "https://oauth.trainingpeaks.com/", + "https://www.strava.com/oauth/", + "https://app.box.com/", + "https://test-sandbox.auth.corp.google.com", + "https://user.gini.net/", + "https://api.netatmo.net/", + "https://slack.com/", +} + +// providerAuthHeaderWorks reports whether the OAuth2 server identified by the tokenURL +// implements the OAuth2 spec correctly +// See https://code.google.com/p/goauth2/issues/detail?id=31 for background. +// In summary: +// - Reddit only accepts client secret in the Authorization header +// - Dropbox accepts either it in URL param or Auth header, but not both. +// - Google only accepts URL param (not spec compliant?), not Auth header +// - Stripe only accepts client secret in Auth header with Bearer method, not Basic +func providerAuthHeaderWorks(tokenURL string) bool { + for _, s := range brokenAuthHeaderProviders { + if strings.HasPrefix(tokenURL, s) { + // Some sites fail to implement the OAuth2 spec fully. + return false + } + } + + // Assume the provider implements the spec properly + // otherwise. We can add more exceptions as they're + // discovered. We will _not_ be adding configurable hooks + // to this package to let users select server bugs. + return true +} + +func RetrieveToken(ctx context.Context, ClientID, ClientSecret, TokenURL string, v url.Values) (*Token, error) { + hc, err := ContextClient(ctx) + if err != nil { + return nil, err + } + v.Set("client_id", ClientID) + bustedAuth := !providerAuthHeaderWorks(TokenURL) + if bustedAuth && ClientSecret != "" { + v.Set("client_secret", ClientSecret) + } + req, err := http.NewRequest("POST", TokenURL, strings.NewReader(v.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + if !bustedAuth { + req.SetBasicAuth(ClientID, ClientSecret) + } + r, err := hc.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + if code := r.StatusCode; code < 200 || code > 299 { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v\nResponse: %s", r.Status, body) + } + + var token *Token + content, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + switch content { + case "application/x-www-form-urlencoded", "text/plain": + vals, err := url.ParseQuery(string(body)) + if err != nil { + return nil, err + } + token = &Token{ + AccessToken: vals.Get("access_token"), + TokenType: vals.Get("token_type"), + RefreshToken: vals.Get("refresh_token"), + Raw: vals, + } + e := vals.Get("expires_in") + if e == "" { + // TODO(jbd): Facebook's OAuth2 implementation is broken and + // returns expires_in field in expires. Remove the fallback to expires, + // when Facebook fixes their implementation. + e = vals.Get("expires") + } + expires, _ := strconv.Atoi(e) + if expires != 0 { + token.Expiry = time.Now().Add(time.Duration(expires) * time.Second) + } + default: + var tj tokenJSON + if err = json.Unmarshal(body, &tj); err != nil { + return nil, err + } + token = &Token{ + AccessToken: tj.AccessToken, + TokenType: tj.TokenType, + RefreshToken: tj.RefreshToken, + Expiry: tj.expiry(), + Raw: make(map[string]interface{}), + } + json.Unmarshal(body, &token.Raw) // no error checks for optional fields + } + // Don't overwrite `RefreshToken` with an empty value + // if this was a token refreshing request. + if token.RefreshToken == "" { + token.RefreshToken = v.Get("refresh_token") + } + return token, nil +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/internal/token_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/internal/token_test.go new file mode 100644 index 0000000000..626e93354a --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/internal/token_test.go @@ -0,0 +1,28 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "fmt" + "testing" +) + +func Test_providerAuthHeaderWorks(t *testing.T) { + for _, p := range brokenAuthHeaderProviders { + if providerAuthHeaderWorks(p) { + t.Errorf("URL: %s not found in list", p) + } + p := fmt.Sprintf("%ssomesuffix", p) + if providerAuthHeaderWorks(p) { + t.Errorf("URL: %s not found in list", p) + } + } + p := "https://api.not-in-the-list-example.com/" + if !providerAuthHeaderWorks(p) { + t.Errorf("URL: %s found in list", p) + } + +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/internal/transport.go b/Godeps/_workspace/src/golang.org/x/oauth2/internal/transport.go new file mode 100644 index 0000000000..33d3669167 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/internal/transport.go @@ -0,0 +1,67 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "net/http" + + "golang.org/x/net/context" +) + +// HTTPClient is the context key to use with golang.org/x/net/context's +// WithValue function to associate an *http.Client value with a context. +var HTTPClient ContextKey + +// ContextKey is just an empty struct. It exists so HTTPClient can be +// an immutable public variable with a unique type. It's immutable +// because nobody else can create a ContextKey, being unexported. +type ContextKey struct{} + +// ContextClientFunc is a func which tries to return an *http.Client +// given a Context value. If it returns an error, the search stops +// with that error. If it returns (nil, nil), the search continues +// down the list of registered funcs. +type ContextClientFunc func(context.Context) (*http.Client, error) + +var contextClientFuncs []ContextClientFunc + +func RegisterContextClientFunc(fn ContextClientFunc) { + contextClientFuncs = append(contextClientFuncs, fn) +} + +func ContextClient(ctx context.Context) (*http.Client, error) { + for _, fn := range contextClientFuncs { + c, err := fn(ctx) + if err != nil { + return nil, err + } + if c != nil { + return c, nil + } + } + if hc, ok := ctx.Value(HTTPClient).(*http.Client); ok { + return hc, nil + } + return http.DefaultClient, nil +} + +func ContextTransport(ctx context.Context) http.RoundTripper { + hc, err := ContextClient(ctx) + // This is a rare error case (somebody using nil on App Engine). + if err != nil { + return ErrorTransport{err} + } + return hc.Transport +} + +// ErrorTransport returns the specified error on RoundTrip. +// This RoundTripper should be used in rare error cases where +// error handling can be postponed to response handling time. +type ErrorTransport struct{ Err error } + +func (t ErrorTransport) RoundTrip(*http.Request) (*http.Response, error) { + return nil, t.Err +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/jws/jws.go b/Godeps/_workspace/src/golang.org/x/oauth2/jws/jws.go new file mode 100644 index 0000000000..6b86a3e131 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/jws/jws.go @@ -0,0 +1,159 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package jws provides encoding and decoding utilities for +// signed JWS messages. +package jws // import "golang.org/x/oauth2/jws" + +import ( + "bytes" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// ClaimSet contains information about the JWT signature including the +// permissions being requested (scopes), the target of the token, the issuer, +// the time the token was issued, and the lifetime of the token. +type ClaimSet struct { + Iss string `json:"iss"` // email address of the client_id of the application making the access token request + Scope string `json:"scope,omitempty"` // space-delimited list of the permissions the application requests + Aud string `json:"aud"` // descriptor of the intended target of the assertion (Optional). + Exp int64 `json:"exp"` // the expiration time of the assertion (seconds since Unix epoch) + Iat int64 `json:"iat"` // the time the assertion was issued (seconds since Unix epoch) + Typ string `json:"typ,omitempty"` // token type (Optional). + + // Email for which the application is requesting delegated access (Optional). + Sub string `json:"sub,omitempty"` + + // The old name of Sub. Client keeps setting Prn to be + // complaint with legacy OAuth 2.0 providers. (Optional) + Prn string `json:"prn,omitempty"` + + // See http://tools.ietf.org/html/draft-jones-json-web-token-10#section-4.3 + // This array is marshalled using custom code (see (c *ClaimSet) encode()). + PrivateClaims map[string]interface{} `json:"-"` +} + +func (c *ClaimSet) encode() (string, error) { + // Reverting time back for machines whose time is not perfectly in sync. + // If client machine's time is in the future according + // to Google servers, an access token will not be issued. + now := time.Now().Add(-10 * time.Second) + if c.Iat == 0 { + c.Iat = now.Unix() + } + if c.Exp == 0 { + c.Exp = now.Add(time.Hour).Unix() + } + if c.Exp < c.Iat { + return "", fmt.Errorf("jws: invalid Exp = %v; must be later than Iat = %v", c.Exp, c.Iat) + } + + b, err := json.Marshal(c) + if err != nil { + return "", err + } + + if len(c.PrivateClaims) == 0 { + return base64Encode(b), nil + } + + // Marshal private claim set and then append it to b. + prv, err := json.Marshal(c.PrivateClaims) + if err != nil { + return "", fmt.Errorf("jws: invalid map of private claims %v", c.PrivateClaims) + } + + // Concatenate public and private claim JSON objects. + if !bytes.HasSuffix(b, []byte{'}'}) { + return "", fmt.Errorf("jws: invalid JSON %s", b) + } + if !bytes.HasPrefix(prv, []byte{'{'}) { + return "", fmt.Errorf("jws: invalid JSON %s", prv) + } + b[len(b)-1] = ',' // Replace closing curly brace with a comma. + b = append(b, prv[1:]...) // Append private claims. + return base64Encode(b), nil +} + +// Header represents the header for the signed JWS payloads. +type Header struct { + // The algorithm used for signature. + Algorithm string `json:"alg"` + + // Represents the token type. + Typ string `json:"typ"` +} + +func (h *Header) encode() (string, error) { + b, err := json.Marshal(h) + if err != nil { + return "", err + } + return base64Encode(b), nil +} + +// Decode decodes a claim set from a JWS payload. +func Decode(payload string) (*ClaimSet, error) { + // decode returned id token to get expiry + s := strings.Split(payload, ".") + if len(s) < 2 { + // TODO(jbd): Provide more context about the error. + return nil, errors.New("jws: invalid token received") + } + decoded, err := base64Decode(s[1]) + if err != nil { + return nil, err + } + c := &ClaimSet{} + err = json.NewDecoder(bytes.NewBuffer(decoded)).Decode(c) + return c, err +} + +// Encode encodes a signed JWS with provided header and claim set. +func Encode(header *Header, c *ClaimSet, signature *rsa.PrivateKey) (string, error) { + head, err := header.encode() + if err != nil { + return "", err + } + cs, err := c.encode() + if err != nil { + return "", err + } + ss := fmt.Sprintf("%s.%s", head, cs) + h := sha256.New() + h.Write([]byte(ss)) + b, err := rsa.SignPKCS1v15(rand.Reader, signature, crypto.SHA256, h.Sum(nil)) + if err != nil { + return "", err + } + sig := base64Encode(b) + return fmt.Sprintf("%s.%s", ss, sig), nil +} + +// base64Encode returns and Base64url encoded version of the input string with any +// trailing "=" stripped. +func base64Encode(b []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") +} + +// base64Decode decodes the Base64url encoded string +func base64Decode(s string) ([]byte, error) { + // add back missing padding + switch len(s) % 4 { + case 2: + s += "==" + case 3: + s += "=" + } + return base64.URLEncoding.DecodeString(s) +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/jwt/example_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/jwt/example_test.go new file mode 100644 index 0000000000..a9533e85f0 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/jwt/example_test.go @@ -0,0 +1,31 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package jwt_test + +import ( + "golang.org/x/oauth2" + "golang.org/x/oauth2/jwt" +) + +func ExampleJWTConfig() { + conf := &jwt.Config{ + Email: "xxx@developer.com", + // The contents of your RSA private key or your PEM file + // that contains a private key. + // If you have a p12 file instead, you + // can use `openssl` to export the private key into a pem file. + // + // $ openssl pkcs12 -in key.p12 -out key.pem -nodes + // + // It only supports PEM containers with no passphrase. + PrivateKey: []byte("-----BEGIN RSA PRIVATE KEY-----..."), + Subject: "user@example.com", + TokenURL: "https://provider.com/o/oauth2/token", + } + // Initiate an http.Client, the following GET request will be + // authorized and authenticated on the behalf of user@example.com. + client := conf.Client(oauth2.NoContext) + client.Get("...") +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/jwt/jwt.go b/Godeps/_workspace/src/golang.org/x/oauth2/jwt/jwt.go new file mode 100644 index 0000000000..2ffad21a60 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/jwt/jwt.go @@ -0,0 +1,153 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package jwt implements the OAuth 2.0 JSON Web Token flow, commonly +// known as "two-legged OAuth 2.0". +// +// See: https://tools.ietf.org/html/draft-ietf-oauth-jwt-bearer-12 +package jwt + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/internal" + "golang.org/x/oauth2/jws" +) + +var ( + defaultGrantType = "urn:ietf:params:oauth:grant-type:jwt-bearer" + defaultHeader = &jws.Header{Algorithm: "RS256", Typ: "JWT"} +) + +// Config is the configuration for using JWT to fetch tokens, +// commonly known as "two-legged OAuth 2.0". +type Config struct { + // Email is the OAuth client identifier used when communicating with + // the configured OAuth provider. + Email string + + // PrivateKey contains the contents of an RSA private key or the + // contents of a PEM file that contains a private key. The provided + // private key is used to sign JWT payloads. + // PEM containers with a passphrase are not supported. + // Use the following command to convert a PKCS 12 file into a PEM. + // + // $ openssl pkcs12 -in key.p12 -out key.pem -nodes + // + PrivateKey []byte + + // Subject is the optional user to impersonate. + Subject string + + // Scopes optionally specifies a list of requested permission scopes. + Scopes []string + + // TokenURL is the endpoint required to complete the 2-legged JWT flow. + TokenURL string + + // Expires optionally specifies how long the token is valid for. + Expires time.Duration +} + +// TokenSource returns a JWT TokenSource using the configuration +// in c and the HTTP client from the provided context. +func (c *Config) TokenSource(ctx context.Context) oauth2.TokenSource { + return oauth2.ReuseTokenSource(nil, jwtSource{ctx, c}) +} + +// Client returns an HTTP client wrapping the context's +// HTTP transport and adding Authorization headers with tokens +// obtained from c. +// +// The returned client and its Transport should not be modified. +func (c *Config) Client(ctx context.Context) *http.Client { + return oauth2.NewClient(ctx, c.TokenSource(ctx)) +} + +// jwtSource is a source that always does a signed JWT request for a token. +// It should typically be wrapped with a reuseTokenSource. +type jwtSource struct { + ctx context.Context + conf *Config +} + +func (js jwtSource) Token() (*oauth2.Token, error) { + pk, err := internal.ParseKey(js.conf.PrivateKey) + if err != nil { + return nil, err + } + hc := oauth2.NewClient(js.ctx, nil) + claimSet := &jws.ClaimSet{ + Iss: js.conf.Email, + Scope: strings.Join(js.conf.Scopes, " "), + Aud: js.conf.TokenURL, + } + if subject := js.conf.Subject; subject != "" { + claimSet.Sub = subject + // prn is the old name of sub. Keep setting it + // to be compatible with legacy OAuth 2.0 providers. + claimSet.Prn = subject + } + if t := js.conf.Expires; t > 0 { + claimSet.Exp = time.Now().Add(t).Unix() + } + payload, err := jws.Encode(defaultHeader, claimSet, pk) + if err != nil { + return nil, err + } + v := url.Values{} + v.Set("grant_type", defaultGrantType) + v.Set("assertion", payload) + resp, err := hc.PostForm(js.conf.TokenURL, v) + if err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + if c := resp.StatusCode; c < 200 || c > 299 { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v\nResponse: %s", resp.Status, body) + } + // tokenRes is the JSON response body. + var tokenRes struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + IDToken string `json:"id_token"` + ExpiresIn int64 `json:"expires_in"` // relative seconds from now + } + if err := json.Unmarshal(body, &tokenRes); err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + token := &oauth2.Token{ + AccessToken: tokenRes.AccessToken, + TokenType: tokenRes.TokenType, + } + raw := make(map[string]interface{}) + json.Unmarshal(body, &raw) // no error checks for optional fields + token = token.WithExtra(raw) + + if secs := tokenRes.ExpiresIn; secs > 0 { + token.Expiry = time.Now().Add(time.Duration(secs) * time.Second) + } + if v := tokenRes.IDToken; v != "" { + // decode returned id token to get expiry + claimSet, err := jws.Decode(v) + if err != nil { + return nil, fmt.Errorf("oauth2: error decoding JWT token: %v", err) + } + token.Expiry = time.Unix(claimSet.Exp, 0) + } + return token, nil +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/jwt/jwt_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/jwt/jwt_test.go new file mode 100644 index 0000000000..a9c126b47a --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/jwt/jwt_test.go @@ -0,0 +1,134 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package jwt + +import ( + "net/http" + "net/http/httptest" + "testing" + + "golang.org/x/oauth2" +) + +var dummyPrivateKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAx4fm7dngEmOULNmAs1IGZ9Apfzh+BkaQ1dzkmbUgpcoghucE +DZRnAGd2aPyB6skGMXUytWQvNYav0WTR00wFtX1ohWTfv68HGXJ8QXCpyoSKSSFY +fuP9X36wBSkSX9J5DVgiuzD5VBdzUISSmapjKm+DcbRALjz6OUIPEWi1Tjl6p5RK +1w41qdbmt7E5/kGhKLDuT7+M83g4VWhgIvaAXtnhklDAggilPPa8ZJ1IFe31lNlr +k4DRk38nc6sEutdf3RL7QoH7FBusI7uXV03DC6dwN1kP4GE7bjJhcRb/7jYt7CQ9 +/E9Exz3c0yAp0yrTg0Fwh+qxfH9dKwN52S7SBwIDAQABAoIBAQCaCs26K07WY5Jt +3a2Cw3y2gPrIgTCqX6hJs7O5ByEhXZ8nBwsWANBUe4vrGaajQHdLj5OKfsIDrOvn +2NI1MqflqeAbu/kR32q3tq8/Rl+PPiwUsW3E6Pcf1orGMSNCXxeducF2iySySzh3 +nSIhCG5uwJDWI7a4+9KiieFgK1pt/Iv30q1SQS8IEntTfXYwANQrfKUVMmVF9aIK +6/WZE2yd5+q3wVVIJ6jsmTzoDCX6QQkkJICIYwCkglmVy5AeTckOVwcXL0jqw5Kf +5/soZJQwLEyBoQq7Kbpa26QHq+CJONetPP8Ssy8MJJXBT+u/bSseMb3Zsr5cr43e +DJOhwsThAoGBAPY6rPKl2NT/K7XfRCGm1sbWjUQyDShscwuWJ5+kD0yudnT/ZEJ1 +M3+KS/iOOAoHDdEDi9crRvMl0UfNa8MAcDKHflzxg2jg/QI+fTBjPP5GOX0lkZ9g +z6VePoVoQw2gpPFVNPPTxKfk27tEzbaffvOLGBEih0Kb7HTINkW8rIlzAoGBAM9y +1yr+jvfS1cGFtNU+Gotoihw2eMKtIqR03Yn3n0PK1nVCDKqwdUqCypz4+ml6cxRK +J8+Pfdh7D+ZJd4LEG6Y4QRDLuv5OA700tUoSHxMSNn3q9As4+T3MUyYxWKvTeu3U +f2NWP9ePU0lV8ttk7YlpVRaPQmc1qwooBA/z/8AdAoGAW9x0HWqmRICWTBnpjyxx +QGlW9rQ9mHEtUotIaRSJ6K/F3cxSGUEkX1a3FRnp6kPLcckC6NlqdNgNBd6rb2rA +cPl/uSkZP42Als+9YMoFPU/xrrDPbUhu72EDrj3Bllnyb168jKLa4VBOccUvggxr +Dm08I1hgYgdN5huzs7y6GeUCgYEAj+AZJSOJ6o1aXS6rfV3mMRve9bQ9yt8jcKXw +5HhOCEmMtaSKfnOF1Ziih34Sxsb7O2428DiX0mV/YHtBnPsAJidL0SdLWIapBzeg +KHArByIRkwE6IvJvwpGMdaex1PIGhx5i/3VZL9qiq/ElT05PhIb+UXgoWMabCp84 +OgxDK20CgYAeaFo8BdQ7FmVX2+EEejF+8xSge6WVLtkaon8bqcn6P0O8lLypoOhd +mJAYH8WU+UAy9pecUnDZj14LAGNVmYcse8HFX71MoshnvCTFEPVo4rZxIAGwMpeJ +5jgQ3slYLpqrGlcbLgUXBUgzEO684Wk/UV9DFPlHALVqCfXQ9dpJPg== +-----END RSA PRIVATE KEY-----`) + +func TestJWTFetch_JSONResponse(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{ + "access_token": "90d64460d14870c08c81352a05dedd3465940a7c", + "scope": "user", + "token_type": "bearer", + "expires_in": 3600 + }`)) + })) + defer ts.Close() + + conf := &Config{ + Email: "aaa@xxx.com", + PrivateKey: dummyPrivateKey, + TokenURL: ts.URL, + } + tok, err := conf.TokenSource(oauth2.NoContext).Token() + if err != nil { + t.Fatal(err) + } + if !tok.Valid() { + t.Errorf("Token invalid") + } + if tok.AccessToken != "90d64460d14870c08c81352a05dedd3465940a7c" { + t.Errorf("Unexpected access token, %#v", tok.AccessToken) + } + if tok.TokenType != "bearer" { + t.Errorf("Unexpected token type, %#v", tok.TokenType) + } + if tok.Expiry.IsZero() { + t.Errorf("Unexpected token expiry, %#v", tok.Expiry) + } + scope := tok.Extra("scope") + if scope != "user" { + t.Errorf("Unexpected value for scope: %v", scope) + } +} + +func TestJWTFetch_BadResponse(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"scope": "user", "token_type": "bearer"}`)) + })) + defer ts.Close() + + conf := &Config{ + Email: "aaa@xxx.com", + PrivateKey: dummyPrivateKey, + TokenURL: ts.URL, + } + tok, err := conf.TokenSource(oauth2.NoContext).Token() + if err != nil { + t.Fatal(err) + } + if tok == nil { + t.Fatalf("token is nil") + } + if tok.Valid() { + t.Errorf("token is valid. want invalid.") + } + if tok.AccessToken != "" { + t.Errorf("Unexpected non-empty access token %q.", tok.AccessToken) + } + if want := "bearer"; tok.TokenType != want { + t.Errorf("TokenType = %q; want %q", tok.TokenType, want) + } + scope := tok.Extra("scope") + if want := "user"; scope != want { + t.Errorf("token scope = %q; want %q", scope, want) + } +} + +func TestJWTFetch_BadResponseType(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"access_token":123, "scope": "user", "token_type": "bearer"}`)) + })) + defer ts.Close() + conf := &Config{ + Email: "aaa@xxx.com", + PrivateKey: dummyPrivateKey, + TokenURL: ts.URL, + } + tok, err := conf.TokenSource(oauth2.NoContext).Token() + if err == nil { + t.Error("got a token; expected error") + if tok.AccessToken != "" { + t.Errorf("Unexpected access token, %#v.", tok.AccessToken) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/linkedin/linkedin.go b/Godeps/_workspace/src/golang.org/x/oauth2/linkedin/linkedin.go new file mode 100644 index 0000000000..b619f93d2b --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/linkedin/linkedin.go @@ -0,0 +1,16 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package linkedin provides constants for using OAuth2 to access LinkedIn. +package linkedin // import "golang.org/x/oauth2/linkedin" + +import ( + "golang.org/x/oauth2" +) + +// Endpoint is LinkedIn's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://www.linkedin.com/uas/oauth2/authorization", + TokenURL: "https://www.linkedin.com/uas/oauth2/accessToken", +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/oauth2.go b/Godeps/_workspace/src/golang.org/x/oauth2/oauth2.go new file mode 100644 index 0000000000..97997f8c13 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/oauth2.go @@ -0,0 +1,325 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package oauth2 provides support for making +// OAuth2 authorized and authenticated HTTP requests. +// It can additionally grant authorization with Bearer JWT. +package oauth2 // import "golang.org/x/oauth2" + +import ( + "bytes" + "errors" + "net/http" + "net/url" + "strings" + "sync" + + "golang.org/x/net/context" + "golang.org/x/oauth2/internal" +) + +// NoContext is the default context you should supply if not using +// your own context.Context (see https://golang.org/x/net/context). +var NoContext = context.TODO() + +// Config describes a typical 3-legged OAuth2 flow, with both the +// client application information and the server's endpoint URLs. +type Config struct { + // ClientID is the application's ID. + ClientID string + + // ClientSecret is the application's secret. + ClientSecret string + + // Endpoint contains the resource server's token endpoint + // URLs. These are constants specific to each server and are + // often available via site-specific packages, such as + // google.Endpoint or github.Endpoint. + Endpoint Endpoint + + // RedirectURL is the URL to redirect users going through + // the OAuth flow, after the resource owner's URLs. + RedirectURL string + + // Scope specifies optional requested permissions. + Scopes []string +} + +// A TokenSource is anything that can return a token. +type TokenSource interface { + // Token returns a token or an error. + // Token must be safe for concurrent use by multiple goroutines. + // The returned Token must not be modified. + Token() (*Token, error) +} + +// Endpoint contains the OAuth 2.0 provider's authorization and token +// endpoint URLs. +type Endpoint struct { + AuthURL string + TokenURL string +} + +var ( + // AccessTypeOnline and AccessTypeOffline are options passed + // to the Options.AuthCodeURL method. They modify the + // "access_type" field that gets sent in the URL returned by + // AuthCodeURL. + // + // Online is the default if neither is specified. If your + // application needs to refresh access tokens when the user + // is not present at the browser, then use offline. This will + // result in your application obtaining a refresh token the + // first time your application exchanges an authorization + // code for a user. + AccessTypeOnline AuthCodeOption = SetAuthURLParam("access_type", "online") + AccessTypeOffline AuthCodeOption = SetAuthURLParam("access_type", "offline") + + // ApprovalForce forces the users to view the consent dialog + // and confirm the permissions request at the URL returned + // from AuthCodeURL, even if they've already done so. + ApprovalForce AuthCodeOption = SetAuthURLParam("approval_prompt", "force") +) + +// An AuthCodeOption is passed to Config.AuthCodeURL. +type AuthCodeOption interface { + setValue(url.Values) +} + +type setParam struct{ k, v string } + +func (p setParam) setValue(m url.Values) { m.Set(p.k, p.v) } + +// SetAuthURLParam builds an AuthCodeOption which passes key/value parameters +// to a provider's authorization endpoint. +func SetAuthURLParam(key, value string) AuthCodeOption { + return setParam{key, value} +} + +// AuthCodeURL returns a URL to OAuth 2.0 provider's consent page +// that asks for permissions for the required scopes explicitly. +// +// State is a token to protect the user from CSRF attacks. You must +// always provide a non-zero string and validate that it matches the +// the state query parameter on your redirect callback. +// See http://tools.ietf.org/html/rfc6749#section-10.12 for more info. +// +// Opts may include AccessTypeOnline or AccessTypeOffline, as well +// as ApprovalForce. +func (c *Config) AuthCodeURL(state string, opts ...AuthCodeOption) string { + var buf bytes.Buffer + buf.WriteString(c.Endpoint.AuthURL) + v := url.Values{ + "response_type": {"code"}, + "client_id": {c.ClientID}, + "redirect_uri": internal.CondVal(c.RedirectURL), + "scope": internal.CondVal(strings.Join(c.Scopes, " ")), + "state": internal.CondVal(state), + } + for _, opt := range opts { + opt.setValue(v) + } + if strings.Contains(c.Endpoint.AuthURL, "?") { + buf.WriteByte('&') + } else { + buf.WriteByte('?') + } + buf.WriteString(v.Encode()) + return buf.String() +} + +// PasswordCredentialsToken converts a resource owner username and password +// pair into a token. +// +// Per the RFC, this grant type should only be used "when there is a high +// degree of trust between the resource owner and the client (e.g., the client +// is part of the device operating system or a highly privileged application), +// and when other authorization grant types are not available." +// See https://tools.ietf.org/html/rfc6749#section-4.3 for more info. +// +// The HTTP client to use is derived from the context. +// If nil, http.DefaultClient is used. +func (c *Config) PasswordCredentialsToken(ctx context.Context, username, password string) (*Token, error) { + return retrieveToken(ctx, c, url.Values{ + "grant_type": {"password"}, + "username": {username}, + "password": {password}, + "scope": internal.CondVal(strings.Join(c.Scopes, " ")), + }) +} + +// Exchange converts an authorization code into a token. +// +// It is used after a resource provider redirects the user back +// to the Redirect URI (the URL obtained from AuthCodeURL). +// +// The HTTP client to use is derived from the context. +// If a client is not provided via the context, http.DefaultClient is used. +// +// The code will be in the *http.Request.FormValue("code"). Before +// calling Exchange, be sure to validate FormValue("state"). +func (c *Config) Exchange(ctx context.Context, code string) (*Token, error) { + return retrieveToken(ctx, c, url.Values{ + "grant_type": {"authorization_code"}, + "code": {code}, + "redirect_uri": internal.CondVal(c.RedirectURL), + "scope": internal.CondVal(strings.Join(c.Scopes, " ")), + }) +} + +// Client returns an HTTP client using the provided token. +// The token will auto-refresh as necessary. The underlying +// HTTP transport will be obtained using the provided context. +// The returned client and its Transport should not be modified. +func (c *Config) Client(ctx context.Context, t *Token) *http.Client { + return NewClient(ctx, c.TokenSource(ctx, t)) +} + +// TokenSource returns a TokenSource that returns t until t expires, +// automatically refreshing it as necessary using the provided context. +// +// Most users will use Config.Client instead. +func (c *Config) TokenSource(ctx context.Context, t *Token) TokenSource { + tkr := &tokenRefresher{ + ctx: ctx, + conf: c, + } + if t != nil { + tkr.refreshToken = t.RefreshToken + } + return &reuseTokenSource{ + t: t, + new: tkr, + } +} + +// tokenRefresher is a TokenSource that makes "grant_type"=="refresh_token" +// HTTP requests to renew a token using a RefreshToken. +type tokenRefresher struct { + ctx context.Context // used to get HTTP requests + conf *Config + refreshToken string +} + +// WARNING: Token is not safe for concurrent access, as it +// updates the tokenRefresher's refreshToken field. +// Within this package, it is used by reuseTokenSource which +// synchronizes calls to this method with its own mutex. +func (tf *tokenRefresher) Token() (*Token, error) { + if tf.refreshToken == "" { + return nil, errors.New("oauth2: token expired and refresh token is not set") + } + + tk, err := retrieveToken(tf.ctx, tf.conf, url.Values{ + "grant_type": {"refresh_token"}, + "refresh_token": {tf.refreshToken}, + }) + + if err != nil { + return nil, err + } + if tf.refreshToken != tk.RefreshToken { + tf.refreshToken = tk.RefreshToken + } + return tk, err +} + +// reuseTokenSource is a TokenSource that holds a single token in memory +// and validates its expiry before each call to retrieve it with +// Token. If it's expired, it will be auto-refreshed using the +// new TokenSource. +type reuseTokenSource struct { + new TokenSource // called when t is expired. + + mu sync.Mutex // guards t + t *Token +} + +// Token returns the current token if it's still valid, else will +// refresh the current token (using r.Context for HTTP client +// information) and return the new one. +func (s *reuseTokenSource) Token() (*Token, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.t.Valid() { + return s.t, nil + } + t, err := s.new.Token() + if err != nil { + return nil, err + } + s.t = t + return t, nil +} + +// StaticTokenSource returns a TokenSource that always returns the same token. +// Because the provided token t is never refreshed, StaticTokenSource is only +// useful for tokens that never expire. +func StaticTokenSource(t *Token) TokenSource { + return staticTokenSource{t} +} + +// staticTokenSource is a TokenSource that always returns the same Token. +type staticTokenSource struct { + t *Token +} + +func (s staticTokenSource) Token() (*Token, error) { + return s.t, nil +} + +// HTTPClient is the context key to use with golang.org/x/net/context's +// WithValue function to associate an *http.Client value with a context. +var HTTPClient internal.ContextKey + +// NewClient creates an *http.Client from a Context and TokenSource. +// The returned client is not valid beyond the lifetime of the context. +// +// As a special case, if src is nil, a non-OAuth2 client is returned +// using the provided context. This exists to support related OAuth2 +// packages. +func NewClient(ctx context.Context, src TokenSource) *http.Client { + if src == nil { + c, err := internal.ContextClient(ctx) + if err != nil { + return &http.Client{Transport: internal.ErrorTransport{err}} + } + return c + } + return &http.Client{ + Transport: &Transport{ + Base: internal.ContextTransport(ctx), + Source: ReuseTokenSource(nil, src), + }, + } +} + +// ReuseTokenSource returns a TokenSource which repeatedly returns the +// same token as long as it's valid, starting with t. +// When its cached token is invalid, a new token is obtained from src. +// +// ReuseTokenSource is typically used to reuse tokens from a cache +// (such as a file on disk) between runs of a program, rather than +// obtaining new tokens unnecessarily. +// +// The initial token t may be nil, in which case the TokenSource is +// wrapped in a caching version if it isn't one already. This also +// means it's always safe to wrap ReuseTokenSource around any other +// TokenSource without adverse effects. +func ReuseTokenSource(t *Token, src TokenSource) TokenSource { + // Don't wrap a reuseTokenSource in itself. That would work, + // but cause an unnecessary number of mutex operations. + // Just build the equivalent one. + if rt, ok := src.(*reuseTokenSource); ok { + if t == nil { + // Just use it directly. + return rt + } + src = rt.new + } + return &reuseTokenSource{ + t: t, + new: src, + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/oauth2_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/oauth2_test.go new file mode 100644 index 0000000000..448673b51c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/oauth2_test.go @@ -0,0 +1,471 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package oauth2 + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "testing" + "time" + + "golang.org/x/net/context" +) + +type mockTransport struct { + rt func(req *http.Request) (resp *http.Response, err error) +} + +func (t *mockTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) { + return t.rt(req) +} + +type mockCache struct { + token *Token + readErr error +} + +func (c *mockCache) ReadToken() (*Token, error) { + return c.token, c.readErr +} + +func (c *mockCache) WriteToken(*Token) { + // do nothing +} + +func newConf(url string) *Config { + return &Config{ + ClientID: "CLIENT_ID", + ClientSecret: "CLIENT_SECRET", + RedirectURL: "REDIRECT_URL", + Scopes: []string{"scope1", "scope2"}, + Endpoint: Endpoint{ + AuthURL: url + "/auth", + TokenURL: url + "/token", + }, + } +} + +func TestAuthCodeURL(t *testing.T) { + conf := newConf("server") + url := conf.AuthCodeURL("foo", AccessTypeOffline, ApprovalForce) + if url != "server/auth?access_type=offline&approval_prompt=force&client_id=CLIENT_ID&redirect_uri=REDIRECT_URL&response_type=code&scope=scope1+scope2&state=foo" { + t.Errorf("Auth code URL doesn't match the expected, found: %v", url) + } +} + +func TestAuthCodeURL_CustomParam(t *testing.T) { + conf := newConf("server") + param := SetAuthURLParam("foo", "bar") + url := conf.AuthCodeURL("baz", param) + if url != "server/auth?client_id=CLIENT_ID&foo=bar&redirect_uri=REDIRECT_URL&response_type=code&scope=scope1+scope2&state=baz" { + t.Errorf("Auth code URL doesn't match the expected, found: %v", url) + } +} + +func TestAuthCodeURL_Optional(t *testing.T) { + conf := &Config{ + ClientID: "CLIENT_ID", + Endpoint: Endpoint{ + AuthURL: "/auth-url", + TokenURL: "/token-url", + }, + } + url := conf.AuthCodeURL("") + if url != "/auth-url?client_id=CLIENT_ID&response_type=code" { + t.Fatalf("Auth code URL doesn't match the expected, found: %v", url) + } +} + +func TestExchangeRequest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() != "/token" { + t.Errorf("Unexpected exchange request URL, %v is found.", r.URL) + } + headerAuth := r.Header.Get("Authorization") + if headerAuth != "Basic Q0xJRU5UX0lEOkNMSUVOVF9TRUNSRVQ=" { + t.Errorf("Unexpected authorization header, %v is found.", headerAuth) + } + headerContentType := r.Header.Get("Content-Type") + if headerContentType != "application/x-www-form-urlencoded" { + t.Errorf("Unexpected Content-Type header, %v is found.", headerContentType) + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("Failed reading request body: %s.", err) + } + if string(body) != "client_id=CLIENT_ID&code=exchange-code&grant_type=authorization_code&redirect_uri=REDIRECT_URL&scope=scope1+scope2" { + t.Errorf("Unexpected exchange payload, %v is found.", string(body)) + } + w.Header().Set("Content-Type", "application/x-www-form-urlencoded") + w.Write([]byte("access_token=90d64460d14870c08c81352a05dedd3465940a7c&scope=user&token_type=bearer")) + })) + defer ts.Close() + conf := newConf(ts.URL) + tok, err := conf.Exchange(NoContext, "exchange-code") + if err != nil { + t.Error(err) + } + if !tok.Valid() { + t.Fatalf("Token invalid. Got: %#v", tok) + } + if tok.AccessToken != "90d64460d14870c08c81352a05dedd3465940a7c" { + t.Errorf("Unexpected access token, %#v.", tok.AccessToken) + } + if tok.TokenType != "bearer" { + t.Errorf("Unexpected token type, %#v.", tok.TokenType) + } + scope := tok.Extra("scope") + if scope != "user" { + t.Errorf("Unexpected value for scope: %v", scope) + } +} + +func TestExchangeRequest_JSONResponse(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() != "/token" { + t.Errorf("Unexpected exchange request URL, %v is found.", r.URL) + } + headerAuth := r.Header.Get("Authorization") + if headerAuth != "Basic Q0xJRU5UX0lEOkNMSUVOVF9TRUNSRVQ=" { + t.Errorf("Unexpected authorization header, %v is found.", headerAuth) + } + headerContentType := r.Header.Get("Content-Type") + if headerContentType != "application/x-www-form-urlencoded" { + t.Errorf("Unexpected Content-Type header, %v is found.", headerContentType) + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("Failed reading request body: %s.", err) + } + if string(body) != "client_id=CLIENT_ID&code=exchange-code&grant_type=authorization_code&redirect_uri=REDIRECT_URL&scope=scope1+scope2" { + t.Errorf("Unexpected exchange payload, %v is found.", string(body)) + } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"access_token": "90d64460d14870c08c81352a05dedd3465940a7c", "scope": "user", "token_type": "bearer", "expires_in": 86400}`)) + })) + defer ts.Close() + conf := newConf(ts.URL) + tok, err := conf.Exchange(NoContext, "exchange-code") + if err != nil { + t.Error(err) + } + if !tok.Valid() { + t.Fatalf("Token invalid. Got: %#v", tok) + } + if tok.AccessToken != "90d64460d14870c08c81352a05dedd3465940a7c" { + t.Errorf("Unexpected access token, %#v.", tok.AccessToken) + } + if tok.TokenType != "bearer" { + t.Errorf("Unexpected token type, %#v.", tok.TokenType) + } + scope := tok.Extra("scope") + if scope != "user" { + t.Errorf("Unexpected value for scope: %v", scope) + } + expiresIn := tok.Extra("expires_in") + if expiresIn != float64(86400) { + t.Errorf("Unexpected non-numeric value for expires_in: %v", expiresIn) + } +} + +func TestExtraValueRetrieval(t *testing.T) { + values := url.Values{} + + kvmap := map[string]string{ + "scope": "user", "token_type": "bearer", "expires_in": "86400.92", + "server_time": "1443571905.5606415", "referer_ip": "10.0.0.1", + "etag": "\"afZYj912P4alikMz_P11982\"", "request_id": "86400", + "untrimmed": " untrimmed ", + } + + for key, value := range kvmap { + values.Set(key, value) + } + + tok := Token{ + raw: values, + } + + scope := tok.Extra("scope") + if scope != "user" { + t.Errorf("Unexpected scope %v wanted \"user\"", scope) + } + serverTime := tok.Extra("server_time") + if serverTime != 1443571905.5606415 { + t.Errorf("Unexpected non-float64 value for server_time: %v", serverTime) + } + refererIp := tok.Extra("referer_ip") + if refererIp != "10.0.0.1" { + t.Errorf("Unexpected non-string value for referer_ip: %v", refererIp) + } + expires_in := tok.Extra("expires_in") + if expires_in != 86400.92 { + t.Errorf("Unexpected value for expires_in, wanted 86400 got %v", expires_in) + } + requestId := tok.Extra("request_id") + if requestId != int64(86400) { + t.Errorf("Unexpected non-int64 value for request_id: %v", requestId) + } + untrimmed := tok.Extra("untrimmed") + if untrimmed != " untrimmed " { + t.Errorf("Unexpected value for untrimmed, got %q expected \" untrimmed \"", untrimmed) + } +} + +const day = 24 * time.Hour + +func TestExchangeRequest_JSONResponse_Expiry(t *testing.T) { + seconds := int32(day.Seconds()) + jsonNumberType := reflect.TypeOf(json.Number("0")) + for _, c := range []struct { + expires string + expect error + }{ + {fmt.Sprintf(`"expires_in": %d`, seconds), nil}, + {fmt.Sprintf(`"expires_in": "%d"`, seconds), nil}, // PayPal case + {fmt.Sprintf(`"expires": %d`, seconds), nil}, // Facebook case + {`"expires": false`, &json.UnmarshalTypeError{Value: "bool", Type: jsonNumberType}}, // wrong type + {`"expires": {}`, &json.UnmarshalTypeError{Value: "object", Type: jsonNumberType}}, // wrong type + {`"expires": "zzz"`, &strconv.NumError{Func: "ParseInt", Num: "zzz", Err: strconv.ErrSyntax}}, // wrong value + } { + testExchangeRequest_JSONResponse_expiry(t, c.expires, c.expect) + } +} + +func testExchangeRequest_JSONResponse_expiry(t *testing.T, exp string, expect error) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(fmt.Sprintf(`{"access_token": "90d", "scope": "user", "token_type": "bearer", %s}`, exp))) + })) + defer ts.Close() + conf := newConf(ts.URL) + t1 := time.Now().Add(day) + tok, err := conf.Exchange(NoContext, "exchange-code") + t2 := time.Now().Add(day) + // Do a fmt.Sprint comparison so either side can be + // nil. fmt.Sprint just stringifies them to "", and no + // non-nil expected error ever stringifies as "", so this + // isn't terribly disgusting. We do this because Go 1.4 and + // Go 1.5 return a different deep value for + // json.UnmarshalTypeError. In Go 1.5, the + // json.UnmarshalTypeError contains a new field with a new + // non-zero value. Rather than ignore it here with reflect or + // add new files and +build tags, just look at the strings. + if fmt.Sprint(err) != fmt.Sprint(expect) { + t.Errorf("Error = %v; want %v", err, expect) + } + if err != nil { + return + } + if !tok.Valid() { + t.Fatalf("Token invalid. Got: %#v", tok) + } + expiry := tok.Expiry + if expiry.Before(t1) || expiry.After(t2) { + t.Errorf("Unexpected value for Expiry: %v (shold be between %v and %v)", expiry, t1, t2) + } +} + +func TestExchangeRequest_BadResponse(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"scope": "user", "token_type": "bearer"}`)) + })) + defer ts.Close() + conf := newConf(ts.URL) + tok, err := conf.Exchange(NoContext, "code") + if err != nil { + t.Fatal(err) + } + if tok.AccessToken != "" { + t.Errorf("Unexpected access token, %#v.", tok.AccessToken) + } +} + +func TestExchangeRequest_BadResponseType(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"access_token":123, "scope": "user", "token_type": "bearer"}`)) + })) + defer ts.Close() + conf := newConf(ts.URL) + _, err := conf.Exchange(NoContext, "exchange-code") + if err == nil { + t.Error("expected error from invalid access_token type") + } +} + +func TestExchangeRequest_NonBasicAuth(t *testing.T) { + tr := &mockTransport{ + rt: func(r *http.Request) (w *http.Response, err error) { + headerAuth := r.Header.Get("Authorization") + if headerAuth != "" { + t.Errorf("Unexpected authorization header, %v is found.", headerAuth) + } + return nil, errors.New("no response") + }, + } + c := &http.Client{Transport: tr} + conf := &Config{ + ClientID: "CLIENT_ID", + Endpoint: Endpoint{ + AuthURL: "https://accounts.google.com/auth", + TokenURL: "https://accounts.google.com/token", + }, + } + + ctx := context.WithValue(context.Background(), HTTPClient, c) + conf.Exchange(ctx, "code") +} + +func TestPasswordCredentialsTokenRequest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + expected := "/token" + if r.URL.String() != expected { + t.Errorf("URL = %q; want %q", r.URL, expected) + } + headerAuth := r.Header.Get("Authorization") + expected = "Basic Q0xJRU5UX0lEOkNMSUVOVF9TRUNSRVQ=" + if headerAuth != expected { + t.Errorf("Authorization header = %q; want %q", headerAuth, expected) + } + headerContentType := r.Header.Get("Content-Type") + expected = "application/x-www-form-urlencoded" + if headerContentType != expected { + t.Errorf("Content-Type header = %q; want %q", headerContentType, expected) + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("Failed reading request body: %s.", err) + } + expected = "client_id=CLIENT_ID&grant_type=password&password=password1&scope=scope1+scope2&username=user1" + if string(body) != expected { + t.Errorf("res.Body = %q; want %q", string(body), expected) + } + w.Header().Set("Content-Type", "application/x-www-form-urlencoded") + w.Write([]byte("access_token=90d64460d14870c08c81352a05dedd3465940a7c&scope=user&token_type=bearer")) + })) + defer ts.Close() + conf := newConf(ts.URL) + tok, err := conf.PasswordCredentialsToken(NoContext, "user1", "password1") + if err != nil { + t.Error(err) + } + if !tok.Valid() { + t.Fatalf("Token invalid. Got: %#v", tok) + } + expected := "90d64460d14870c08c81352a05dedd3465940a7c" + if tok.AccessToken != expected { + t.Errorf("AccessToken = %q; want %q", tok.AccessToken, expected) + } + expected = "bearer" + if tok.TokenType != expected { + t.Errorf("TokenType = %q; want %q", tok.TokenType, expected) + } +} + +func TestTokenRefreshRequest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() == "/somethingelse" { + return + } + if r.URL.String() != "/token" { + t.Errorf("Unexpected token refresh request URL, %v is found.", r.URL) + } + headerContentType := r.Header.Get("Content-Type") + if headerContentType != "application/x-www-form-urlencoded" { + t.Errorf("Unexpected Content-Type header, %v is found.", headerContentType) + } + body, _ := ioutil.ReadAll(r.Body) + if string(body) != "client_id=CLIENT_ID&grant_type=refresh_token&refresh_token=REFRESH_TOKEN" { + t.Errorf("Unexpected refresh token payload, %v is found.", string(body)) + } + })) + defer ts.Close() + conf := newConf(ts.URL) + c := conf.Client(NoContext, &Token{RefreshToken: "REFRESH_TOKEN"}) + c.Get(ts.URL + "/somethingelse") +} + +func TestFetchWithNoRefreshToken(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() == "/somethingelse" { + return + } + if r.URL.String() != "/token" { + t.Errorf("Unexpected token refresh request URL, %v is found.", r.URL) + } + headerContentType := r.Header.Get("Content-Type") + if headerContentType != "application/x-www-form-urlencoded" { + t.Errorf("Unexpected Content-Type header, %v is found.", headerContentType) + } + body, _ := ioutil.ReadAll(r.Body) + if string(body) != "client_id=CLIENT_ID&grant_type=refresh_token&refresh_token=REFRESH_TOKEN" { + t.Errorf("Unexpected refresh token payload, %v is found.", string(body)) + } + })) + defer ts.Close() + conf := newConf(ts.URL) + c := conf.Client(NoContext, nil) + _, err := c.Get(ts.URL + "/somethingelse") + if err == nil { + t.Errorf("Fetch should return an error if no refresh token is set") + } +} + +func TestRefreshToken_RefreshTokenReplacement(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"access_token":"ACCESS TOKEN", "scope": "user", "token_type": "bearer", "refresh_token": "NEW REFRESH TOKEN"}`)) + return + })) + defer ts.Close() + conf := newConf(ts.URL) + tkr := tokenRefresher{ + conf: conf, + ctx: NoContext, + refreshToken: "OLD REFRESH TOKEN", + } + tk, err := tkr.Token() + if err != nil { + t.Errorf("Unexpected refreshToken error returned: %v", err) + return + } + if tk.RefreshToken != tkr.refreshToken { + t.Errorf("tokenRefresher.refresh_token = %s; want %s", tkr.refreshToken, tk.RefreshToken) + } +} + +func TestConfigClientWithToken(t *testing.T) { + tok := &Token{ + AccessToken: "abc123", + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("Authorization"), fmt.Sprintf("Bearer %s", tok.AccessToken); got != want { + t.Errorf("Authorization header = %q; want %q", got, want) + } + return + })) + defer ts.Close() + conf := newConf(ts.URL) + + c := conf.Client(NoContext, tok) + req, err := http.NewRequest("GET", ts.URL, nil) + if err != nil { + t.Error(err) + } + _, err = c.Do(req) + if err != nil { + t.Error(err) + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/odnoklassniki/odnoklassniki.go b/Godeps/_workspace/src/golang.org/x/oauth2/odnoklassniki/odnoklassniki.go new file mode 100644 index 0000000000..c0d093ccc5 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/odnoklassniki/odnoklassniki.go @@ -0,0 +1,16 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package odnoklassniki provides constants for using OAuth2 to access Odnoklassniki. +package odnoklassniki // import "golang.org/x/oauth2/odnoklassniki" + +import ( + "golang.org/x/oauth2" +) + +// Endpoint is Odnoklassniki's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://www.odnoklassniki.ru/oauth/authorize", + TokenURL: "https://api.odnoklassniki.ru/oauth/token.do", +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/paypal/paypal.go b/Godeps/_workspace/src/golang.org/x/oauth2/paypal/paypal.go new file mode 100644 index 0000000000..2e713c53c8 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/paypal/paypal.go @@ -0,0 +1,22 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package paypal provides constants for using OAuth2 to access PayPal. +package paypal // import "golang.org/x/oauth2/paypal" + +import ( + "golang.org/x/oauth2" +) + +// Endpoint is PayPal's OAuth 2.0 endpoint in live (production) environment. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://www.paypal.com/webapps/auth/protocol/openidconnect/v1/authorize", + TokenURL: "https://api.paypal.com/v1/identity/openidconnect/tokenservice", +} + +// SandboxEndpoint is PayPal's OAuth 2.0 endpoint in sandbox (testing) environment. +var SandboxEndpoint = oauth2.Endpoint{ + AuthURL: "https://www.sandbox.paypal.com/webapps/auth/protocol/openidconnect/v1/authorize", + TokenURL: "https://api.sandbox.paypal.com/v1/identity/openidconnect/tokenservice", +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/token.go b/Godeps/_workspace/src/golang.org/x/oauth2/token.go new file mode 100644 index 0000000000..7a3167f15b --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/token.go @@ -0,0 +1,158 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package oauth2 + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2/internal" +) + +// expiryDelta determines how earlier a token should be considered +// expired than its actual expiration time. It is used to avoid late +// expirations due to client-server time mismatches. +const expiryDelta = 10 * time.Second + +// Token represents the crendentials used to authorize +// the requests to access protected resources on the OAuth 2.0 +// provider's backend. +// +// Most users of this package should not access fields of Token +// directly. They're exported mostly for use by related packages +// implementing derivative OAuth2 flows. +type Token struct { + // AccessToken is the token that authorizes and authenticates + // the requests. + AccessToken string `json:"access_token"` + + // TokenType is the type of token. + // The Type method returns either this or "Bearer", the default. + TokenType string `json:"token_type,omitempty"` + + // RefreshToken is a token that's used by the application + // (as opposed to the user) to refresh the access token + // if it expires. + RefreshToken string `json:"refresh_token,omitempty"` + + // Expiry is the optional expiration time of the access token. + // + // If zero, TokenSource implementations will reuse the same + // token forever and RefreshToken or equivalent + // mechanisms for that TokenSource will not be used. + Expiry time.Time `json:"expiry,omitempty"` + + // raw optionally contains extra metadata from the server + // when updating a token. + raw interface{} +} + +// Type returns t.TokenType if non-empty, else "Bearer". +func (t *Token) Type() string { + if strings.EqualFold(t.TokenType, "bearer") { + return "Bearer" + } + if strings.EqualFold(t.TokenType, "mac") { + return "MAC" + } + if strings.EqualFold(t.TokenType, "basic") { + return "Basic" + } + if t.TokenType != "" { + return t.TokenType + } + return "Bearer" +} + +// SetAuthHeader sets the Authorization header to r using the access +// token in t. +// +// This method is unnecessary when using Transport or an HTTP Client +// returned by this package. +func (t *Token) SetAuthHeader(r *http.Request) { + r.Header.Set("Authorization", t.Type()+" "+t.AccessToken) +} + +// WithExtra returns a new Token that's a clone of t, but using the +// provided raw extra map. This is only intended for use by packages +// implementing derivative OAuth2 flows. +func (t *Token) WithExtra(extra interface{}) *Token { + t2 := new(Token) + *t2 = *t + t2.raw = extra + return t2 +} + +// Extra returns an extra field. +// Extra fields are key-value pairs returned by the server as a +// part of the token retrieval response. +func (t *Token) Extra(key string) interface{} { + if raw, ok := t.raw.(map[string]interface{}); ok { + return raw[key] + } + + vals, ok := t.raw.(url.Values) + if !ok { + return nil + } + + v := vals.Get(key) + switch s := strings.TrimSpace(v); strings.Count(s, ".") { + case 0: // Contains no "."; try to parse as int + if i, err := strconv.ParseInt(s, 10, 64); err == nil { + return i + } + case 1: // Contains a single "."; try to parse as float + if f, err := strconv.ParseFloat(s, 64); err == nil { + return f + } + } + + return v +} + +// expired reports whether the token is expired. +// t must be non-nil. +func (t *Token) expired() bool { + if t.Expiry.IsZero() { + return false + } + return t.Expiry.Add(-expiryDelta).Before(time.Now()) +} + +// Valid reports whether t is non-nil, has an AccessToken, and is not expired. +func (t *Token) Valid() bool { + return t != nil && t.AccessToken != "" && !t.expired() +} + +// tokenFromInternal maps an *internal.Token struct into +// a *Token struct. +func tokenFromInternal(t *internal.Token) *Token { + if t == nil { + return nil + } + return &Token{ + AccessToken: t.AccessToken, + TokenType: t.TokenType, + RefreshToken: t.RefreshToken, + Expiry: t.Expiry, + raw: t.Raw, + } +} + +// retrieveToken takes a *Config and uses that to retrieve an *internal.Token. +// This token is then mapped from *internal.Token into an *oauth2.Token which is returned along +// with an error.. +func retrieveToken(ctx context.Context, c *Config, v url.Values) (*Token, error) { + tk, err := internal.RetrieveToken(ctx, c.ClientID, c.ClientSecret, c.Endpoint.TokenURL, v) + if err != nil { + return nil, err + } + return tokenFromInternal(tk), nil +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/token_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/token_test.go new file mode 100644 index 0000000000..80db83c29c --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/token_test.go @@ -0,0 +1,72 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package oauth2 + +import ( + "testing" + "time" +) + +func TestTokenExtra(t *testing.T) { + type testCase struct { + key string + val interface{} + want interface{} + } + const key = "extra-key" + cases := []testCase{ + {key: key, val: "abc", want: "abc"}, + {key: key, val: 123, want: 123}, + {key: key, val: "", want: ""}, + {key: "other-key", val: "def", want: nil}, + } + for _, tc := range cases { + extra := make(map[string]interface{}) + extra[tc.key] = tc.val + tok := &Token{raw: extra} + if got, want := tok.Extra(key), tc.want; got != want { + t.Errorf("Extra(%q) = %q; want %q", key, got, want) + } + } +} + +func TestTokenExpiry(t *testing.T) { + now := time.Now() + cases := []struct { + name string + tok *Token + want bool + }{ + {name: "12 seconds", tok: &Token{Expiry: now.Add(12 * time.Second)}, want: false}, + {name: "10 seconds", tok: &Token{Expiry: now.Add(expiryDelta)}, want: true}, + {name: "-1 hour", tok: &Token{Expiry: now.Add(-1 * time.Hour)}, want: true}, + } + for _, tc := range cases { + if got, want := tc.tok.expired(), tc.want; got != want { + t.Errorf("expired (%q) = %v; want %v", tc.name, got, want) + } + } +} + +func TestTokenTypeMethod(t *testing.T) { + cases := []struct { + name string + tok *Token + want string + }{ + {name: "bearer-mixed_case", tok: &Token{TokenType: "beAREr"}, want: "Bearer"}, + {name: "default-bearer", tok: &Token{}, want: "Bearer"}, + {name: "basic", tok: &Token{TokenType: "basic"}, want: "Basic"}, + {name: "basic-capitalized", tok: &Token{TokenType: "Basic"}, want: "Basic"}, + {name: "mac", tok: &Token{TokenType: "mac"}, want: "MAC"}, + {name: "mac-caps", tok: &Token{TokenType: "MAC"}, want: "MAC"}, + {name: "mac-mixed_case", tok: &Token{TokenType: "mAc"}, want: "MAC"}, + } + for _, tc := range cases { + if got, want := tc.tok.Type(), tc.want; got != want { + t.Errorf("TokenType(%q) = %v; want %v", tc.name, got, want) + } + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/transport.go b/Godeps/_workspace/src/golang.org/x/oauth2/transport.go new file mode 100644 index 0000000000..92ac7e2531 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/transport.go @@ -0,0 +1,132 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package oauth2 + +import ( + "errors" + "io" + "net/http" + "sync" +) + +// Transport is an http.RoundTripper that makes OAuth 2.0 HTTP requests, +// wrapping a base RoundTripper and adding an Authorization header +// with a token from the supplied Sources. +// +// Transport is a low-level mechanism. Most code will use the +// higher-level Config.Client method instead. +type Transport struct { + // Source supplies the token to add to outgoing requests' + // Authorization headers. + Source TokenSource + + // Base is the base RoundTripper used to make HTTP requests. + // If nil, http.DefaultTransport is used. + Base http.RoundTripper + + mu sync.Mutex // guards modReq + modReq map[*http.Request]*http.Request // original -> modified +} + +// RoundTrip authorizes and authenticates the request with an +// access token. If no token exists or token is expired, +// tries to refresh/fetch a new token. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.Source == nil { + return nil, errors.New("oauth2: Transport's Source is nil") + } + token, err := t.Source.Token() + if err != nil { + return nil, err + } + + req2 := cloneRequest(req) // per RoundTripper contract + token.SetAuthHeader(req2) + t.setModReq(req, req2) + res, err := t.base().RoundTrip(req2) + if err != nil { + t.setModReq(req, nil) + return nil, err + } + res.Body = &onEOFReader{ + rc: res.Body, + fn: func() { t.setModReq(req, nil) }, + } + return res, nil +} + +// CancelRequest cancels an in-flight request by closing its connection. +func (t *Transport) CancelRequest(req *http.Request) { + type canceler interface { + CancelRequest(*http.Request) + } + if cr, ok := t.base().(canceler); ok { + t.mu.Lock() + modReq := t.modReq[req] + delete(t.modReq, req) + t.mu.Unlock() + cr.CancelRequest(modReq) + } +} + +func (t *Transport) base() http.RoundTripper { + if t.Base != nil { + return t.Base + } + return http.DefaultTransport +} + +func (t *Transport) setModReq(orig, mod *http.Request) { + t.mu.Lock() + defer t.mu.Unlock() + if t.modReq == nil { + t.modReq = make(map[*http.Request]*http.Request) + } + if mod == nil { + delete(t.modReq, orig) + } else { + t.modReq[orig] = mod + } +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + return r2 +} + +type onEOFReader struct { + rc io.ReadCloser + fn func() +} + +func (r *onEOFReader) Read(p []byte) (n int, err error) { + n, err = r.rc.Read(p) + if err == io.EOF { + r.runFunc() + } + return +} + +func (r *onEOFReader) Close() error { + err := r.rc.Close() + r.runFunc() + return err +} + +func (r *onEOFReader) runFunc() { + if fn := r.fn; fn != nil { + fn() + r.fn = nil + } +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/transport_test.go b/Godeps/_workspace/src/golang.org/x/oauth2/transport_test.go new file mode 100644 index 0000000000..35cb25ed56 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/transport_test.go @@ -0,0 +1,86 @@ +package oauth2 + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +type tokenSource struct{ token *Token } + +func (t *tokenSource) Token() (*Token, error) { + return t.token, nil +} + +func TestTransportTokenSource(t *testing.T) { + ts := &tokenSource{ + token: &Token{ + AccessToken: "abc", + }, + } + tr := &Transport{ + Source: ts, + } + server := newMockServer(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer abc" { + t.Errorf("Transport doesn't set the Authorization header from the fetched token") + } + }) + defer server.Close() + client := http.Client{Transport: tr} + client.Get(server.URL) +} + +// Test for case-sensitive token types, per https://github.com/golang/oauth2/issues/113 +func TestTransportTokenSourceTypes(t *testing.T) { + const val = "abc" + tests := []struct { + key string + val string + want string + }{ + {key: "bearer", val: val, want: "Bearer abc"}, + {key: "mac", val: val, want: "MAC abc"}, + {key: "basic", val: val, want: "Basic abc"}, + } + for _, tc := range tests { + ts := &tokenSource{ + token: &Token{ + AccessToken: tc.val, + TokenType: tc.key, + }, + } + tr := &Transport{ + Source: ts, + } + server := newMockServer(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("Authorization"), tc.want; got != want { + t.Errorf("Authorization header (%q) = %q; want %q", val, got, want) + } + }) + defer server.Close() + client := http.Client{Transport: tr} + client.Get(server.URL) + } +} + +func TestTokenValidNoAccessToken(t *testing.T) { + token := &Token{} + if token.Valid() { + t.Errorf("Token should not be valid with no access token") + } +} + +func TestExpiredWithExpiry(t *testing.T) { + token := &Token{ + Expiry: time.Now().Add(-5 * time.Hour), + } + if token.Valid() { + t.Errorf("Token should not be valid if it expired in the past") + } +} + +func newMockServer(handler func(w http.ResponseWriter, r *http.Request)) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(handler)) +} diff --git a/Godeps/_workspace/src/golang.org/x/oauth2/vk/vk.go b/Godeps/_workspace/src/golang.org/x/oauth2/vk/vk.go new file mode 100644 index 0000000000..bd8e159488 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/oauth2/vk/vk.go @@ -0,0 +1,16 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package vk provides constants for using OAuth2 to access VK.com. +package vk // import "golang.org/x/oauth2/vk" + +import ( + "golang.org/x/oauth2" +) + +// Endpoint is VK's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://oauth.vk.com/authorize", + TokenURL: "https://oauth.vk.com/access_token", +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/.travis.yml b/Godeps/_workspace/src/google.golang.org/cloud/.travis.yml new file mode 100644 index 0000000000..86e3423261 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/.travis.yml @@ -0,0 +1,11 @@ +sudo: false +language: go +go: +- 1.4 +- tip +install: +- go get -v google.golang.org/cloud/... +script: +- openssl aes-256-cbc -K $encrypted_912ff8fa81ad_key -iv $encrypted_912ff8fa81ad_iv -in key.json.enc -out key.json -d +- GCLOUD_TESTS_GOLANG_PROJECT_ID="dulcet-port-762" GCLOUD_TESTS_GOLANG_KEY="$(pwd)/key.json" + go test -v -tags=integration google.golang.org/cloud/... diff --git a/Godeps/_workspace/src/google.golang.org/cloud/AUTHORS b/Godeps/_workspace/src/google.golang.org/cloud/AUTHORS new file mode 100644 index 0000000000..3da443dc9f --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/AUTHORS @@ -0,0 +1,12 @@ +# This is the official list of cloud authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. + +Google Inc. +Palm Stone Games, Inc. +Péter SzilÑgyi +Tyler Treat diff --git a/Godeps/_workspace/src/google.golang.org/cloud/CONTRIBUTING.md b/Godeps/_workspace/src/google.golang.org/cloud/CONTRIBUTING.md new file mode 100644 index 0000000000..9a1cab2878 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/CONTRIBUTING.md @@ -0,0 +1,114 @@ +# Contributing + +1. Sign one of the contributor license agreements below. +1. `go get golang.org/x/review/git-codereview` to install the code reviewing tool. +1. Get the cloud package by running `go get -d google.golang.org/cloud`. + 1. If you have already checked out the source, make sure that the remote git + origin is https://code.googlesource.com/gocloud: + + git remote set-url origin https://code.googlesource.com/gocloud +1. Make changes and create a change by running `git codereview change `, +provide a command message, and use `git codereview mail` to create a Gerrit CL. +1. Keep amending to the change and mail as your recieve feedback. + +## Integration Tests + +Additional to the unit tests, you may run the integration test suite. + +To run the integrations tests, creating and configuration of a project in the +Google Developers Console is required. Once you create a project, set the +following environment variables to be able to run the against the actual APIs. + +- **GCLOUD_TESTS_GOLANG_PROJECT_ID**: Developers Console project's ID (e.g. bamboo-shift-455) +- **GCLOUD_TESTS_GOLANG_KEY**: The path to the JSON key file. + +Create a storage bucket with the same name as the project id set in **GCLOUD_TESTS_GOLANG_PROJECT_ID**. +The storage integration test will create and delete some objects in this bucket. + +Install the [gcloud command-line tool][gcloudcli] to your machine and use it +to create the indexes used in the datastore integration tests with indexes +found in `datastore/testdata/index.yaml`: + +From the project's root directory: + +``` sh +# Install the app component +$ gcloud components update app + +# Set the default project in your env +$ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID + +# Authenticate the gcloud tool with your account +$ gcloud auth login + +# Create the indexes +$ gcloud preview datastore create-indexes datastore/testdata/index.yaml + +``` + +You can run the integration tests by running: + +``` sh +$ go test -v -tags=integration google.golang.org/cloud/... +``` + +## Contributor License Agreements + +Before we can accept your pull requests you'll need to sign a Contributor +License Agreement (CLA): + +- **If you are an individual writing original source code** and **you own the +- intellectual property**, then you'll need to sign an [individual CLA][indvcla]. +- **If you work for a company that wants to allow you to contribute your work**, +then you'll need to sign a [corporate CLA][corpcla]. + +You can sign these electronically (just scroll to the bottom). After that, +we'll be able to accept your pull requests. + +## Contributor Code of Conduct + +As contributors and maintainers of this project, +and in the interest of fostering an open and welcoming community, +we pledge to respect all people who contribute through reporting issues, +posting feature requests, updating documentation, +submitting pull requests or patches, and other activities. + +We are committed to making participation in this project +a harassment-free experience for everyone, +regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, +such as physical or electronic +addresses, without explicit permission +* Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct. +By adopting this Code of Conduct, +project maintainers commit themselves to fairly and consistently +applying these principles to every aspect of managing this project. +Project maintainers who do not follow or enforce the Code of Conduct +may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by opening an issue +or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0, +available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/) + +[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/ +[indvcla]: https://developers.google.com/open-source/cla/individual +[corpcla]: https://developers.google.com/open-source/cla/corporate diff --git a/Godeps/_workspace/src/google.golang.org/cloud/CONTRIBUTORS b/Godeps/_workspace/src/google.golang.org/cloud/CONTRIBUTORS new file mode 100644 index 0000000000..475ac6a667 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/CONTRIBUTORS @@ -0,0 +1,24 @@ +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# Names should be added to this file as: +# Name + +# Keep the list alphabetically sorted. + +Andrew Gerrand +Brad Fitzpatrick +Burcu Dogan +Dave Day +David Symonds +Glenn Lewis +Johan Euphrosine +Luna Duclos +Michael McGreevy +Péter SzilÑgyi +Tyler Treat diff --git a/Godeps/_workspace/src/google.golang.org/cloud/LICENSE b/Godeps/_workspace/src/google.golang.org/cloud/LICENSE new file mode 100644 index 0000000000..a4c5efd822 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Godeps/_workspace/src/google.golang.org/cloud/README.md b/Godeps/_workspace/src/google.golang.org/cloud/README.md new file mode 100644 index 0000000000..10d3995d58 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/README.md @@ -0,0 +1,135 @@ +# Google Cloud for Go + +[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-golang.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-golang) + +**NOTE:** These packages are experimental, and may occasionally make +backwards-incompatible changes. + +**NOTE:** Github repo is a mirror of [https://code.googlesource.com/gocloud](https://code.googlesource.com/gocloud). + +Go packages for Google Cloud Platform services. Supported APIs include: + + * Google Cloud Datastore + * Google Cloud Storage + * Google Cloud Pub/Sub + * Google Cloud Container Engine + +``` go +import "google.golang.org/cloud" +``` + +Documentation and examples are available at +[https://godoc.org/google.golang.org/cloud](https://godoc.org/google.golang.org/cloud). + +## Authorization + +Authorization, throughout the package, is delegated to the godoc.org/golang.org/x/oauth2. +Refer to the [godoc documentation](https://godoc.org/golang.org/x/oauth2) +for examples on using oauth2 with the Cloud package. + +## Google Cloud Datastore + +[Google Cloud Datastore][cloud-datastore] ([docs][cloud-datastore-docs]) is a fully +managed, schemaless database for storing non-relational data. Cloud Datastore +automatically scales with your users and supports ACID transactions, high availability +of reads and writes, strong consistency for reads and ancestor queries, and eventual +consistency for all other queries. + +Follow the [activation instructions][cloud-datastore-activation] to use the Google +Cloud Datastore API with your project. + +[https://godoc.org/google.golang.org/cloud/datastore](https://godoc.org/google.golang.org/cloud/datastore) + + +```go +type Post struct { + Title string + Body string `datastore:",noindex"` + PublishedAt time.Time +} +keys := []*datastore.Key{ + datastore.NewKey(ctx, "Post", "post1", 0, nil), + datastore.NewKey(ctx, "Post", "post2", 0, nil), +} +posts := []*Post{ + {Title: "Post 1", Body: "...", PublishedAt: time.Now()}, + {Title: "Post 2", Body: "...", PublishedAt: time.Now()}, +} +if _, err := datastore.PutMulti(ctx, keys, posts); err != nil { + log.Println(err) +} +``` + +## Google Cloud Storage + +[Google Cloud Storage][cloud-storage] ([docs][cloud-storage-docs]) allows you to store +data on Google infrastructure with very high reliability, performance and availability, +and can be used to distribute large data objects to users via direct download. + +[https://godoc.org/google.golang.org/cloud/storage](https://godoc.org/google.golang.org/cloud/storage) + + +```go +// Read the object1 from bucket. +rc, err := storage.NewReader(ctx, "bucket", "object1") +if err != nil { + log.Fatal(err) +} +slurp, err := ioutil.ReadAll(rc) +rc.Close() +if err != nil { + log.Fatal(err) +} +``` + +## Google Cloud Pub/Sub (Alpha) + +> Google Cloud Pub/Sub is in **Alpha status**. As a result, it might change in +> backward-incompatible ways and is not recommended for production use. It is not +> subject to any SLA or deprecation policy. + +[Google Cloud Pub/Sub][cloud-pubsub] ([docs][cloud-pubsub-docs]) allows you to connect +your services with reliable, many-to-many, asynchronous messaging hosted on Google's +infrastructure. Cloud Pub/Sub automatically scales as you need it and provides a foundation +for building your own robust, global services. + +[https://godoc.org/google.golang.org/cloud/pubsub](https://godoc.org/google.golang.org/cloud/pubsub) + + +```go +// Publish "hello world" on topic1. +msgIDs, err := pubsub.Publish(ctx, "topic1", &pubsub.Message{ + Data: []byte("hello world"), +}) +if err != nil { + log.Println(err) +} +// Pull messages via subscription1. +msgs, err := pubsub.Pull(ctx, "subscription1", 1) +if err != nil { + log.Println(err) +} +``` + +## Contributing + +Contributions are welcome. Please, see the +[CONTRIBUTING](https://github.com/GoogleCloudPlatform/gcloud-golang/blob/master/CONTRIBUTING.md) +document for details. We're using Gerrit for our code reviews. Please don't open pull +requests against this repo, new pull requests will be automatically closed. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. +See [Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/gcloud-golang/blob/master/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. + +[cloud-datastore]: https://cloud.google.com/datastore/ +[cloud-datastore-docs]: https://cloud.google.com/datastore/docs +[cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate + +[cloud-pubsub]: https://cloud.google.com/pubsub/ +[cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs + +[cloud-storage]: https://cloud.google.com/storage/ +[cloud-storage-docs]: https://cloud.google.com/storage/docs/overview +[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/bigquery.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/bigquery.go new file mode 100644 index 0000000000..bc23488d4a --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/bigquery.go @@ -0,0 +1,147 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +// TODO(mcgreevy): support dry-run mode when creating jobs. + +import ( + "fmt" + "net/http" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +// A Source is a source of data for the Copy function. +type Source interface { + implementsSource() +} + +// A Destination is a destination of data for the Copy function. +type Destination interface { + implementsDestination() +} + +// An Option is an optional argument to Copy. +type Option interface { + implementsOption() +} + +// A ReadSource is a source of data for the Read function. +type ReadSource interface { + implementsReadSource() +} + +// A ReadOption is an optional argument to Read. +type ReadOption interface { + customizeRead(conf *pagingConf) +} + +const Scope = "https://www.googleapis.com/auth/bigquery" + +// Client may be used to perform BigQuery operations. +type Client struct { + service service + projectID string +} + +// Note: many of the methods on *Client appear in the various *_op.go source files. + +// NewClient constructs a new Client which can perform BigQuery operations. +// Operations performed via the client are billed to the specified GCP project. +// The supplied http.Client is used for making requests to the BigQuery server and must be capable of +// authenticating requests with Scope. +func NewClient(client *http.Client, projectID string) (*Client, error) { + bqService, err := newBigqueryService(client) + if err != nil { + return nil, fmt.Errorf("constructing bigquery client: %v", err) + } + + c := &Client{ + service: bqService, + projectID: projectID, + } + return c, nil +} + +// initJobProto creates and returns a bigquery Job proto. +// The proto is customized using any jobOptions in options. +// The list of Options is returned with the jobOptions removed. +func initJobProto(projectID string, options []Option) (*bq.Job, []Option) { + job := &bq.Job{} + + var other []Option + for _, opt := range options { + if o, ok := opt.(jobOption); ok { + o.customizeJob(job, projectID) + } else { + other = append(other, opt) + } + } + return job, other +} + +// Copy starts a BigQuery operation to copy data from a Source to a Destination. +func (c *Client) Copy(ctx context.Context, dst Destination, src Source, options ...Option) (*Job, error) { + switch dst := dst.(type) { + case *Table: + switch src := src.(type) { + case *GCSReference: + return c.load(ctx, dst, src, options) + case *Table: + return c.cp(ctx, dst, Tables{src}, options) + case Tables: + return c.cp(ctx, dst, src, options) + case *Query: + return c.query(ctx, dst, src, options) + } + case *GCSReference: + if src, ok := src.(*Table); ok { + return c.extract(ctx, dst, src, options) + } + } + return nil, fmt.Errorf("no Copy operation matches dst/src pair: dst: %T ; src: %T", dst, src) +} + +// Read fetches data from a ReadSource and returns the data via an Iterator. +func (c *Client) Read(ctx context.Context, src ReadSource, options ...ReadOption) (*Iterator, error) { + switch src := src.(type) { + case *Job: + return c.readQueryResults(src, options) + case *Query: + return c.executeQuery(ctx, src, options...) + case *Table: + return c.readTable(src, options) + } + return nil, fmt.Errorf("src (%T) does not support the Read operation", src) +} + +// executeQuery submits a query for execution and returns the results via an Iterator. +func (c *Client) executeQuery(ctx context.Context, q *Query, options ...ReadOption) (*Iterator, error) { + dest := &Table{} + job, err := c.Copy(ctx, dest, q, WriteTruncate) + if err != nil { + return nil, err + } + + return c.Read(ctx, job, options...) +} + +func (c *Client) Dataset(id string) *Dataset { + return &Dataset{ + id: id, + client: c, + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/copy_op.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/copy_op.go new file mode 100644 index 0000000000..ec0e45e392 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/copy_op.go @@ -0,0 +1,47 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "fmt" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +type copyOption interface { + customizeCopy(conf *bq.JobConfigurationTableCopy, projectID string) +} + +func (c *Client) cp(ctx context.Context, dst *Table, src Tables, options []Option) (*Job, error) { + job, options := initJobProto(c.projectID, options) + payload := &bq.JobConfigurationTableCopy{} + + dst.customizeCopyDst(payload, c.projectID) + src.customizeCopySrc(payload, c.projectID) + + for _, opt := range options { + o, ok := opt.(copyOption) + if !ok { + return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src) + } + o.customizeCopy(payload, c.projectID) + } + + job.Configuration = &bq.JobConfiguration{ + Copy: payload, + } + return c.service.insertJob(ctx, job, c.projectID) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/copy_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/copy_test.go new file mode 100644 index 0000000000..26e40ec3b1 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/copy_test.go @@ -0,0 +1,104 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "reflect" + "testing" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +func defaultCopyJob() *bq.Job { + return &bq.Job{ + Configuration: &bq.JobConfiguration{ + Copy: &bq.JobConfigurationTableCopy{ + DestinationTable: &bq.TableReference{ + ProjectId: "d-project-id", + DatasetId: "d-dataset-id", + TableId: "d-table-id", + }, + SourceTables: []*bq.TableReference{ + { + ProjectId: "s-project-id", + DatasetId: "s-dataset-id", + TableId: "s-table-id", + }, + }, + }, + }, + } +} + +func TestCopy(t *testing.T) { + testCases := []struct { + dst *Table + src Tables + options []Option + want *bq.Job + }{ + { + dst: &Table{ + ProjectID: "d-project-id", + DatasetID: "d-dataset-id", + TableID: "d-table-id", + }, + src: Tables{ + { + ProjectID: "s-project-id", + DatasetID: "s-dataset-id", + TableID: "s-table-id", + }, + }, + want: defaultCopyJob(), + }, + { + dst: &Table{ + ProjectID: "d-project-id", + DatasetID: "d-dataset-id", + TableID: "d-table-id", + }, + src: Tables{ + { + ProjectID: "s-project-id", + DatasetID: "s-dataset-id", + TableID: "s-table-id", + }, + }, + options: []Option{CreateNever, WriteTruncate}, + want: func() *bq.Job { + j := defaultCopyJob() + j.Configuration.Copy.CreateDisposition = "CREATE_NEVER" + j.Configuration.Copy.WriteDisposition = "WRITE_TRUNCATE" + return j + }(), + }, + } + + for _, tc := range testCases { + s := &testService{} + c := &Client{ + service: s, + } + if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil { + t.Errorf("err calling cp: %v", err) + continue + } + if !reflect.DeepEqual(s.Job, tc.want) { + t.Errorf("copying: got:\n%v\nwant:\n%v", s.Job, tc.want) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/create_table_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/create_table_test.go new file mode 100644 index 0000000000..e9a498852f --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/create_table_test.go @@ -0,0 +1,55 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "reflect" + "testing" + "time" + + "golang.org/x/net/context" +) + +type createTableRecorder struct { + conf *createTableConf + service +} + +func (rec *createTableRecorder) createTable(ctx context.Context, conf *createTableConf) error { + rec.conf = conf + return nil +} + +func TestCreateTableOptions(t *testing.T) { + s := &createTableRecorder{} + c := &Client{ + service: s, + } + exp := time.Now() + q := "query" + if _, err := c.CreateTable(context.Background(), "p", "d", "t", TableExpiration(exp), ViewQuery(q)); err != nil { + t.Fatalf("err calling CreateTable: %v", err) + } + want := createTableConf{ + projectID: "p", + datasetID: "d", + tableID: "t", + expiration: exp, + viewQuery: q, + } + if !reflect.DeepEqual(*s.conf, want) { + t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/dataset.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/dataset.go new file mode 100644 index 0000000000..3f85935a24 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/dataset.go @@ -0,0 +1,41 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import "golang.org/x/net/context" + +// Dataset is a reference to a BigQuery dataset. +type Dataset struct { + id string + client *Client +} + +// ListTables returns a list of all the tables contained in the Dataset. +func (d *Dataset) ListTables(ctx context.Context) ([]*Table, error) { + var tables []*Table + + err := getPages("", func(pageToken string) (string, error) { + ts, tok, err := d.client.service.listTables(ctx, d.client.projectID, d.id, pageToken) + if err == nil { + tables = append(tables, ts...) + } + return tok, err + }) + + if err != nil { + return nil, err + } + return tables, nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/dataset_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/dataset_test.go new file mode 100644 index 0000000000..79c9c4e04d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/dataset_test.go @@ -0,0 +1,105 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "errors" + "reflect" + "testing" + + "golang.org/x/net/context" +) + +// readServiceStub services read requests by returning data from an in-memory list of values. +type listTablesServiceStub struct { + expectedProject, expectedDataset string + values [][]*Table // contains pages of tables. + pageTokens map[string]string // maps incoming page token to returned page token. + + service +} + +func (s *listTablesServiceStub) listTables(ctx context.Context, projectID, datasetID, pageToken string) ([]*Table, string, error) { + if projectID != s.expectedProject { + return nil, "", errors.New("wrong project id") + } + if datasetID != s.expectedDataset { + return nil, "", errors.New("wrong dataset id") + } + + tables := s.values[0] + s.values = s.values[1:] + return tables, s.pageTokens[pageToken], nil +} + +func TestListTables(t *testing.T) { + t1 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t1"} + t2 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t2"} + t3 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t3"} + testCases := []struct { + data [][]*Table + pageTokens map[string]string + want []*Table + }{ + { + data: [][]*Table{{t1, t2}, {t3}}, + pageTokens: map[string]string{"": "a", "a": ""}, + want: []*Table{t1, t2, t3}, + }, + { + data: [][]*Table{{t1, t2}, {t3}}, + pageTokens: map[string]string{"": ""}, // no more pages after first one. + want: []*Table{t1, t2}, + }, + } + + for _, tc := range testCases { + c := &Client{ + service: &listTablesServiceStub{ + expectedProject: "x", + expectedDataset: "y", + values: tc.data, + pageTokens: tc.pageTokens, + }, + projectID: "x", + } + got, err := c.Dataset("y").ListTables(context.Background()) + if err != nil { + t.Errorf("err calling ListTables: %v", err) + continue + } + + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("reading: got:\n%v\nwant:\n%v", got, tc.want) + } + } +} + +func TestListTablesError(t *testing.T) { + c := &Client{ + service: &listTablesServiceStub{ + expectedProject: "x", + expectedDataset: "y", + }, + projectID: "x", + } + // Test that service read errors are propagated back to the caller. + // Passing "not y" as the dataset id will cause the service to return an error. + _, err := c.Dataset("not y").ListTables(context.Background()) + if err == nil { + // Read should not return an error; only Err should. + t.Errorf("ListTables expected: non-nil err, got: nil") + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/doc.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/doc.go new file mode 100644 index 0000000000..03f851a9db --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/doc.go @@ -0,0 +1,18 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bigquery provides a client for the BigQuery service. +// +// Note: This package is a work-in-progress. Backwards-incompatible changes should be expected. +package bigquery // import "google.golang.org/cloud/bigquery" diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/error.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/error.go new file mode 100644 index 0000000000..b2e3e3f287 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/error.go @@ -0,0 +1,42 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "fmt" + + bq "google.golang.org/api/bigquery/v2" +) + +// An Error contains detailed information about an error encountered while processing a job. +type Error struct { + // Mirrors bq.ErrorProto, but drops DebugInfo + Location, Message, Reason string +} + +func (e Error) Error() string { + return fmt.Sprintf("{Location: %q; Message: %q; Reason: %q}", e.Location, e.Message, e.Reason) +} + +func errorFromErrorProto(ep *bq.ErrorProto) *Error { + if ep == nil { + return nil + } + return &Error{ + Location: ep.Location, + Message: ep.Message, + Reason: ep.Reason, + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/extract_op.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/extract_op.go new file mode 100644 index 0000000000..63a6428f1d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/extract_op.go @@ -0,0 +1,59 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "fmt" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +type extractOption interface { + customizeExtract(conf *bq.JobConfigurationExtract, projectID string) +} + +// DisableHeader returns an Option that disables the printing of a header row in exported data. +func DisableHeader() Option { return disableHeader{} } + +type disableHeader struct{} + +func (opt disableHeader) implementsOption() {} + +func (opt disableHeader) customizeExtract(conf *bq.JobConfigurationExtract, projectID string) { + f := false + conf.PrintHeader = &f +} + +func (c *Client) extract(ctx context.Context, dst *GCSReference, src *Table, options []Option) (*Job, error) { + job, options := initJobProto(c.projectID, options) + payload := &bq.JobConfigurationExtract{} + + dst.customizeExtractDst(payload, c.projectID) + src.customizeExtractSrc(payload, c.projectID) + + for _, opt := range options { + o, ok := opt.(extractOption) + if !ok { + return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src) + } + o.customizeExtract(payload, c.projectID) + } + + job.Configuration = &bq.JobConfiguration{ + Extract: payload, + } + return c.service.insertJob(ctx, job, c.projectID) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/extract_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/extract_test.go new file mode 100644 index 0000000000..4fa647491d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/extract_test.go @@ -0,0 +1,97 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "reflect" + "testing" + + "golang.org/x/net/context" + + bq "google.golang.org/api/bigquery/v2" +) + +func defaultExtractJob() *bq.Job { + return &bq.Job{ + Configuration: &bq.JobConfiguration{ + Extract: &bq.JobConfigurationExtract{ + SourceTable: &bq.TableReference{ + ProjectId: "project-id", + DatasetId: "dataset-id", + TableId: "table-id", + }, + DestinationUris: []string{"uri"}, + }, + }, + } +} + +func TestExtract(t *testing.T) { + testCases := []struct { + dst *GCSReference + src *Table + options []Option + want *bq.Job + }{ + { + dst: defaultGCS, + src: defaultTable, + want: defaultExtractJob(), + }, + { + dst: defaultGCS, + src: defaultTable, + options: []Option{ + DisableHeader(), + }, + want: func() *bq.Job { + j := defaultExtractJob() + f := false + j.Configuration.Extract.PrintHeader = &f + return j + }(), + }, + { + dst: &GCSReference{ + uris: []string{"uri"}, + Compression: Gzip, + DestinationFormat: JSON, + FieldDelimiter: "\t", + }, + src: defaultTable, + want: func() *bq.Job { + j := defaultExtractJob() + j.Configuration.Extract.Compression = "GZIP" + j.Configuration.Extract.DestinationFormat = "NEWLINE_DELIMITED_JSON" + j.Configuration.Extract.FieldDelimiter = "\t" + return j + }(), + }, + } + + for _, tc := range testCases { + s := &testService{} + c := &Client{ + service: s, + } + if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil { + t.Errorf("err calling extract: %v", err) + continue + } + if !reflect.DeepEqual(s.Job, tc.want) { + t.Errorf("extracting: got:\n%v\nwant:\n%v", s.Job, tc.want) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/gcs.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/gcs.go new file mode 100644 index 0000000000..d48859bf06 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/gcs.go @@ -0,0 +1,112 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import bq "google.golang.org/api/bigquery/v2" + +// GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute +// an input or output to a BigQuery operation. +type GCSReference struct { + uris []string + + // FieldDelimiter is the separator for fields in a CSV file, used when loading or exporting data. + // The default is ",". + FieldDelimiter string + + // The number of rows at the top of a CSV file that BigQuery will skip when loading the data. + SkipLeadingRows int64 + + // SourceFormat is the format of the GCS data to be loaded into BigQuery. + // Allowed values are: CSV, JSON, DatastoreBackup. The default is CSV. + SourceFormat DataFormat + // Only used when loading data. + Encoding Encoding + + // Quote is the value used to quote data sections in a CSV file. + // The default quotation character is the double quote ("), which is used if both Quote and ForceZeroQuote are unset. + // To specify that no character should be interpreted as a quotation character, set ForceZeroQuote to true. + // Only used when loading data. + Quote string + ForceZeroQuote bool + + // DestinationFormat is the format to use when writing exported files. + // Allowed values are: CSV, Avro, JSON. The default is CSV. + // CSV is not supported for tables with nested or repeated fields. + DestinationFormat DataFormat + // Only used when writing data. Default is None. + Compression Compression +} + +func (gcs *GCSReference) implementsSource() {} +func (gcs *GCSReference) implementsDestination() {} + +// NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination. +// In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object. +// Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided. +// Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name. +// For more information about the treatment of wildcards and multiple URIs, +// see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple +func (c *Client) NewGCSReference(uri ...string) *GCSReference { + return &GCSReference{uris: uri} +} + +type DataFormat string + +const ( + CSV DataFormat = "CSV" + Avro DataFormat = "AVRO" + JSON DataFormat = "NEWLINE_DELIMITED_JSON" + DatastoreBackup DataFormat = "DATASTORE_BACKUP" +) + +// Encoding specifies the character encoding of data to be loaded into BigQuery. +// See https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.encoding +// for more details about how this is used. +type Encoding string + +const ( + UTF_8 Encoding = "UTF-8" + ISO_8859_1 Encoding = "ISO-8859-1" +) + +// Compression is the type of compression to apply when writing data to Google Cloud Storage. +type Compression string + +const ( + None Compression = "NONE" + Gzip Compression = "GZIP" +) + +func (gcs *GCSReference) customizeLoadSrc(conf *bq.JobConfigurationLoad, projectID string) { + conf.SourceUris = gcs.uris + conf.SkipLeadingRows = gcs.SkipLeadingRows + conf.SourceFormat = string(gcs.SourceFormat) + conf.Encoding = string(gcs.Encoding) + conf.FieldDelimiter = gcs.FieldDelimiter + + if gcs.ForceZeroQuote { + quote := "" + conf.Quote = "e + } else if gcs.Quote != "" { + conf.Quote = &gcs.Quote + } +} + +func (gcs *GCSReference) customizeExtractDst(conf *bq.JobConfigurationExtract, projectID string) { + conf.DestinationUris = gcs.uris + conf.Compression = string(gcs.Compression) + conf.DestinationFormat = string(gcs.DestinationFormat) + conf.FieldDelimiter = gcs.FieldDelimiter +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/iterator.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/iterator.go new file mode 100644 index 0000000000..ff8fdc0275 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/iterator.go @@ -0,0 +1,168 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "errors" + "fmt" + + "golang.org/x/net/context" +) + +// A pageFetcher returns a page of rows, starting from the row specified by token. +type pageFetcher interface { + fetch(ctx context.Context, c *Client, token string) (*readDataResult, error) +} + +// Iterator provides access to the result of a BigQuery lookup. +// Next must be called before the first call to Get. +type Iterator struct { + c *Client + + err error // contains any error encountered during calls to Next. + + // Once Next has been called at least once, rs contains the current + // page of data and nextToken contains the token for fetching the next + // page (empty if there is no more data to be fetched). + rs [][]Value + nextToken string + + // The remaining fields contain enough information to fetch the current + // page of data, and determine which row of data from this page is the + // current row. + + pf pageFetcher + pageToken string + + // The offset from the start of the current page to the current row. + // For a new iterator, this is -1. + offset int64 +} + +func newIterator(c *Client, pf pageFetcher) *Iterator { + return &Iterator{ + c: c, + pf: pf, + offset: -1, + } +} + +// fetchPage loads the current page of data from the server. +// The contents of rs and nextToken are replaced with the loaded data. +// If there is an error while fetching, the error is stored in it.err and false is returned. +func (it *Iterator) fetchPage(ctx context.Context) bool { + var res *readDataResult + var err error + for { + res, err = it.pf.fetch(ctx, it.c, it.pageToken) + if err != errIncompleteJob { + break + } + } + + if err != nil { + it.err = err + return false + } + + it.rs = res.rows + it.nextToken = res.pageToken + return true +} + +// getEnoughData loads new data into rs until offset no longer points beyond the end of rs. +func (it *Iterator) getEnoughData(ctx context.Context) bool { + if len(it.rs) == 0 { + // Either we have not yet fetched any pages, or we are iterating over an empty dataset. + // In the former case, we should fetch a page of data, so that we can depend on the resultant nextToken. + // In the latter case, it is harmless to fetch a page of data. + if !it.fetchPage(ctx) { + return false + } + } + + for it.offset >= int64(len(it.rs)) { + // If offset is still outside the bounds of the loaded data, + // but there are no more pages of data to fetch, then we have + // failed to satisfy the offset. + if it.nextToken == "" { + return false + } + + // offset cannot be satisfied with the currently loaded data, + // so we fetch the next page. We no longer need the existing + // cached rows, so we remove them and update the offset to be + // relative to the new page that we're about to fetch. + // NOTE: we can't just set offset to 0, because after + // marshalling/unmarshalling, it's possible for the offset to + // point arbitrarily far beyond the end of rs. + // This can happen if the server returns a different size + // results page before and after marshalling. + it.offset -= int64(len(it.rs)) + it.pageToken = it.nextToken + if !it.fetchPage(ctx) { + return false + } + } + return true +} + +// Next advances the Iterator to the next row, making that row available +// via the Get method. +// Next must be called before the first call to Get, and blocks until data is available. +// Next returns false when there are no more rows available, either because +// the end of the output was reached, or because there was an error (consult +// the Err method to determine which). +func (it *Iterator) Next(ctx context.Context) bool { + if it.err != nil { + return false + } + + // Advance offset to where we want it to be for the next call to Get. + it.offset++ + + // offset may now point beyond the end of rs, so we fetch data + // until offset is within its bounds again. If there are no more + // results available, offset will be left pointing beyond the bounds + // of rs. + // At the end of this method, rs will contain at least one element + // unless the dataset we are iterating over is empty. + return it.getEnoughData(ctx) +} + +// Err returns the last error encountered by Next, or nil for no error. +func (it *Iterator) Err() error { + return it.err +} + +// Get loads the current row into dst, which must implement ValueLoader. +func (it *Iterator) Get(dst interface{}) error { + if it.err != nil { + return fmt.Errorf("Get called on iterator in error state: %v", it.err) + } + + // If Next has been called, then offset should always index into a + // valid row in rs, as long as there is still data available. + if it.offset >= int64(len(it.rs)) || it.offset < 0 { + return errors.New("Get called without preceding successful call to Next") + } + + if dst, ok := dst.(ValueLoader); ok { + return dst.Load(it.rs[it.offset]) + } + return errors.New("Get called with unsupported argument type") +} + +// TODO(mcgreevy): Add a method to *Iterator that returns a schema which describes the data. diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/iterator_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/iterator_test.go new file mode 100644 index 0000000000..b1f23c28a7 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/iterator_test.go @@ -0,0 +1,415 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "errors" + "fmt" + "reflect" + "testing" + + "golang.org/x/net/context" +) + +type fetchResponse struct { + result *readDataResult // The result to return. + err error // The error to return. +} + +// pageFetcherStub services fetch requests by returning data from an in-memory list of values. +type pageFetcherStub struct { + fetchResponses map[string]fetchResponse + + err error +} + +func (pf *pageFetcherStub) fetch(ctx context.Context, c *Client, token string) (*readDataResult, error) { + call, ok := pf.fetchResponses[token] + if !ok { + pf.err = fmt.Errorf("Unexpected page token: %q", token) + } + return call.result, call.err +} + +func TestIterator(t *testing.T) { + fetchFailure := errors.New("fetch failure") + + testCases := []struct { + desc string + alreadyConsumed int64 // amount to advance offset before commencing reading. + fetchResponses map[string]fetchResponse + want []ValueList + wantErr error + }{ + { + desc: "Iteration over single empty page", + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{}, + }, + }, + }, + want: []ValueList{}, + }, + { + desc: "Iteration over single page", + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + }, + want: []ValueList{{1, 2}, {11, 12}}, + }, + { + desc: "Iteration over two pages", + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + want: []ValueList{{1, 2}, {11, 12}, {101, 102}, {111, 112}}, + }, + { + desc: "Server response includes empty page", + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + result: &readDataResult{ + pageToken: "b", + rows: [][]Value{}, + }, + }, + "b": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + want: []ValueList{{1, 2}, {11, 12}, {101, 102}, {111, 112}}, + }, + { + desc: "Fetch error", + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + // We returns some data from this fetch, but also an error. + // So the end result should include only data from the previous fetch. + err: fetchFailure, + result: &readDataResult{ + pageToken: "b", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + want: []ValueList{{1, 2}, {11, 12}}, + wantErr: fetchFailure, + }, + { + desc: "Skip over a single element", + alreadyConsumed: 1, + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + want: []ValueList{{11, 12}, {101, 102}, {111, 112}}, + }, + { + desc: "Skip over an entire page", + alreadyConsumed: 2, + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + want: []ValueList{{101, 102}, {111, 112}}, + }, + { + desc: "Skip beyond start of second page", + alreadyConsumed: 3, + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + want: []ValueList{{111, 112}}, + }, + { + desc: "Skip beyond all data", + alreadyConsumed: 4, + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + // In this test case, Next will return false on its first call, + // so we won't even attempt to call Get. + want: []ValueList{}, + }, + } + + for _, tc := range testCases { + pf := &pageFetcherStub{ + fetchResponses: tc.fetchResponses, + } + it := newIterator(nil, pf) + it.offset += tc.alreadyConsumed + + values, err := consumeIterator(it) + if err != nil { + t.Fatalf("%s: %v", tc.desc, err) + } + + if (len(values) != 0 || len(tc.want) != 0) && !reflect.DeepEqual(values, tc.want) { + t.Errorf("%s: values:\ngot: %v\nwant:%v", tc.desc, values, tc.want) + } + if it.Err() != tc.wantErr { + t.Errorf("%s: iterator.Err:\ngot: %v\nwant: %v", tc.desc, it.Err(), tc.wantErr) + } + } +} + +// consumeIterator reads all values from an iterator and returns them. +func consumeIterator(it *Iterator) ([]ValueList, error) { + var got []ValueList + for it.Next(context.Background()) { + var vals ValueList + if err := it.Get(&vals); err != nil { + return nil, fmt.Errorf("err calling Get: %v", err) + } else { + got = append(got, vals) + } + } + + return got, nil +} + +func TestGetBeforeNext(t *testing.T) { + // TODO: once mashalling/unmarshalling of iterators is implemented, do a similar test for unmarshalled iterators. + pf := &pageFetcherStub{ + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + }, + } + it := newIterator(nil, pf) + var vals ValueList + if err := it.Get(&vals); err == nil { + t.Errorf("Expected error calling Get before Next") + } +} + +type delayedPageFetcher struct { + pageFetcherStub + delayCount int +} + +func (pf *delayedPageFetcher) fetch(ctx context.Context, c *Client, token string) (*readDataResult, error) { + if pf.delayCount > 0 { + pf.delayCount-- + return nil, errIncompleteJob + } + return pf.pageFetcherStub.fetch(ctx, c, token) +} + +func TestIterateIncompleteJob(t *testing.T) { + want := []ValueList{{1, 2}, {11, 12}, {101, 102}, {111, 112}} + pf := pageFetcherStub{ + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "a", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + "a": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{101, 102}, {111, 112}}, + }, + }, + }, + } + dpf := &delayedPageFetcher{ + pageFetcherStub: pf, + delayCount: 1, + } + it := newIterator(nil, dpf) + + values, err := consumeIterator(it) + if err != nil { + t.Fatal(err) + } + + if (len(values) != 0 || len(want) != 0) && !reflect.DeepEqual(values, want) { + t.Errorf("values: got:\n%v\nwant:\n%v", values, want) + } + if it.Err() != nil { + t.Fatalf("iterator.Err: got:\n%v", it.Err()) + } + if dpf.delayCount != 0 { + t.Errorf("delayCount: got: %v, want: 0", dpf.delayCount) + } +} + +func TestGetDuringErrorState(t *testing.T) { + pf := &pageFetcherStub{ + fetchResponses: map[string]fetchResponse{ + "": {err: errors.New("bang")}, + }, + } + it := newIterator(nil, pf) + var vals ValueList + it.Next(context.Background()) + if it.Err() == nil { + t.Errorf("Expected error after calling Next") + } + if err := it.Get(&vals); err == nil { + t.Errorf("Expected error calling Get when iterator has a non-nil error.") + } +} + +func TestGetAfterFinished(t *testing.T) { + testCases := []struct { + alreadyConsumed int64 // amount to advance offset before commencing reading. + fetchResponses map[string]fetchResponse + want []ValueList + }{ + { + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + }, + want: []ValueList{{1, 2}, {11, 12}}, + }, + { + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{}, + }, + }, + }, + want: []ValueList{}, + }, + { + alreadyConsumed: 100, + fetchResponses: map[string]fetchResponse{ + "": { + result: &readDataResult{ + pageToken: "", + rows: [][]Value{{1, 2}, {11, 12}}, + }, + }, + }, + want: []ValueList{}, + }, + } + + for _, tc := range testCases { + pf := &pageFetcherStub{ + fetchResponses: tc.fetchResponses, + } + it := newIterator(nil, pf) + it.offset += tc.alreadyConsumed + + values, err := consumeIterator(it) + if err != nil { + t.Fatal(err) + } + + if (len(values) != 0 || len(tc.want) != 0) && !reflect.DeepEqual(values, tc.want) { + t.Errorf("values: got:\n%v\nwant:\n%v", values, tc.want) + } + if it.Err() != nil { + t.Fatalf("iterator.Err: got:\n%v\nwant:\n:nil", it.Err()) + } + // Try calling Get again. + var vals ValueList + if err := it.Get(&vals); err == nil { + t.Errorf("Expected error calling Get when there are no more values") + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/job.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/job.go new file mode 100644 index 0000000000..832f0e9cb2 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/job.go @@ -0,0 +1,124 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "errors" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +// A Job represents an operation which has been submitted to BigQuery for processing. +type Job struct { + service service + projectID string + jobID string + + isQuery bool +} + +// JobFromID creates a Job which refers to an existing BigQuery job. The job +// need not have been created by this package. For example, the job may have +// been created in the BigQuery console. +func (c *Client) JobFromID(ctx context.Context, id string) (*Job, error) { + jobType, err := c.service.getJobType(ctx, c.projectID, id) + if err != nil { + return nil, err + } + + return &Job{ + service: c.service, + projectID: c.projectID, + jobID: id, + isQuery: jobType == queryJobType, + }, nil +} + +func (j *Job) ID() string { + return j.jobID +} + +// State is one of a sequence of states that a Job progresses through as it is processed. +type State int + +const ( + Pending State = iota + Running + Done +) + +// JobStatus contains the current State of a job, and errors encountered while processing that job. +type JobStatus struct { + State State + + err error + + // All errors encountered during the running of the job. + // Not all Errors are fatal, so errors here do not necessarily mean that the job has completed or was unsuccessful. + Errors []*Error +} + +// jobOption is an Option which modifies a bq.Job proto. +// This is used for configuring values that apply to all operations, such as setting a jobReference. +type jobOption interface { + customizeJob(job *bq.Job, projectID string) +} + +type jobID string + +// JobID returns an Option that sets the job ID of a BigQuery job. +// If this Option is not used, a job ID is generated automatically. +func JobID(ID string) Option { + return jobID(ID) +} + +func (opt jobID) implementsOption() {} + +func (opt jobID) customizeJob(job *bq.Job, projectID string) { + job.JobReference = &bq.JobReference{ + JobId: string(opt), + ProjectId: projectID, + } +} + +// Done reports whether the job has completed. +// After Done returns true, the Err method will return an error if the job completed unsuccesfully. +func (s *JobStatus) Done() bool { + return s.State == Done +} + +// Err returns the error that caused the job to complete unsuccesfully (if any). +func (s *JobStatus) Err() error { + return s.err +} + +// Status returns the current status of the job. It fails if the Status could not be determined. +func (j *Job) Status(ctx context.Context) (*JobStatus, error) { + return j.service.jobStatus(ctx, j.projectID, j.jobID) +} + +func (j *Job) implementsReadSource() {} + +func (j *Job) customizeReadQuery(cursor *readQueryConf) error { + // There are mulitple kinds of jobs, but only a query job is suitable for reading. + if !j.isQuery { + return errors.New("Cannot read from a non-query job") + } + + cursor.projectID = j.projectID + cursor.jobID = j.jobID + return nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/load_op.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/load_op.go new file mode 100644 index 0000000000..46ec88cdc2 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/load_op.go @@ -0,0 +1,112 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "fmt" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +type loadOption interface { + customizeLoad(conf *bq.JobConfigurationLoad, projectID string) +} + +// DestinationSchema returns an Option that specifies the schema to use when loading data into a new table. +// A DestinationSchema Option must be supplied when loading data from Google Cloud Storage into a non-existent table. +// Caveat: DestinationSchema is not required if the data being loaded is a datastore backup. +// schema must not be nil. +func DestinationSchema(schema Schema) Option { return destSchema{Schema: schema} } + +type destSchema struct { + Schema +} + +func (opt destSchema) implementsOption() {} + +func (opt destSchema) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) { + conf.Schema = opt.asTableSchema() +} + +// MaxBadRecords returns an Option that sets the maximum number of bad records that will be ignored. +// If this maximum is exceeded, the operation will be unsuccessful. +func MaxBadRecords(n int64) Option { return maxBadRecords(n) } + +type maxBadRecords int64 + +func (opt maxBadRecords) implementsOption() {} + +func (opt maxBadRecords) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) { + conf.MaxBadRecords = int64(opt) +} + +// AllowJaggedRows returns an Option that causes missing trailing optional columns to be tolerated in CSV data. Missing values are treated as nulls. +func AllowJaggedRows() Option { return allowJaggedRows{} } + +type allowJaggedRows struct{} + +func (opt allowJaggedRows) implementsOption() {} + +func (opt allowJaggedRows) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) { + conf.AllowJaggedRows = true +} + +// AllowQuotedNewlines returns an Option that allows quoted data sections containing newlines in CSV data. +func AllowQuotedNewlines() Option { return allowQuotedNewlines{} } + +type allowQuotedNewlines struct{} + +func (opt allowQuotedNewlines) implementsOption() {} + +func (opt allowQuotedNewlines) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) { + conf.AllowQuotedNewlines = true +} + +// IgnoreUnknownValues returns an Option that causes values not matching the schema to be tolerated. +// Unknown values are ignored. For CSV this ignores extra values at the end of a line. +// For JSON this ignores named values that do not match any column name. +// If this Option is not used, records containing unknown values are treated as bad records. +// The MaxBadRecords Option can be used to customize how bad records are handled. +func IgnoreUnknownValues() Option { return ignoreUnknownValues{} } + +type ignoreUnknownValues struct{} + +func (opt ignoreUnknownValues) implementsOption() {} + +func (opt ignoreUnknownValues) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) { + conf.IgnoreUnknownValues = true +} + +func (c *Client) load(ctx context.Context, dst *Table, src *GCSReference, options []Option) (*Job, error) { + job, options := initJobProto(c.projectID, options) + payload := &bq.JobConfigurationLoad{} + + dst.customizeLoadDst(payload, c.projectID) + src.customizeLoadSrc(payload, c.projectID) + + for _, opt := range options { + o, ok := opt.(loadOption) + if !ok { + return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src) + } + o.customizeLoad(payload, c.projectID) + } + + job.Configuration = &bq.JobConfiguration{ + Load: payload, + } + return c.service.insertJob(ctx, job, c.projectID) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/load_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/load_test.go new file mode 100644 index 0000000000..df6cec0cf6 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/load_test.go @@ -0,0 +1,198 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "reflect" + "testing" + + "golang.org/x/net/context" + + bq "google.golang.org/api/bigquery/v2" +) + +func defaultLoadJob() *bq.Job { + return &bq.Job{ + Configuration: &bq.JobConfiguration{ + Load: &bq.JobConfigurationLoad{ + DestinationTable: &bq.TableReference{ + ProjectId: "project-id", + DatasetId: "dataset-id", + TableId: "table-id", + }, + SourceUris: []string{"uri"}, + }, + }, + } +} + +func stringFieldSchema() *FieldSchema { + return &FieldSchema{Name: "fieldname", Type: StringFieldType} +} + +func nestedFieldSchema() *FieldSchema { + return &FieldSchema{ + Name: "nested", + Type: RecordFieldType, + Schema: Schema{stringFieldSchema()}, + } +} + +func bqStringFieldSchema() *bq.TableFieldSchema { + return &bq.TableFieldSchema{ + Name: "fieldname", + Type: "STRING", + } +} + +func bqNestedFieldSchema() *bq.TableFieldSchema { + return &bq.TableFieldSchema{ + Name: "nested", + Type: "RECORD", + Fields: []*bq.TableFieldSchema{bqStringFieldSchema()}, + } +} + +func TestLoad(t *testing.T) { + testCases := []struct { + dst *Table + src *GCSReference + options []Option + want *bq.Job + }{ + { + dst: defaultTable, + src: defaultGCS, + want: defaultLoadJob(), + }, + { + dst: defaultTable, + src: defaultGCS, + options: []Option{ + MaxBadRecords(1), + AllowJaggedRows(), + AllowQuotedNewlines(), + IgnoreUnknownValues(), + }, + want: func() *bq.Job { + j := defaultLoadJob() + j.Configuration.Load.MaxBadRecords = 1 + j.Configuration.Load.AllowJaggedRows = true + j.Configuration.Load.AllowQuotedNewlines = true + j.Configuration.Load.IgnoreUnknownValues = true + return j + }(), + }, + { + dst: &Table{ + ProjectID: "project-id", + DatasetID: "dataset-id", + TableID: "table-id", + }, + options: []Option{CreateNever, WriteTruncate}, + src: defaultGCS, + want: func() *bq.Job { + j := defaultLoadJob() + j.Configuration.Load.CreateDisposition = "CREATE_NEVER" + j.Configuration.Load.WriteDisposition = "WRITE_TRUNCATE" + return j + }(), + }, + { + dst: &Table{ + ProjectID: "project-id", + DatasetID: "dataset-id", + TableID: "table-id", + }, + src: defaultGCS, + options: []Option{ + DestinationSchema(Schema{ + stringFieldSchema(), + nestedFieldSchema(), + }), + }, + want: func() *bq.Job { + j := defaultLoadJob() + j.Configuration.Load.Schema = &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqStringFieldSchema(), + bqNestedFieldSchema(), + }} + return j + }(), + }, + { + dst: defaultTable, + src: &GCSReference{ + uris: []string{"uri"}, + SkipLeadingRows: 1, + SourceFormat: JSON, + Encoding: UTF_8, + FieldDelimiter: "\t", + Quote: "-", + }, + want: func() *bq.Job { + j := defaultLoadJob() + j.Configuration.Load.SkipLeadingRows = 1 + j.Configuration.Load.SourceFormat = "NEWLINE_DELIMITED_JSON" + j.Configuration.Load.Encoding = "UTF-8" + j.Configuration.Load.FieldDelimiter = "\t" + hyphen := "-" + j.Configuration.Load.Quote = &hyphen + return j + }(), + }, + { + dst: defaultTable, + src: &GCSReference{ + uris: []string{"uri"}, + Quote: "", + }, + want: func() *bq.Job { + j := defaultLoadJob() + j.Configuration.Load.Quote = nil + return j + }(), + }, + { + dst: defaultTable, + src: &GCSReference{ + uris: []string{"uri"}, + Quote: "", + ForceZeroQuote: true, + }, + want: func() *bq.Job { + j := defaultLoadJob() + empty := "" + j.Configuration.Load.Quote = &empty + return j + }(), + }, + } + + for _, tc := range testCases { + s := &testService{} + c := &Client{ + service: s, + } + if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil { + t.Errorf("err calling load: %v", err) + continue + } + if !reflect.DeepEqual(s.Job, tc.want) { + t.Errorf("loading: got:\n%v\nwant:\n%v", s.Job, tc.want) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query.go new file mode 100644 index 0000000000..99807a73ec --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query.go @@ -0,0 +1,42 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import bq "google.golang.org/api/bigquery/v2" + +// Query represents a query to be executed. +type Query struct { + // The query to execute. See https://cloud.google.com/bigquery/query-reference for details. + Q string + + // DefaultProjectID and DefaultDatasetID specify the dataset to use for unqualified table names in the query. + // If DefaultProjectID is set, DefaultDatasetID must also be set. + DefaultProjectID string + DefaultDatasetID string +} + +func (q *Query) implementsSource() {} + +func (q *Query) implementsReadSource() {} + +func (q *Query) customizeQuerySrc(conf *bq.JobConfigurationQuery, projectID string) { + conf.Query = q.Q + if q.DefaultProjectID != "" || q.DefaultDatasetID != "" { + conf.DefaultDataset = &bq.DatasetReference{ + DatasetId: q.DefaultDatasetID, + ProjectId: q.DefaultProjectID, + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query_op.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query_op.go new file mode 100644 index 0000000000..af62b5b3bf --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query_op.go @@ -0,0 +1,89 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "fmt" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +type queryOption interface { + customizeQuery(conf *bq.JobConfigurationQuery, projectID string) +} + +// DisableQueryCache returns an Option that prevents results being fetched from the query cache. +// If this Option is not used, results are fetched from the cache if they are available. +// The query cache is a best-effort cache that is flushed whenever tables in the query are modified. +// Cached results are only available when TableID is unspecified in the query's destination Table. +// For more information, see https://cloud.google.com/bigquery/querying-data#querycaching +func DisableQueryCache() Option { return disableQueryCache{} } + +type disableQueryCache struct{} + +func (opt disableQueryCache) implementsOption() {} + +func (opt disableQueryCache) customizeQuery(conf *bq.JobConfigurationQuery, projectID string) { + f := false + conf.UseQueryCache = &f +} + +// JobPriority returns an Option that causes a query to be scheduled with the specified priority. +// The default priority is InteractivePriority. +// For more information, see https://cloud.google.com/bigquery/querying-data#batchqueries +func JobPriority(priority string) Option { return jobPriority(priority) } + +type jobPriority string + +func (opt jobPriority) implementsOption() {} + +func (opt jobPriority) customizeQuery(conf *bq.JobConfigurationQuery, projectID string) { + conf.Priority = string(opt) +} + +const ( + BatchPriority = "BATCH" + InteractivePriority = "INTERACTIVE" +) + +// TODO(mcgreevy): support large results. +// TODO(mcgreevy): support non-flattened results. + +func (c *Client) query(ctx context.Context, dst *Table, src *Query, options []Option) (*Job, error) { + job, options := initJobProto(c.projectID, options) + payload := &bq.JobConfigurationQuery{} + + dst.customizeQueryDst(payload, c.projectID) + src.customizeQuerySrc(payload, c.projectID) + + for _, opt := range options { + o, ok := opt.(queryOption) + if !ok { + return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src) + } + o.customizeQuery(payload, c.projectID) + } + + job.Configuration = &bq.JobConfiguration{ + Query: payload, + } + j, err := c.service.insertJob(ctx, job, c.projectID) + if err != nil { + return nil, err + } + j.isQuery = true + return j, nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query_test.go new file mode 100644 index 0000000000..119d01b2b0 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/query_test.go @@ -0,0 +1,118 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "reflect" + "testing" + + "golang.org/x/net/context" + + bq "google.golang.org/api/bigquery/v2" +) + +func defaultQueryJob() *bq.Job { + return &bq.Job{ + Configuration: &bq.JobConfiguration{ + Query: &bq.JobConfigurationQuery{ + DestinationTable: &bq.TableReference{ + ProjectId: "project-id", + DatasetId: "dataset-id", + TableId: "table-id", + }, + Query: "query string", + DefaultDataset: &bq.DatasetReference{ + ProjectId: "def-project-id", + DatasetId: "def-dataset-id", + }, + }, + }, + } +} + +func TestQuery(t *testing.T) { + testCases := []struct { + dst *Table + src *Query + options []Option + want *bq.Job + }{ + { + dst: defaultTable, + src: defaultQuery, + want: defaultQueryJob(), + }, + { + dst: defaultTable, + src: &Query{ + Q: "query string", + }, + want: func() *bq.Job { + j := defaultQueryJob() + j.Configuration.Query.DefaultDataset = nil + return j + }(), + }, + { + dst: &Table{}, + src: defaultQuery, + want: func() *bq.Job { + j := defaultQueryJob() + j.Configuration.Query.DestinationTable = nil + return j + }(), + }, + { + dst: &Table{ + ProjectID: "project-id", + DatasetID: "dataset-id", + TableID: "table-id", + }, + src: defaultQuery, + options: []Option{CreateNever, WriteTruncate}, + want: func() *bq.Job { + j := defaultQueryJob() + j.Configuration.Query.WriteDisposition = "WRITE_TRUNCATE" + j.Configuration.Query.CreateDisposition = "CREATE_NEVER" + return j + }(), + }, + { + dst: defaultTable, + src: defaultQuery, + options: []Option{DisableQueryCache()}, + want: func() *bq.Job { + j := defaultQueryJob() + f := false + j.Configuration.Query.UseQueryCache = &f + return j + }(), + }, + } + + for _, tc := range testCases { + s := &testService{} + c := &Client{ + service: s, + } + if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil { + t.Errorf("err calling query: %v", err) + continue + } + if !reflect.DeepEqual(s.Job, tc.want) { + t.Errorf("querying: got:\n%v\nwant:\n%v", s.Job, tc.want) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/read_op.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/read_op.go new file mode 100644 index 0000000000..0159b12a22 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/read_op.go @@ -0,0 +1,68 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import "golang.org/x/net/context" + +// RecordsPerRequest returns a ReadOption that sets the number of records to fetch per request when streaming data from BigQuery. +func RecordsPerRequest(n int64) ReadOption { return recordsPerRequest(n) } + +type recordsPerRequest int64 + +func (opt recordsPerRequest) customizeRead(conf *pagingConf) { + conf.recordsPerRequest = int64(opt) + conf.setRecordsPerRequest = true +} + +// StartIndex returns a ReadOption that sets the zero-based index of the row to start reading from. +func StartIndex(i uint64) ReadOption { return startIndex(i) } + +type startIndex uint64 + +func (opt startIndex) customizeRead(conf *pagingConf) { + conf.startIndex = uint64(opt) +} + +func (conf *readTableConf) fetch(ctx context.Context, c *Client, token string) (*readDataResult, error) { + return c.service.readTabledata(ctx, conf, token) +} + +func (c *Client) readTable(t *Table, options []ReadOption) (*Iterator, error) { + conf := &readTableConf{} + t.customizeReadSrc(conf) + + for _, o := range options { + o.customizeRead(&conf.paging) + } + + return newIterator(c, conf), nil +} + +func (conf *readQueryConf) fetch(ctx context.Context, c *Client, token string) (*readDataResult, error) { + return c.service.readQuery(ctx, conf, token) +} + +func (c *Client) readQueryResults(job *Job, options []ReadOption) (*Iterator, error) { + conf := &readQueryConf{} + if err := job.customizeReadQuery(conf); err != nil { + return nil, err + } + + for _, o := range options { + o.customizeRead(&conf.paging) + } + + return newIterator(c, conf), nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/read_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/read_test.go new file mode 100644 index 0000000000..a03bd3f455 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/read_test.go @@ -0,0 +1,308 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "errors" + "reflect" + "testing" + + "golang.org/x/net/context" +) + +type readTabledataArgs struct { + conf *readTableConf + tok string +} + +type readQueryArgs struct { + conf *readQueryConf + tok string +} + +// readServiceStub services read requests by returning data from an in-memory list of values. +type readServiceStub struct { + // values and pageTokens are used as sources of data to return in response to calls to readTabledata or readQuery. + values [][][]Value // contains pages / rows / columns. + pageTokens map[string]string // maps incoming page token to returned page token. + + // arguments are recorded for later inspection. + readTabledataCalls []readTabledataArgs + readQueryCalls []readQueryArgs + + service +} + +func (s *readServiceStub) readValues(tok string) *readDataResult { + result := &readDataResult{ + pageToken: s.pageTokens[tok], + rows: s.values[0], + } + s.values = s.values[1:] + + return result +} +func (s *readServiceStub) readTabledata(ctx context.Context, conf *readTableConf, token string) (*readDataResult, error) { + s.readTabledataCalls = append(s.readTabledataCalls, readTabledataArgs{conf, token}) + return s.readValues(token), nil +} + +func (s *readServiceStub) readQuery(ctx context.Context, conf *readQueryConf, token string) (*readDataResult, error) { + s.readQueryCalls = append(s.readQueryCalls, readQueryArgs{conf, token}) + return s.readValues(token), nil +} + +func TestRead(t *testing.T) { + // The data for the service stub to return is populated for each test case in the testCases for loop. + service := &readServiceStub{} + c := &Client{ + service: service, + } + + queryJob := &Job{ + projectID: "project-id", + jobID: "job-id", + service: service, + isQuery: true, + } + + for _, src := range []ReadSource{defaultTable, queryJob} { + testCases := []struct { + data [][][]Value + pageTokens map[string]string + want []ValueList + }{ + { + data: [][][]Value{{{1, 2}, {11, 12}}, {{30, 40}, {31, 41}}}, + pageTokens: map[string]string{"": "a", "a": ""}, + want: []ValueList{{1, 2}, {11, 12}, {30, 40}, {31, 41}}, + }, + { + data: [][][]Value{{{1, 2}, {11, 12}}, {{30, 40}, {31, 41}}}, + pageTokens: map[string]string{"": ""}, // no more pages after first one. + want: []ValueList{{1, 2}, {11, 12}}, + }, + } + + for _, tc := range testCases { + service.values = tc.data + service.pageTokens = tc.pageTokens + if got, ok := doRead(t, c, src); ok { + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("reading: got:\n%v\nwant:\n%v", got, tc.want) + } + } + } + } +} + +// doRead calls Read with a ReadSource. Get is repeatedly called on the Iterator returned by Read and the results are returned. +func doRead(t *testing.T, c *Client, src ReadSource) ([]ValueList, bool) { + it, err := c.Read(context.Background(), src) + if err != nil { + t.Errorf("err calling Read: %v", err) + return nil, false + } + var got []ValueList + for it.Next(context.Background()) { + var vals ValueList + if err := it.Get(&vals); err != nil { + t.Errorf("err calling Get: %v", err) + return nil, false + } else { + got = append(got, vals) + } + } + + return got, true +} + +func TestNoMoreValues(t *testing.T) { + c := &Client{ + service: &readServiceStub{ + values: [][][]Value{{{1, 2}, {11, 12}}}, + }, + } + it, err := c.Read(context.Background(), defaultTable) + if err != nil { + t.Fatalf("err calling Read: %v", err) + } + var vals ValueList + // We expect to retrieve two values and then fail on the next attempt. + if !it.Next(context.Background()) { + t.Fatalf("Next: got: false: want: true") + } + if !it.Next(context.Background()) { + t.Fatalf("Next: got: false: want: true") + } + if err := it.Get(&vals); err != nil { + t.Fatalf("Get: got: %v: want: nil", err) + } + if it.Next(context.Background()) { + t.Fatalf("Next: got: true: want: false") + } + if err := it.Get(&vals); err == nil { + t.Fatalf("Get: got: %v: want: non-nil", err) + } +} + +// delayedReadStub simulates reading results from a query that has not yet +// completed. Its readQuery method initially reports that the query job is not +// yet complete. Subsequently, it proxies the request through to another +// service stub. +type delayedReadStub struct { + numDelays int + + readServiceStub +} + +func (s *delayedReadStub) readQuery(ctx context.Context, conf *readQueryConf, token string) (*readDataResult, error) { + if s.numDelays > 0 { + s.numDelays-- + return nil, errIncompleteJob + } + return s.readServiceStub.readQuery(ctx, conf, token) +} + +// TestIncompleteJob tests that an Iterator which reads from a query job will block until the job is complete. +func TestIncompleteJob(t *testing.T) { + service := &delayedReadStub{ + numDelays: 2, + readServiceStub: readServiceStub{ + values: [][][]Value{{{1, 2}}}, + }, + } + c := &Client{service: service} + queryJob := &Job{ + projectID: "project-id", + jobID: "job-id", + service: service, + isQuery: true, + } + it, err := c.Read(context.Background(), queryJob) + if err != nil { + t.Fatalf("err calling Read: %v", err) + } + var got ValueList + want := ValueList{1, 2} + if !it.Next(context.Background()) { + t.Fatalf("Next: got: false: want: true") + } + if err := it.Get(&got); err != nil { + t.Fatalf("Error calling Get: %v", err) + } + if service.numDelays != 0 { + t.Errorf("remaining numDelays : got: %v want:0", service.numDelays) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("reading: got:\n%v\nwant:\n%v", got, want) + } +} + +type errorReadService struct { + service +} + +func (s *errorReadService) readTabledata(ctx context.Context, conf *readTableConf, token string) (*readDataResult, error) { + return nil, errors.New("bang!") +} + +func TestReadError(t *testing.T) { + // test that service read errors are propagated back to the caller. + c := &Client{service: &errorReadService{}} + it, err := c.Read(context.Background(), defaultTable) + if err != nil { + // Read should not return an error; only Err should. + t.Fatalf("err calling Read: %v", err) + } + if it.Next(context.Background()) { + t.Fatalf("Next: got: true: want: false") + } + if err := it.Err(); err.Error() != "bang!" { + t.Fatalf("Get: got: %v: want: bang!", err) + } +} + +func TestReadTabledataOptions(t *testing.T) { + // test that read options are propagated. + s := &readServiceStub{ + values: [][][]Value{{{1, 2}}}, + } + c := &Client{service: s} + it, err := c.Read(context.Background(), defaultTable, RecordsPerRequest(5)) + + if err != nil { + t.Fatalf("err calling Read: %v", err) + } + if !it.Next(context.Background()) { + t.Fatalf("Next: got: false: want: true") + } + + want := []readTabledataArgs{{ + conf: &readTableConf{ + projectID: "project-id", + datasetID: "dataset-id", + tableID: "table-id", + paging: pagingConf{ + recordsPerRequest: 5, + setRecordsPerRequest: true, + }, + }, + tok: "", + }} + + if !reflect.DeepEqual(s.readTabledataCalls, want) { + t.Errorf("reading: got:\n%v\nwant:\n%v", s.readTabledataCalls, want) + } +} + +func TestReadQueryOptions(t *testing.T) { + // test that read options are propagated. + s := &readServiceStub{ + values: [][][]Value{{{1, 2}}}, + } + c := &Client{service: s} + + queryJob := &Job{ + projectID: "project-id", + jobID: "job-id", + service: s, + isQuery: true, + } + it, err := c.Read(context.Background(), queryJob, RecordsPerRequest(5)) + + if err != nil { + t.Fatalf("err calling Read: %v", err) + } + if !it.Next(context.Background()) { + t.Fatalf("Next: got: false: want: true") + } + + want := []readQueryArgs{{ + conf: &readQueryConf{ + projectID: "project-id", + jobID: "job-id", + paging: pagingConf{ + recordsPerRequest: 5, + setRecordsPerRequest: true, + }, + }, + tok: "", + }} + + if !reflect.DeepEqual(s.readQueryCalls, want) { + t.Errorf("reading: got:\n%v\nwant:\n%v", s.readQueryCalls, want) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/schema.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/schema.go new file mode 100644 index 0000000000..abc159a53c --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/schema.go @@ -0,0 +1,106 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import bq "google.golang.org/api/bigquery/v2" + +// Schema describes the fields in a table or query result. +type Schema []*FieldSchema + +// TODO(mcgreevy): add a function to generate a schema from a struct. + +type FieldSchema struct { + // The field name. + // Must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_), + // and must start with a letter or underscore. + // The maximum length is 128 characters. + Name string + + // A description of the field. The maximum length is 16,384 characters. + Description string + + // Whether the field may contain multiple values. + Repeated bool + // Whether the field is required. Ignored if Repeated is true. + Required bool + + // The field data type. If Type is Record, then this field contains a nested schema, + // which is described by Schema. + Type FieldType + // Describes the nested schema if Type is set to Record. + Schema Schema +} + +func (fs *FieldSchema) asTableFieldSchema() *bq.TableFieldSchema { + tfs := &bq.TableFieldSchema{ + Description: fs.Description, + Name: fs.Name, + Type: string(fs.Type), + } + + if fs.Repeated { + tfs.Mode = "REPEATED" + } else if fs.Required { + tfs.Mode = "REQUIRED" + } // else leave as default, which is interpreted as NULLABLE. + + for _, f := range fs.Schema { + tfs.Fields = append(tfs.Fields, f.asTableFieldSchema()) + } + + return tfs +} + +func (s Schema) asTableSchema() *bq.TableSchema { + var fields []*bq.TableFieldSchema + for _, f := range s { + fields = append(fields, f.asTableFieldSchema()) + } + return &bq.TableSchema{Fields: fields} +} + +func convertTableFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema { + fs := &FieldSchema{ + Description: tfs.Description, + Name: tfs.Name, + Repeated: tfs.Mode == "REPEATED", + Required: tfs.Mode == "REQUIRED", + Type: FieldType(tfs.Type), + } + + for _, f := range tfs.Fields { + fs.Schema = append(fs.Schema, convertTableFieldSchema(f)) + } + return fs +} + +func convertTableSchema(ts *bq.TableSchema) Schema { + var s Schema + for _, f := range ts.Fields { + s = append(s, convertTableFieldSchema(f)) + } + return s +} + +type FieldType string + +const ( + StringFieldType FieldType = "STRING" + IntegerFieldType FieldType = "INTEGER" + FloatFieldType FieldType = "FLOAT" + BooleanFieldType FieldType = "BOOLEAN" + TimestampFieldType FieldType = "TIMESTAMP" + RecordFieldType FieldType = "RECORD" +) diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/schema_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/schema_test.go new file mode 100644 index 0000000000..4ff147fe0b --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/schema_test.go @@ -0,0 +1,168 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "reflect" + "testing" + + bq "google.golang.org/api/bigquery/v2" +) + +func bqTableFieldSchema(desc, name, typ, mode string) *bq.TableFieldSchema { + return &bq.TableFieldSchema{ + Description: desc, + Name: name, + Mode: mode, + Type: typ, + } +} + +func fieldSchema(desc, name, typ string, repeated, required bool) *FieldSchema { + return &FieldSchema{ + Description: desc, + Name: name, + Repeated: repeated, + Required: required, + Type: FieldType(typ), + } +} + +func TestSchemaConversion(t *testing.T) { + testCases := []struct { + schema Schema + bqSchema *bq.TableSchema + }{ + { + // required + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"), + }, + }, + schema: Schema{ + fieldSchema("desc", "name", "STRING", false, true), + }, + }, + { + // repeated + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("desc", "name", "STRING", "REPEATED"), + }, + }, + schema: Schema{ + fieldSchema("desc", "name", "STRING", true, false), + }, + }, + { + // nullable, string + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("desc", "name", "STRING", ""), + }, + }, + schema: Schema{ + fieldSchema("desc", "name", "STRING", false, false), + }, + }, + { + // integer + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("desc", "name", "INTEGER", ""), + }, + }, + schema: Schema{ + fieldSchema("desc", "name", "INTEGER", false, false), + }, + }, + { + // float + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("desc", "name", "FLOAT", ""), + }, + }, + schema: Schema{ + fieldSchema("desc", "name", "FLOAT", false, false), + }, + }, + { + // boolean + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("desc", "name", "BOOLEAN", ""), + }, + }, + schema: Schema{ + fieldSchema("desc", "name", "BOOLEAN", false, false), + }, + }, + { + // timestamp + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("desc", "name", "TIMESTAMP", ""), + }, + }, + schema: Schema{ + fieldSchema("desc", "name", "TIMESTAMP", false, false), + }, + }, + { + // nested + bqSchema: &bq.TableSchema{ + Fields: []*bq.TableFieldSchema{ + &bq.TableFieldSchema{ + Description: "An outer schema wrapping a nested schema", + Name: "outer", + Mode: "REQUIRED", + Type: "RECORD", + Fields: []*bq.TableFieldSchema{ + bqTableFieldSchema("inner field", "inner", "STRING", ""), + }, + }, + }, + }, + schema: Schema{ + &FieldSchema{ + Description: "An outer schema wrapping a nested schema", + Name: "outer", + Required: true, + Type: "RECORD", + Schema: []*FieldSchema{ + &FieldSchema{ + Description: "inner field", + Name: "inner", + Type: "STRING", + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + bqSchema := tc.schema.asTableSchema() + if !reflect.DeepEqual(bqSchema, tc.bqSchema) { + t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v", bqSchema, tc.bqSchema) + } + schema := convertTableSchema(tc.bqSchema) + if !reflect.DeepEqual(schema, tc.schema) { + t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/service.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/service.go new file mode 100644 index 0000000000..b57f84e82d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/service.go @@ -0,0 +1,403 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "errors" + "fmt" + "net/http" + "sync" + "time" + + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +// service provides an internal abstraction to isolate the generated +// BigQuery API; most of this package uses this interface instead. +// The single implementation, *bigqueryService, contains all the knowledge +// of the generated BigQuery API. +type service interface { + // Jobs + insertJob(ctx context.Context, job *bq.Job, projectId string) (*Job, error) + getJobType(ctx context.Context, projectId, jobID string) (jobType, error) + jobStatus(ctx context.Context, projectId, jobID string) (*JobStatus, error) + + // Queries + + // readQuery reads data resulting from a query job. If the job is not + // yet complete, an errIncompleteJob is returned. readQuery may be + // called repeatedly to wait for results indefinitely. + readQuery(ctx context.Context, conf *readQueryConf, pageToken string) (*readDataResult, error) + + readTabledata(ctx context.Context, conf *readTableConf, pageToken string) (*readDataResult, error) + + // Tables + createTable(ctx context.Context, conf *createTableConf) error + getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error) + deleteTable(ctx context.Context, projectID, datasetID, tableID string) error + listTables(ctx context.Context, projectID, datasetID, pageToken string) ([]*Table, string, error) + patchTable(ctx context.Context, projectID, datasetID, tableID string, conf *patchTableConf) (*TableMetadata, error) +} + +type bigqueryService struct { + s *bq.Service +} + +func newBigqueryService(client *http.Client) (*bigqueryService, error) { + s, err := bq.New(client) + if err != nil { + return nil, fmt.Errorf("constructing bigquery client: %v", err) + } + + return &bigqueryService{s: s}, nil +} + +// getPages calls the supplied getPage function repeatedly until there are no pages left to get. +// token is the token of the initial page to start from. Use an empty string to start from the beginning. +func getPages(token string, getPage func(token string) (nextToken string, err error)) error { + for { + var err error + token, err = getPage(token) + if err != nil { + return err + } + if token == "" { + return nil + } + } +} + +func (s *bigqueryService) insertJob(ctx context.Context, job *bq.Job, projectID string) (*Job, error) { + res, err := s.s.Jobs.Insert(projectID, job).Context(ctx).Do() + if err != nil { + return nil, err + } + return &Job{service: s, projectID: projectID, jobID: res.JobReference.JobId}, nil +} + +type pagingConf struct { + recordsPerRequest int64 + setRecordsPerRequest bool + + startIndex uint64 +} + +type readTableConf struct { + projectID, datasetID, tableID string + paging pagingConf + schema Schema // lazily initialized when the first page of data is fetched. +} + +type readDataResult struct { + pageToken string + rows [][]Value + totalRows uint64 + schema Schema +} + +type readQueryConf struct { + projectID, jobID string + paging pagingConf +} + +func (s *bigqueryService) readTabledata(ctx context.Context, conf *readTableConf, pageToken string) (*readDataResult, error) { + // Prepare request to fetch one page of table data. + req := s.s.Tabledata.List(conf.projectID, conf.datasetID, conf.tableID) + + if pageToken != "" { + req.PageToken(pageToken) + } else { + req.StartIndex(conf.paging.startIndex) + } + + if conf.paging.setRecordsPerRequest { + req.MaxResults(conf.paging.recordsPerRequest) + } + + // Fetch the table schema in the background, if necessary. + var schemaErr error + var schemaFetch sync.WaitGroup + if conf.schema == nil { + schemaFetch.Add(1) + go func() { + defer schemaFetch.Done() + var t *bq.Table + t, schemaErr = s.s.Tables.Get(conf.projectID, conf.datasetID, conf.tableID). + Fields("schema"). + Context(ctx). + Do() + if schemaErr == nil && t.Schema != nil { + conf.schema = convertTableSchema(t.Schema) + } + }() + } + + res, err := req.Context(ctx).Do() + if err != nil { + return nil, err + } + + schemaFetch.Wait() + if schemaErr != nil { + return nil, schemaErr + } + + result := &readDataResult{ + pageToken: res.PageToken, + totalRows: uint64(res.TotalRows), + schema: conf.schema, + } + result.rows, err = convertRows(res.Rows, conf.schema) + if err != nil { + return nil, err + } + return result, nil +} + +var errIncompleteJob = errors.New("internal error: query results not available because job is not complete") + +// getQueryResultsTimeout controls the maximum duration of a request to the +// BigQuery GetQueryResults endpoint. Setting a long timeout here does not +// cause increased overall latency, as results are returned as soon as they are +// available. +const getQueryResultsTimeout = time.Minute + +func (s *bigqueryService) readQuery(ctx context.Context, conf *readQueryConf, pageToken string) (*readDataResult, error) { + req := s.s.Jobs.GetQueryResults(conf.projectID, conf.jobID). + TimeoutMs(getQueryResultsTimeout.Nanoseconds() / 1e6) + + if pageToken != "" { + req.PageToken(pageToken) + } else { + req.StartIndex(conf.paging.startIndex) + } + + if conf.paging.setRecordsPerRequest { + req.MaxResults(conf.paging.recordsPerRequest) + } + + res, err := req.Context(ctx).Do() + if err != nil { + return nil, err + } + + if !res.JobComplete { + return nil, errIncompleteJob + } + schema := convertTableSchema(res.Schema) + result := &readDataResult{ + pageToken: res.PageToken, + totalRows: res.TotalRows, + schema: schema, + } + result.rows, err = convertRows(res.Rows, schema) + if err != nil { + return nil, err + } + return result, nil +} + +type jobType int + +const ( + copyJobType jobType = iota + extractJobType + loadJobType + queryJobType +) + +func (s *bigqueryService) getJobType(ctx context.Context, projectID, jobID string) (jobType, error) { + res, err := s.s.Jobs.Get(projectID, jobID). + Fields("configuration"). + Context(ctx). + Do() + + if err != nil { + return 0, err + } + + switch { + case res.Configuration.Copy != nil: + return copyJobType, nil + case res.Configuration.Extract != nil: + return extractJobType, nil + case res.Configuration.Load != nil: + return loadJobType, nil + case res.Configuration.Query != nil: + return queryJobType, nil + default: + return 0, errors.New("unknown job type") + } +} + +func (s *bigqueryService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) { + res, err := s.s.Jobs.Get(projectID, jobID). + Fields("status"). // Only fetch what we need. + Context(ctx). + Do() + if err != nil { + return nil, err + } + return jobStatusFromProto(res.Status) +} + +var stateMap = map[string]State{"PENDING": Pending, "RUNNING": Running, "DONE": Done} + +func jobStatusFromProto(status *bq.JobStatus) (*JobStatus, error) { + state, ok := stateMap[status.State] + if !ok { + return nil, fmt.Errorf("unexpected job state: %v", status.State) + } + + newStatus := &JobStatus{ + State: state, + err: nil, + } + if err := errorFromErrorProto(status.ErrorResult); state == Done && err != nil { + newStatus.err = err + } + + for _, ep := range status.Errors { + newStatus.Errors = append(newStatus.Errors, errorFromErrorProto(ep)) + } + return newStatus, nil +} + +// listTables returns a subset of tables that belong to a dataset, and a token for fetching the next subset. +func (s *bigqueryService) listTables(ctx context.Context, projectID, datasetID, pageToken string) ([]*Table, string, error) { + var tables []*Table + res, err := s.s.Tables.List(projectID, datasetID). + PageToken(pageToken). + Context(ctx). + Do() + if err != nil { + return nil, "", err + } + for _, t := range res.Tables { + tables = append(tables, convertListedTable(t)) + } + return tables, res.NextPageToken, nil +} + +type createTableConf struct { + projectID, datasetID, tableID string + expiration time.Time + viewQuery string +} + +// createTable creates a table in the BigQuery service. +// expiration is an optional time after which the table will be deleted and its storage reclaimed. +// If viewQuery is non-empty, the created table will be of type VIEW. +// Note: expiration can only be set during table creation. +// Note: after table creation, a view can be modified only if its table was initially created with a view. +func (s *bigqueryService) createTable(ctx context.Context, conf *createTableConf) error { + table := &bq.Table{ + TableReference: &bq.TableReference{ + ProjectId: conf.projectID, + DatasetId: conf.datasetID, + TableId: conf.tableID, + }, + } + if !conf.expiration.IsZero() { + table.ExpirationTime = conf.expiration.UnixNano() / 1000 + } + if conf.viewQuery != "" { + table.View = &bq.ViewDefinition{ + Query: conf.viewQuery, + } + } + + _, err := s.s.Tables.Insert(conf.projectID, conf.datasetID, table).Context(ctx).Do() + return err +} + +func (s *bigqueryService) getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error) { + table, err := s.s.Tables.Get(projectID, datasetID, tableID).Context(ctx).Do() + if err != nil { + return nil, err + } + return bqTableToMetadata(table), nil +} + +func (s *bigqueryService) deleteTable(ctx context.Context, projectID, datasetID, tableID string) error { + return s.s.Tables.Delete(projectID, datasetID, tableID).Context(ctx).Do() +} + +func bqTableToMetadata(t *bq.Table) *TableMetadata { + md := &TableMetadata{ + Description: t.Description, + Name: t.FriendlyName, + Type: TableType(t.Type), + ID: t.Id, + NumBytes: t.NumBytes, + NumRows: t.NumRows, + } + if t.ExpirationTime != 0 { + md.ExpirationTime = time.Unix(0, t.ExpirationTime*1e6) + } + if t.CreationTime != 0 { + md.CreationTime = time.Unix(0, t.CreationTime*1e6) + } + if t.LastModifiedTime != 0 { + md.LastModifiedTime = time.Unix(0, int64(t.LastModifiedTime*1e6)) + } + if t.Schema != nil { + md.Schema = convertTableSchema(t.Schema) + } + if t.View != nil { + md.View = t.View.Query + } + + return md +} + +func convertListedTable(t *bq.TableListTables) *Table { + return &Table{ + ProjectID: t.TableReference.ProjectId, + DatasetID: t.TableReference.DatasetId, + TableID: t.TableReference.TableId, + } +} + +// patchTableConf contains fields to be patched. +type patchTableConf struct { + // These fields are omitted from the patch operation if nil. + Description *string + Name *string +} + +func (s *bigqueryService) patchTable(ctx context.Context, projectID, datasetID, tableID string, conf *patchTableConf) (*TableMetadata, error) { + t := &bq.Table{} + forceSend := func(field string) { + t.ForceSendFields = append(t.ForceSendFields, field) + } + + if conf.Description != nil { + t.Description = *conf.Description + forceSend("Description") + } + if conf.Name != nil { + t.FriendlyName = *conf.Name + forceSend("FriendlyName") + } + table, err := s.s.Tables.Patch(projectID, datasetID, tableID, t). + Context(ctx). + Do() + if err != nil { + return nil, err + } + return bqTableToMetadata(table), nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/table.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/table.go new file mode 100644 index 0000000000..5ac54873fe --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/table.go @@ -0,0 +1,278 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "fmt" + "time" + + "golang.org/x/net/context" + + bq "google.golang.org/api/bigquery/v2" +) + +// A Table is a reference to a BigQuery table. +type Table struct { + // ProjectID, DatasetID and TableID may be omitted if the Table is the destination for a query. + // In this case the result will be stored in an ephemeral table. + ProjectID string + DatasetID string + // TableID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_). + // The maximum length is 1,024 characters. + TableID string + + service service +} + +// TableMetadata contains information about a BigQuery table. +type TableMetadata struct { + Description string // The user-friendly description of this table. + Name string // The user-friendly name for this table. + Schema Schema + View string + + ID string // An opaque ID uniquely identifying the table. + Type TableType + + // The time when this table expires. If not set, the table will persist + // indefinitely. Expired tables will be deleted and their storage reclaimed. + ExpirationTime time.Time + + CreationTime time.Time + LastModifiedTime time.Time + + // The size of the table in bytes. + // This does not include data that is being buffered during a streaming insert. + NumBytes int64 + + // The number of rows of data in this table. + // This does not include data that is being buffered during a streaming insert. + NumRows uint64 +} + +// Tables is a group of tables. The tables may belong to differing projects or datasets. +type Tables []*Table + +// CreateDisposition specifies the circumstances under which destination table will be created. +// Default is CreateIfNeeded. +type TableCreateDisposition string + +const ( + // The table will be created if it does not already exist. Tables are created atomically on successful completion of a job. + CreateIfNeeded TableCreateDisposition = "CREATE_IF_NEEDED" + + // The table must already exist and will not be automatically created. + CreateNever TableCreateDisposition = "CREATE_NEVER" +) + +func CreateDisposition(disp TableCreateDisposition) Option { return disp } + +func (opt TableCreateDisposition) implementsOption() {} + +func (opt TableCreateDisposition) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) { + conf.CreateDisposition = string(opt) +} + +func (opt TableCreateDisposition) customizeCopy(conf *bq.JobConfigurationTableCopy, projectID string) { + conf.CreateDisposition = string(opt) +} + +func (opt TableCreateDisposition) customizeQuery(conf *bq.JobConfigurationQuery, projectID string) { + conf.CreateDisposition = string(opt) +} + +// TableWriteDisposition specifies how existing data in a destination table is treated. +// Default is WriteAppend. +type TableWriteDisposition string + +const ( + // Data will be appended to any existing data in the destination table. + // Data is appended atomically on successful completion of a job. + WriteAppend TableWriteDisposition = "WRITE_APPEND" + + // Existing data in the destination table will be overwritten. + // Data is overwritten atomically on successful completion of a job. + WriteTruncate TableWriteDisposition = "WRITE_TRUNCATE" + + // Writes will fail if the destination table already contains data. + WriteEmpty TableWriteDisposition = "WRITE_EMPTY" +) + +func WriteDisposition(disp TableWriteDisposition) Option { return disp } + +func (opt TableWriteDisposition) implementsOption() {} + +func (opt TableWriteDisposition) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) { + conf.WriteDisposition = string(opt) +} + +func (opt TableWriteDisposition) customizeCopy(conf *bq.JobConfigurationTableCopy, projectID string) { + conf.WriteDisposition = string(opt) +} + +func (opt TableWriteDisposition) customizeQuery(conf *bq.JobConfigurationQuery, projectID string) { + conf.WriteDisposition = string(opt) +} + +// TableType is the type of table. +type TableType string + +const ( + RegularTable TableType = "TABLE" + ViewTable TableType = "VIEW" +) + +func (t *Table) implementsSource() {} +func (t *Table) implementsReadSource() {} +func (t *Table) implementsDestination() {} +func (ts Tables) implementsSource() {} + +func (t *Table) tableRefProto() *bq.TableReference { + return &bq.TableReference{ + ProjectId: t.ProjectID, + DatasetId: t.DatasetID, + TableId: t.TableID, + } +} + +// FullyQualifiedName returns the ID of the table in projectID:datasetID.tableID format. +func (t *Table) FullyQualifiedName() string { + return fmt.Sprintf("%s:%s.%s", t.ProjectID, t.DatasetID, t.TableID) +} + +// implicitTable reports whether Table is an empty placeholder, which signifies that a new table should be created with an auto-generated Table ID. +func (t *Table) implicitTable() bool { + return t.ProjectID == "" && t.DatasetID == "" && t.TableID == "" +} + +func (t *Table) customizeLoadDst(conf *bq.JobConfigurationLoad, projectID string) { + conf.DestinationTable = t.tableRefProto() +} + +func (t *Table) customizeExtractSrc(conf *bq.JobConfigurationExtract, projectID string) { + conf.SourceTable = t.tableRefProto() +} + +func (t *Table) customizeCopyDst(conf *bq.JobConfigurationTableCopy, projectID string) { + conf.DestinationTable = t.tableRefProto() +} + +func (ts Tables) customizeCopySrc(conf *bq.JobConfigurationTableCopy, projectID string) { + for _, t := range ts { + conf.SourceTables = append(conf.SourceTables, t.tableRefProto()) + } +} + +func (t *Table) customizeQueryDst(conf *bq.JobConfigurationQuery, projectID string) { + if !t.implicitTable() { + conf.DestinationTable = t.tableRefProto() + } +} + +func (t *Table) customizeReadSrc(cursor *readTableConf) { + cursor.projectID = t.ProjectID + cursor.datasetID = t.DatasetID + cursor.tableID = t.TableID +} + +// OpenTable creates a handle to an existing BigQuery table. If the table does not already exist, subsequent uses of the *Table will fail. +func (c *Client) OpenTable(projectID, datasetID, tableID string) *Table { + return &Table{ProjectID: projectID, DatasetID: datasetID, TableID: tableID, service: c.service} +} + +// CreateTable creates a table in the BigQuery service and returns a handle to it. +func (c *Client) CreateTable(ctx context.Context, projectID, datasetID, tableID string, options ...CreateTableOption) (*Table, error) { + conf := &createTableConf{ + projectID: projectID, + datasetID: datasetID, + tableID: tableID, + } + for _, o := range options { + o.customizeCreateTable(conf) + } + if err := c.service.createTable(ctx, conf); err != nil { + return nil, err + } + return &Table{ProjectID: projectID, DatasetID: datasetID, TableID: tableID, service: c.service}, nil +} + +// Metadata fetches the metadata for the table. +func (t *Table) Metadata(ctx context.Context) (*TableMetadata, error) { + return t.service.getTableMetadata(ctx, t.ProjectID, t.DatasetID, t.TableID) +} + +// Delete deletes the table. +func (t *Table) Delete(ctx context.Context) error { + return t.service.deleteTable(ctx, t.ProjectID, t.DatasetID, t.TableID) +} + +// A CreateTableOption is an optional argument to CreateTable. +type CreateTableOption interface { + customizeCreateTable(*createTableConf) +} + +type tableExpiration time.Time + +// TableExpiration returns a CreateTableOption which will cause the created table to be deleted after the expiration time. +func TableExpiration(exp time.Time) CreateTableOption { return tableExpiration(exp) } + +func (opt tableExpiration) customizeCreateTable(conf *createTableConf) { + conf.expiration = time.Time(opt) +} + +type viewQuery string + +// ViewQuery returns a CreateTableOption that causes the created table to be a virtual table defined by the supplied query. +// For more information see: https://cloud.google.com/bigquery/querying-data#views +func ViewQuery(query string) CreateTableOption { return viewQuery(query) } + +func (opt viewQuery) customizeCreateTable(conf *createTableConf) { + conf.viewQuery = string(opt) +} + +// TableMetadataPatch represents a set of changes to a table's metadata. +type TableMetadataPatch struct { + s service + projectID, datasetID, tableID string + conf patchTableConf +} + +// Patch returns a *TableMetadataPatch, which can be used to modify specific Table metadata fields. +// In order to apply the changes, the TableMetadataPatch's Apply method must be called. +func (t *Table) Patch() *TableMetadataPatch { + return &TableMetadataPatch{ + s: t.service, + projectID: t.ProjectID, + datasetID: t.DatasetID, + tableID: t.TableID, + } +} + +// Description sets the table description. +func (p *TableMetadataPatch) Description(desc string) { + p.conf.Description = &desc +} + +// Name sets the table name. +func (p *TableMetadataPatch) Name(name string) { + p.conf.Name = &name +} + +// TODO(mcgreevy): support patching the schema. + +// Apply applies the patch operation. +func (p *TableMetadataPatch) Apply(ctx context.Context) (*TableMetadata, error) { + return p.s.patchTable(ctx, p.projectID, p.datasetID, p.tableID, &p.conf) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/utils_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/utils_test.go new file mode 100644 index 0000000000..6d441cb94e --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/utils_test.go @@ -0,0 +1,51 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "golang.org/x/net/context" + bq "google.golang.org/api/bigquery/v2" +) + +var defaultTable = &Table{ + ProjectID: "project-id", + DatasetID: "dataset-id", + TableID: "table-id", +} + +var defaultGCS = &GCSReference{ + uris: []string{"uri"}, +} + +var defaultQuery = &Query{ + Q: "query string", + DefaultProjectID: "def-project-id", + DefaultDatasetID: "def-dataset-id", +} + +type testService struct { + *bq.Job + + service +} + +func (s *testService) insertJob(ctx context.Context, job *bq.Job, projectID string) (*Job, error) { + s.Job = job + return &Job{}, nil +} + +func (s *testService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) { + return &JobStatus{State: Done}, nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/value.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/value.go new file mode 100644 index 0000000000..369bcd46cd --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/value.go @@ -0,0 +1,145 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "errors" + "fmt" + "strconv" + "time" + + bq "google.golang.org/api/bigquery/v2" +) + +// Value stores the contents of a single cell from a BigQuery result. +type Value interface{} + +// ValueLoader stores a slice of Values representing a result row from a Read operation. +// See Iterator.Get for more information. +type ValueLoader interface { + Load(v []Value) error +} + +// ValueList converts a []Value to implement ValueLoader. +type ValueList []Value + +// Load stores a sequence of values in a ValueList. +func (vs *ValueList) Load(v []Value) error { + *vs = append(*vs, v...) + return nil +} + +// convertRows converts a series of TableRows into a series of Value slices. +// schema is used to interpret the data from rows; its length must match the +// length of each row. +func convertRows(rows []*bq.TableRow, schema Schema) ([][]Value, error) { + var rs [][]Value + for _, r := range rows { + row, err := convertRow(r, schema) + if err != nil { + return nil, err + } + rs = append(rs, row) + } + return rs, nil +} + +func convertRow(r *bq.TableRow, schema Schema) ([]Value, error) { + if len(schema) != len(r.F) { + return nil, errors.New("schema length does not match row length") + } + var values []Value + for i, cell := range r.F { + fs := schema[i] + v, err := convertValue(cell.V, fs.Type, fs.Schema) + if err != nil { + return nil, err + } + values = append(values, v) + } + return values, nil +} + +func convertValue(val interface{}, typ FieldType, schema Schema) (Value, error) { + switch val := val.(type) { + case nil: + return nil, nil + case []interface{}: + return convertRepeatedRecord(val, typ, schema) + case map[string]interface{}: + return convertNestedRecord(val, schema) + case string: + return convertBasicType(val, typ) + default: + return nil, fmt.Errorf("got value %v; expected a value of type %s", val, typ) + } +} + +func convertRepeatedRecord(vals []interface{}, typ FieldType, schema Schema) (Value, error) { + var values []Value + for _, cell := range vals { + // each cell contains a single entry, keyed by "v" + val := cell.(map[string]interface{})["v"] + v, err := convertValue(val, typ, schema) + if err != nil { + return nil, err + } + values = append(values, v) + } + return values, nil +} + +func convertNestedRecord(val map[string]interface{}, schema Schema) (Value, error) { + // convertNestedRecord is similar to convertRow, as a record has the same structure as a row. + + // Nested records are wrapped in a map with a single key, "f". + record := val["f"].([]interface{}) + if len(record) != len(schema) { + return nil, errors.New("schema length does not match record length") + } + + var values []Value + for i, cell := range record { + // each cell contains a single entry, keyed by "v" + val := cell.(map[string]interface{})["v"] + + fs := schema[i] + v, err := convertValue(val, fs.Type, fs.Schema) + if err != nil { + return nil, err + } + values = append(values, v) + } + return values, nil +} + +// convertBasicType returns val as an interface with a concrete type specified by typ. +func convertBasicType(val string, typ FieldType) (Value, error) { + switch typ { + case StringFieldType: + return val, nil + case IntegerFieldType: + return strconv.Atoi(val) + case FloatFieldType: + return strconv.ParseFloat(val, 64) + case BooleanFieldType: + return strconv.ParseBool(val) + case TimestampFieldType: + f, err := strconv.ParseFloat(val, 64) + return Value(time.Unix(0, int64(f*1e9))), err + default: + return nil, errors.New("unrecognized type") + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigquery/value_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/value_test.go new file mode 100644 index 0000000000..fbd8089329 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigquery/value_test.go @@ -0,0 +1,325 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquery + +import ( + "fmt" + "reflect" + "testing" + "time" + + bq "google.golang.org/api/bigquery/v2" +) + +func TestConvertBasicValues(t *testing.T) { + schema := []*FieldSchema{ + {Type: StringFieldType}, + {Type: IntegerFieldType}, + {Type: FloatFieldType}, + {Type: BooleanFieldType}, + } + row := &bq.TableRow{ + F: []*bq.TableCell{ + {V: "a"}, + {V: "1"}, + {V: "1.2"}, + {V: "true"}, + }, + } + got, err := convertRow(row, schema) + if err != nil { + t.Fatalf("error converting: %v", err) + } + want := []Value{"a", 1, 1.2, true} + if !reflect.DeepEqual(got, want) { + t.Errorf("converting basic values: got:\n%v\nwant:\n%v", got, want) + } +} + +func TestConvertTime(t *testing.T) { + schema := []*FieldSchema{ + {Type: TimestampFieldType}, + } + thyme := time.Date(1970, 1, 1, 10, 0, 0, 10, time.UTC) + row := &bq.TableRow{ + F: []*bq.TableCell{ + {V: fmt.Sprintf("%.10f", float64(thyme.UnixNano())/1e9)}, + }, + } + got, err := convertRow(row, schema) + if err != nil { + t.Fatalf("error converting: %v", err) + } + if !got[0].(time.Time).Equal(thyme) { + t.Errorf("converting basic values: got:\n%v\nwant:\n%v", got, thyme) + } +} + +func TestConvertNullValues(t *testing.T) { + schema := []*FieldSchema{ + {Type: StringFieldType}, + } + row := &bq.TableRow{ + F: []*bq.TableCell{ + {V: nil}, + }, + } + got, err := convertRow(row, schema) + if err != nil { + t.Fatalf("error converting: %v", err) + } + want := []Value{nil} + if !reflect.DeepEqual(got, want) { + t.Errorf("converting null values: got:\n%v\nwant:\n%v", got, want) + } +} + +func TestBasicRepetition(t *testing.T) { + schema := []*FieldSchema{ + {Type: IntegerFieldType, Repeated: true}, + } + row := &bq.TableRow{ + F: []*bq.TableCell{ + { + V: []interface{}{ + map[string]interface{}{ + "v": "1", + }, + map[string]interface{}{ + "v": "2", + }, + map[string]interface{}{ + "v": "3", + }, + }, + }, + }, + } + got, err := convertRow(row, schema) + if err != nil { + t.Fatalf("error converting: %v", err) + } + want := []Value{[]Value{1, 2, 3}} + if !reflect.DeepEqual(got, want) { + t.Errorf("converting basic repeated values: got:\n%v\nwant:\n%v", got, want) + } +} + +func TestNestedRecordContainingRepetition(t *testing.T) { + schema := []*FieldSchema{ + { + Type: RecordFieldType, + Schema: Schema{ + {Type: IntegerFieldType, Repeated: true}, + }, + }, + } + row := &bq.TableRow{ + F: []*bq.TableCell{ + { + V: map[string]interface{}{ + "f": []interface{}{ + map[string]interface{}{ + "v": []interface{}{ + map[string]interface{}{"v": "1"}, + map[string]interface{}{"v": "2"}, + map[string]interface{}{"v": "3"}, + }, + }, + }, + }, + }, + }, + } + + got, err := convertRow(row, schema) + if err != nil { + t.Fatalf("error converting: %v", err) + } + want := []Value{[]Value{[]Value{1, 2, 3}}} + if !reflect.DeepEqual(got, want) { + t.Errorf("converting basic repeated values: got:\n%v\nwant:\n%v", got, want) + } +} + +func TestRepeatedRecordContainingRepetition(t *testing.T) { + schema := []*FieldSchema{ + { + Type: RecordFieldType, + Repeated: true, + Schema: Schema{ + {Type: IntegerFieldType, Repeated: true}, + }, + }, + } + row := &bq.TableRow{F: []*bq.TableCell{ + { + V: []interface{}{ // repeated records. + map[string]interface{}{ // first record. + "v": map[string]interface{}{ // pointless single-key-map wrapper. + "f": []interface{}{ // list of record fields. + map[string]interface{}{ // only record (repeated ints) + "v": []interface{}{ // pointless wrapper. + map[string]interface{}{ + "v": "1", + }, + map[string]interface{}{ + "v": "2", + }, + map[string]interface{}{ + "v": "3", + }, + }, + }, + }, + }, + }, + map[string]interface{}{ // second record. + "v": map[string]interface{}{ + "f": []interface{}{ + map[string]interface{}{ + "v": []interface{}{ + map[string]interface{}{ + "v": "4", + }, + map[string]interface{}{ + "v": "5", + }, + map[string]interface{}{ + "v": "6", + }, + }, + }, + }, + }, + }, + }, + }, + }} + + got, err := convertRow(row, schema) + if err != nil { + t.Fatalf("error converting: %v", err) + } + want := []Value{ // the row is a list of length 1, containing an entry for the repeated record. + []Value{ // the repeated record is a list of length 2, containing an entry for each repetition. + []Value{ // the record is a list of length 1, containing an entry for the repeated integer field. + []Value{1, 2, 3}, // the repeated integer field is a list of length 3. + }, + []Value{ // second record + []Value{4, 5, 6}, + }, + }, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("converting repeated records with repeated values: got:\n%v\nwant:\n%v", got, want) + } +} + +func TestRepeatedRecordContainingRecord(t *testing.T) { + schema := []*FieldSchema{ + { + Type: RecordFieldType, + Repeated: true, + Schema: Schema{ + { + Type: StringFieldType, + }, + { + Type: RecordFieldType, + Schema: Schema{ + {Type: IntegerFieldType}, + {Type: StringFieldType}, + }, + }, + }, + }, + } + row := &bq.TableRow{F: []*bq.TableCell{ + { + V: []interface{}{ // repeated records. + map[string]interface{}{ // first record. + "v": map[string]interface{}{ // pointless single-key-map wrapper. + "f": []interface{}{ // list of record fields. + map[string]interface{}{ // first record field (name) + "v": "first repeated record", + }, + map[string]interface{}{ // second record field (nested record). + "v": map[string]interface{}{ // pointless single-key-map wrapper. + "f": []interface{}{ // nested record fields + map[string]interface{}{ + "v": "1", + }, + map[string]interface{}{ + "v": "two", + }, + }, + }, + }, + }, + }, + }, + map[string]interface{}{ // second record. + "v": map[string]interface{}{ + "f": []interface{}{ + map[string]interface{}{ + "v": "second repeated record", + }, + map[string]interface{}{ + "v": map[string]interface{}{ + "f": []interface{}{ + map[string]interface{}{ + "v": "3", + }, + map[string]interface{}{ + "v": "four", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }} + + got, err := convertRow(row, schema) + if err != nil { + t.Fatalf("error converting: %v", err) + } + // TODO: test with flattenresults. + want := []Value{ // the row is a list of length 1, containing an entry for the repeated record. + []Value{ // the repeated record is a list of length 2, containing an entry for each repetition. + []Value{ // record contains a string followed by a nested record. + "first repeated record", + []Value{ + 1, + "two", + }, + }, + []Value{ // second record. + "second repeated record", + []Value{ + 3, + "four", + }, + }, + }, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("converting repeated records containing record : got:\n%v\nwant:\n%v", got, want) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/admin.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/admin.go new file mode 100644 index 0000000000..cfda78d2f5 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/admin.go @@ -0,0 +1,267 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bigtable + +import ( + "fmt" + "regexp" + "strings" + + "golang.org/x/net/context" + "google.golang.org/cloud" + btcspb "google.golang.org/cloud/bigtable/internal/cluster_service_proto" + bttspb "google.golang.org/cloud/bigtable/internal/table_service_proto" + "google.golang.org/cloud/internal/transport" + "google.golang.org/grpc" +) + +const adminAddr = "bigtabletableadmin.googleapis.com:443" + +// AdminClient is a client type for performing admin operations within a specific cluster. +type AdminClient struct { + conn *grpc.ClientConn + tClient bttspb.BigtableTableServiceClient + + project, zone, cluster string +} + +// NewAdminClient creates a new AdminClient for a given project, zone and cluster. +func NewAdminClient(ctx context.Context, project, zone, cluster string, opts ...cloud.ClientOption) (*AdminClient, error) { + o := []cloud.ClientOption{ + cloud.WithEndpoint(adminAddr), + cloud.WithScopes(AdminScope), + cloud.WithUserAgent(clientUserAgent), + } + o = append(o, opts...) + conn, err := transport.DialGRPC(ctx, o...) + if err != nil { + return nil, fmt.Errorf("dialing: %v", err) + } + return &AdminClient{ + conn: conn, + tClient: bttspb.NewBigtableTableServiceClient(conn), + + project: project, + zone: zone, + cluster: cluster, + }, nil +} + +// Close closes the AdminClient. +func (ac *AdminClient) Close() { + ac.conn.Close() +} + +func (ac *AdminClient) clusterPrefix() string { + return fmt.Sprintf("projects/%s/zones/%s/clusters/%s", ac.project, ac.zone, ac.cluster) +} + +// Tables returns a list of the tables in the cluster. +func (ac *AdminClient) Tables(ctx context.Context) ([]string, error) { + prefix := ac.clusterPrefix() + req := &bttspb.ListTablesRequest{ + Name: prefix, + } + res, err := ac.tClient.ListTables(ctx, req) + if err != nil { + return nil, err + } + names := make([]string, 0, len(res.Tables)) + for _, tbl := range res.Tables { + names = append(names, strings.TrimPrefix(tbl.Name, prefix+"/tables/")) + } + return names, nil +} + +// CreateTable creates a new table in the cluster. +// This method may return before the table's creation is complete. +func (ac *AdminClient) CreateTable(ctx context.Context, table string) error { + prefix := ac.clusterPrefix() + req := &bttspb.CreateTableRequest{ + Name: prefix, + TableId: table, + } + _, err := ac.tClient.CreateTable(ctx, req) + if err != nil { + return err + } + return nil +} + +// CreateColumnFamily creates a new column family in a table. +func (ac *AdminClient) CreateColumnFamily(ctx context.Context, table, family string) error { + // TODO(dsymonds): Permit specifying gcexpr and any other family settings. + prefix := ac.clusterPrefix() + req := &bttspb.CreateColumnFamilyRequest{ + Name: prefix + "/tables/" + table, + ColumnFamilyId: family, + } + _, err := ac.tClient.CreateColumnFamily(ctx, req) + return err +} + +// DeleteTable deletes a table and all of its data. +func (ac *AdminClient) DeleteTable(ctx context.Context, table string) error { + prefix := ac.clusterPrefix() + req := &bttspb.DeleteTableRequest{ + Name: prefix + "/tables/" + table, + } + _, err := ac.tClient.DeleteTable(ctx, req) + return err +} + +// DeleteColumnFamily deletes a column family in a table and all of its data. +func (ac *AdminClient) DeleteColumnFamily(ctx context.Context, table, family string) error { + prefix := ac.clusterPrefix() + req := &bttspb.DeleteColumnFamilyRequest{ + Name: prefix + "/tables/" + table + "/columnFamilies/" + family, + } + _, err := ac.tClient.DeleteColumnFamily(ctx, req) + return err +} + +// TableInfo represents information about a table. +type TableInfo struct { + Families []string +} + +// TableInfo retrieves information about a table. +func (ac *AdminClient) TableInfo(ctx context.Context, table string) (*TableInfo, error) { + prefix := ac.clusterPrefix() + req := &bttspb.GetTableRequest{ + Name: prefix + "/tables/" + table, + } + res, err := ac.tClient.GetTable(ctx, req) + if err != nil { + return nil, err + } + ti := &TableInfo{} + for fam := range res.ColumnFamilies { + ti.Families = append(ti.Families, fam) + } + return ti, nil +} + +// SetGCPolicy specifies which cells in a column family should be garbage collected. +// GC executes opportunistically in the background; table reads may return data +// matching the GC policy. +func (ac *AdminClient) SetGCPolicy(ctx context.Context, table, family string, policy GCPolicy) error { + prefix := ac.clusterPrefix() + tbl, err := ac.tClient.GetTable(ctx, &bttspb.GetTableRequest{ + Name: prefix + "/tables/" + table, + }) + if err != nil { + return err + } + fam, ok := tbl.ColumnFamilies[family] + if !ok { + return fmt.Errorf("unknown column family %q", family) + } + fam.GcRule = policy.proto() + _, err = ac.tClient.UpdateColumnFamily(ctx, fam) + return err +} + +const clusterAdminAddr = "bigtableclusteradmin.googleapis.com:443" + +// ClusterAdminClient is a client type for performing admin operations on clusters. +// These operations can be substantially more dangerous than those provided by AdminClient. +type ClusterAdminClient struct { + conn *grpc.ClientConn + cClient btcspb.BigtableClusterServiceClient + + project string +} + +// NewClusterAdminClient creates a new ClusterAdminClient for a given project. +func NewClusterAdminClient(ctx context.Context, project string, opts ...cloud.ClientOption) (*ClusterAdminClient, error) { + o := []cloud.ClientOption{ + cloud.WithEndpoint(clusterAdminAddr), + cloud.WithScopes(ClusterAdminScope), + cloud.WithUserAgent(clientUserAgent), + } + o = append(o, opts...) + conn, err := transport.DialGRPC(ctx, o...) + if err != nil { + return nil, fmt.Errorf("dialing: %v", err) + } + return &ClusterAdminClient{ + conn: conn, + cClient: btcspb.NewBigtableClusterServiceClient(conn), + + project: project, + }, nil +} + +// Close closes the ClusterAdminClient. +func (cac *ClusterAdminClient) Close() { + cac.conn.Close() +} + +// ClusterInfo represents information about a cluster. +type ClusterInfo struct { + Name string // name of the cluster + Zone string // GCP zone of the cluster (e.g. "us-central1-a") + DisplayName string // display name for UIs + ServeNodes int // number of allocated serve nodes +} + +var clusterNameRegexp = regexp.MustCompile(`^projects/([^/]+)/zones/([^/]+)/clusters/([a-z][-a-z0-9]*)$`) + +// Clusters returns a list of clusters in the project. +func (cac *ClusterAdminClient) Clusters(ctx context.Context) ([]*ClusterInfo, error) { + req := &btcspb.ListClustersRequest{ + Name: "projects/" + cac.project, + } + res, err := cac.cClient.ListClusters(ctx, req) + if err != nil { + return nil, err + } + // TODO(dsymonds): Deal with failed_zones. + var cis []*ClusterInfo + for _, c := range res.Clusters { + m := clusterNameRegexp.FindStringSubmatch(c.Name) + if m == nil { + return nil, fmt.Errorf("malformed cluster name %q", c.Name) + } + cis = append(cis, &ClusterInfo{ + Name: m[3], + Zone: m[2], + DisplayName: c.DisplayName, + ServeNodes: int(c.ServeNodes), + }) + } + return cis, nil +} + +/* TODO(dsymonds): Re-enable when there's a ClusterAdmin API. + +// SetClusterSize sets the number of server nodes for this cluster. +func (ac *AdminClient) SetClusterSize(ctx context.Context, nodes int) error { + req := &btcspb.GetClusterRequest{ + Name: ac.clusterPrefix(), + } + clu, err := ac.cClient.GetCluster(ctx, req) + if err != nil { + return err + } + clu.ServeNodes = int32(nodes) + _, err = ac.cClient.UpdateCluster(ctx, clu) + return err +} + +*/ diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/admin_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/admin_test.go new file mode 100644 index 0000000000..191afc20c7 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/admin_test.go @@ -0,0 +1,59 @@ +package bigtable + +import ( + "reflect" + "sort" + "testing" + "time" + + "golang.org/x/net/context" + "google.golang.org/cloud" + "google.golang.org/cloud/bigtable/bttest" + "google.golang.org/grpc" +) + +func TestAdminIntegration(t *testing.T) { + srv, err := bttest.NewServer() + if err != nil { + t.Fatal(err) + } + defer srv.Close() + t.Logf("bttest.Server running on %s", srv.Addr) + + ctx, _ := context.WithTimeout(context.Background(), 2*time.Second) + + conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure()) + if err != nil { + t.Fatalf("grpc.Dial: %v", err) + } + + adminClient, err := NewAdminClient(ctx, "proj", "zone", "cluster", cloud.WithBaseGRPC(conn)) + if err != nil { + t.Fatalf("NewAdminClient: %v", err) + } + defer adminClient.Close() + + list := func() []string { + tbls, err := adminClient.Tables(ctx) + if err != nil { + t.Fatalf("Fetching list of tables: %v", err) + } + sort.Strings(tbls) + return tbls + } + if err := adminClient.CreateTable(ctx, "mytable"); err != nil { + t.Fatalf("Creating table: %v", err) + } + if err := adminClient.CreateTable(ctx, "myothertable"); err != nil { + t.Fatalf("Creating table: %v", err) + } + if got, want := list(), []string{"myothertable", "mytable"}; !reflect.DeepEqual(got, want) { + t.Errorf("adminClient.Tables returned %#v, want %#v", got, want) + } + if err := adminClient.DeleteTable(ctx, "myothertable"); err != nil { + t.Fatalf("Deleting table: %v", err) + } + if got, want := list(), []string{"mytable"}; !reflect.DeepEqual(got, want) { + t.Errorf("adminClient.Tables returned %#v, want %#v", got, want) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bigtable.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bigtable.go new file mode 100644 index 0000000000..08ebe7b801 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bigtable.go @@ -0,0 +1,529 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bigtable // import "google.golang.org/cloud/bigtable" + +import ( + "fmt" + "io" + "strconv" + "time" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + "google.golang.org/cloud" + btdpb "google.golang.org/cloud/bigtable/internal/data_proto" + btspb "google.golang.org/cloud/bigtable/internal/service_proto" + "google.golang.org/cloud/internal/transport" + "google.golang.org/grpc" +) + +const prodAddr = "bigtable.googleapis.com:443" + +// Client is a client for reading and writing data to tables in a cluster. +type Client struct { + conn *grpc.ClientConn + client btspb.BigtableServiceClient + + project, zone, cluster string +} + +// NewClient creates a new Client for a given project, zone and cluster. +func NewClient(ctx context.Context, project, zone, cluster string, opts ...cloud.ClientOption) (*Client, error) { + o := []cloud.ClientOption{ + cloud.WithEndpoint(prodAddr), + cloud.WithScopes(Scope), + cloud.WithUserAgent(clientUserAgent), + } + o = append(o, opts...) + conn, err := transport.DialGRPC(ctx, o...) + if err != nil { + return nil, fmt.Errorf("dialing: %v", err) + } + return &Client{ + conn: conn, + client: btspb.NewBigtableServiceClient(conn), + + project: project, + zone: zone, + cluster: cluster, + }, nil +} + +// Close closes the Client. +func (c *Client) Close() { + c.conn.Close() +} + +func (c *Client) fullTableName(table string) string { + return fmt.Sprintf("projects/%s/zones/%s/clusters/%s/tables/%s", c.project, c.zone, c.cluster, table) +} + +// A Table refers to a table. +type Table struct { + c *Client + table string +} + +// Open opens a table. +func (c *Client) Open(table string) *Table { + return &Table{ + c: c, + table: table, + } +} + +// TODO(dsymonds): Read method that returns a sequence of ReadItems. + +// ReadRows reads rows from a table. f is called for each row. +// If f returns false, the stream is shut down and ReadRows returns. +// f owns its argument, and f is called serially. +// +// By default, the yielded rows will contain all values in all cells. +// Use RowFilter to limit the cells returned. +func (t *Table) ReadRows(ctx context.Context, arg RowRange, f func(Row) bool, opts ...ReadOption) error { + req := &btspb.ReadRowsRequest{ + TableName: t.c.fullTableName(t.table), + Target: &btspb.ReadRowsRequest_RowRange{arg.proto()}, + } + for _, opt := range opts { + opt.set(req) + } + ctx, cancel := context.WithCancel(ctx) // for aborting the stream + stream, err := t.c.client.ReadRows(ctx, req) + if err != nil { + return err + } + cr := new(chunkReader) + for { + res, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return err + } + if row := cr.process(res); row != nil { + if !f(row) { + // Cancel and drain stream. + cancel() + for { + if _, err := stream.Recv(); err != nil { + return nil + } + } + } + } + } + return nil +} + +// ReadRow is a convenience implementation of a single-row reader. +// A missing row will return a zero-length map and a nil error. +func (t *Table) ReadRow(ctx context.Context, row string, opts ...ReadOption) (Row, error) { + var r Row + err := t.ReadRows(ctx, SingleRow(row), func(rr Row) bool { + r = rr + return true + }, opts...) + return r, err +} + +type chunkReader struct { + partial map[string]Row // incomplete rows +} + +// process handles a single btspb.ReadRowsResponse. +// If it completes a row, that row is returned. +func (cr *chunkReader) process(rrr *btspb.ReadRowsResponse) Row { + if cr.partial == nil { + cr.partial = make(map[string]Row) + } + row := string(rrr.RowKey) + r := cr.partial[row] + if r == nil { + r = make(Row) + cr.partial[row] = r + } + for _, chunk := range rrr.Chunks { + switch c := chunk.Chunk.(type) { + case *btspb.ReadRowsResponse_Chunk_ResetRow: + r = make(Row) + cr.partial[row] = r + continue + case *btspb.ReadRowsResponse_Chunk_CommitRow: + delete(cr.partial, row) + if len(r) == 0 { + // Treat zero-content commits as absent. + continue + } + return r // assume that this is the last chunk + case *btspb.ReadRowsResponse_Chunk_RowContents: + decodeFamilyProto(r, row, c.RowContents) + } + } + return nil +} + +// decodeFamilyProto adds the cell data from f to the given row. +func decodeFamilyProto(r Row, row string, f *btdpb.Family) { + fam := f.Name // does not have colon + for _, col := range f.Columns { + for _, cell := range col.Cells { + ri := ReadItem{ + Row: row, + Column: fmt.Sprintf("%s:%s", fam, col.Qualifier), + Timestamp: Timestamp(cell.TimestampMicros), + Value: cell.Value, + } + r[fam] = append(r[fam], ri) + } + } +} + +// A RowRange is used to describe the rows to be read. +// A RowRange is a half-open interval [Start, Limit) encompassing +// all the rows with keys at least as large as Start, and less than Limit. +// (Bigtable string comparison is the same as Go's.) +// A RowRange can be unbounded, encompassing all keys at least as large as Start. +type RowRange struct { + start string + limit string +} + +// NewRange returns the new RowRange [begin, end). +func NewRange(begin, end string) RowRange { + return RowRange{ + start: begin, + limit: end, + } +} + +// Unbounded tests whether a RowRange is unbounded. +func (r RowRange) Unbounded() bool { + return r.limit == "" +} + +// Contains says whether the RowRange contains the key. +func (r RowRange) Contains(row string) bool { + return r.start <= row && (r.limit == "" || r.limit > row) +} + +// String provides a printable description of a RowRange. +func (r RowRange) String() string { + a := strconv.Quote(r.start) + if r.Unbounded() { + return fmt.Sprintf("[%s,∞)", a) + } + return fmt.Sprintf("[%s,%q)", a, r.limit) +} + +func (r RowRange) proto() *btdpb.RowRange { + if r.Unbounded() { + return &btdpb.RowRange{StartKey: []byte(r.start)} + } + return &btdpb.RowRange{ + StartKey: []byte(r.start), + EndKey: []byte(r.limit), + } +} + +// SingleRow returns a RowRange for reading a single row. +func SingleRow(row string) RowRange { + return RowRange{ + start: row, + limit: row + "\x00", + } +} + +// PrefixRange returns a RowRange consisting of all keys starting with the prefix. +func PrefixRange(prefix string) RowRange { + return RowRange{ + start: prefix, + limit: prefixSuccessor(prefix), + } +} + +// InfiniteRange returns the RowRange consisting of all keys at least as +// large as start. +func InfiniteRange(start string) RowRange { + return RowRange{ + start: start, + limit: "", + } +} + +// prefixSuccessor returns the lexically smallest string greater than the +// prefix, if it exists, or "" otherwise. In either case, it is the string +// needed for the Limit of a RowRange. +func prefixSuccessor(prefix string) string { + if prefix == "" { + return "" // infinite range + } + n := len(prefix) + for n--; n >= 0 && prefix[n] == '\xff'; n-- { + } + if n == -1 { + return "" + } + ans := []byte(prefix[:n]) + ans = append(ans, prefix[n]+1) + return string(ans) +} + +// A ReadOption is an optional argument to ReadRows. +type ReadOption interface { + set(req *btspb.ReadRowsRequest) +} + +// RowFilter returns a ReadOption that applies f to the contents of read rows. +func RowFilter(f Filter) ReadOption { return rowFilter{f} } + +type rowFilter struct{ f Filter } + +func (rf rowFilter) set(req *btspb.ReadRowsRequest) { req.Filter = rf.f.proto() } + +// LimitRows returns a ReadOption that will limit the number of rows to be read. +func LimitRows(limit int64) ReadOption { return limitRows{limit} } + +type limitRows struct{ limit int64 } + +func (lr limitRows) set(req *btspb.ReadRowsRequest) { req.NumRowsLimit = lr.limit } + +// A Row is returned by ReadRow. The map is keyed by column family (the prefix +// of the column name before the colon). The values are the returned ReadItems +// for that column family in the order returned by Read. +type Row map[string][]ReadItem + +// Key returns the row's key, or "" if the row is empty. +func (r Row) Key() string { + for _, items := range r { + if len(items) > 0 { + return items[0].Row + } + } + return "" +} + +// A ReadItem is returned by Read. A ReadItem contains data from a specific row and column. +type ReadItem struct { + Row, Column string + Timestamp Timestamp + Value []byte +} + +// Apply applies a Mutation to a specific row. +func (t *Table) Apply(ctx context.Context, row string, m *Mutation, opts ...ApplyOption) error { + after := func(res proto.Message) { + for _, o := range opts { + o.after(res) + } + } + + if m.cond == nil { + req := &btspb.MutateRowRequest{ + TableName: t.c.fullTableName(t.table), + RowKey: []byte(row), + Mutations: m.ops, + } + res, err := t.c.client.MutateRow(ctx, req) + if err == nil { + after(res) + } + return err + } + req := &btspb.CheckAndMutateRowRequest{ + TableName: t.c.fullTableName(t.table), + RowKey: []byte(row), + PredicateFilter: m.cond.proto(), + } + if m.mtrue != nil { + req.TrueMutations = m.mtrue.ops + } + if m.mfalse != nil { + req.FalseMutations = m.mfalse.ops + } + res, err := t.c.client.CheckAndMutateRow(ctx, req) + if err == nil { + after(res) + } + return err +} + +// An ApplyOption is an optional argument to Apply. +type ApplyOption interface { + after(res proto.Message) +} + +type applyAfterFunc func(res proto.Message) + +func (a applyAfterFunc) after(res proto.Message) { a(res) } + +// GetCondMutationResult returns an ApplyOption that reports whether the conditional +// mutation's condition matched. +func GetCondMutationResult(matched *bool) ApplyOption { + return applyAfterFunc(func(res proto.Message) { + if res, ok := res.(*btspb.CheckAndMutateRowResponse); ok { + *matched = res.PredicateMatched + } + }) +} + +// Mutation represents a set of changes for a single row of a table. +type Mutation struct { + ops []*btdpb.Mutation + + // for conditional mutations + cond Filter + mtrue, mfalse *Mutation +} + +// NewMutation returns a new mutation. +func NewMutation() *Mutation { + return new(Mutation) +} + +// NewCondMutation returns a conditional mutation. +// The given row filter determines which mutation is applied: +// If the filter matches any cell in the row, mtrue is applied; +// otherwise, mfalse is applied. +// Either given mutation may be nil. +func NewCondMutation(cond Filter, mtrue, mfalse *Mutation) *Mutation { + return &Mutation{cond: cond, mtrue: mtrue, mfalse: mfalse} +} + +// Set sets a value in a specified column, with the given timestamp. +// The timestamp will be truncated to millisecond resolution. +// A timestamp of ServerTime means to use the server timestamp. +func (m *Mutation) Set(family, column string, ts Timestamp, value []byte) { + if ts != ServerTime { + // Truncate to millisecond resolution, since that's the default table config. + // TODO(dsymonds): Provide a way to override this behaviour. + ts -= ts % 1000 + } + m.ops = append(m.ops, &btdpb.Mutation{Mutation: &btdpb.Mutation_SetCell_{&btdpb.Mutation_SetCell{ + FamilyName: family, + ColumnQualifier: []byte(column), + TimestampMicros: int64(ts), + Value: value, + }}}) +} + +// DeleteCellsInColumn will delete all the cells whose columns are family:column. +func (m *Mutation) DeleteCellsInColumn(family, column string) { + m.ops = append(m.ops, &btdpb.Mutation{Mutation: &btdpb.Mutation_DeleteFromColumn_{&btdpb.Mutation_DeleteFromColumn{ + FamilyName: family, + ColumnQualifier: []byte(column), + }}}) +} + +// DeleteTimestampRange deletes all cells whose columns are family:column +// and whose timestamps are in the half-open interval [start, end). +// If end is zero, it will be interpreted as infinity. +func (m *Mutation) DeleteTimestampRange(family, column string, start, end Timestamp) { + m.ops = append(m.ops, &btdpb.Mutation{Mutation: &btdpb.Mutation_DeleteFromColumn_{&btdpb.Mutation_DeleteFromColumn{ + FamilyName: family, + ColumnQualifier: []byte(column), + TimeRange: &btdpb.TimestampRange{ + StartTimestampMicros: int64(start), + EndTimestampMicros: int64(end), + }, + }}}) +} + +// DeleteCellsInFamily will delete all the cells whose columns are family:*. +func (m *Mutation) DeleteCellsInFamily(family string) { + m.ops = append(m.ops, &btdpb.Mutation{Mutation: &btdpb.Mutation_DeleteFromFamily_{&btdpb.Mutation_DeleteFromFamily{ + FamilyName: family, + }}}) +} + +// DeleteRow deletes the entire row. +func (m *Mutation) DeleteRow() { + m.ops = append(m.ops, &btdpb.Mutation{Mutation: &btdpb.Mutation_DeleteFromRow_{&btdpb.Mutation_DeleteFromRow{}}}) +} + +// Timestamp is in units of microseconds since 1 January 1970. +type Timestamp int64 + +// ServerTime is a specific Timestamp that may be passed to (*Mutation).Set. +// It indicates that the server's timestamp should be used. +const ServerTime Timestamp = -1 + +// Time converts a time.Time into a Timestamp. +func Time(t time.Time) Timestamp { return Timestamp(t.UnixNano() / 1e3) } + +// Now returns the Timestamp representation of the current time on the client. +func Now() Timestamp { return Time(time.Now()) } + +// Time converts a Timestamp into a time.Time. +func (ts Timestamp) Time() time.Time { return time.Unix(0, int64(ts)*1e3) } + +// ApplyReadModifyWrite applies a ReadModifyWrite to a specific row. +// It returns the newly written cells. +func (t *Table) ApplyReadModifyWrite(ctx context.Context, row string, m *ReadModifyWrite) (Row, error) { + req := &btspb.ReadModifyWriteRowRequest{ + TableName: t.c.fullTableName(t.table), + RowKey: []byte(row), + Rules: m.ops, + } + res, err := t.c.client.ReadModifyWriteRow(ctx, req) + if err != nil { + return nil, err + } + r := make(Row) + for _, fam := range res.Families { // res is *btdpb.Row, fam is *btdpb.Family + decodeFamilyProto(r, row, fam) + } + return r, nil +} + +// ReadModifyWrite represents a set of operations on a single row of a table. +// It is like Mutation but for non-idempotent changes. +// When applied, these operations operate on the latest values of the row's cells, +// and result in a new value being written to the relevant cell with a timestamp +// that is max(existing timestamp, current server time). +// +// The application of a ReadModifyWrite is atomic; concurrent ReadModifyWrites will +// be executed serially by the server. +type ReadModifyWrite struct { + ops []*btdpb.ReadModifyWriteRule +} + +// NewReadModifyWrite returns a new ReadModifyWrite. +func NewReadModifyWrite() *ReadModifyWrite { return new(ReadModifyWrite) } + +// AppendValue appends a value to a specific cell's value. +// If the cell is unset, it will be treated as an empty value. +func (m *ReadModifyWrite) AppendValue(family, column string, v []byte) { + m.ops = append(m.ops, &btdpb.ReadModifyWriteRule{ + FamilyName: family, + ColumnQualifier: []byte(column), + Rule: &btdpb.ReadModifyWriteRule_AppendValue{v}, + }) +} + +// Increment interprets the value in a specific cell as a 64-bit big-endian signed integer, +// and adds a value to it. If the cell is unset, it will be treated as zero. +// If the cell is set and is not an 8-byte value, the entire ApplyReadModifyWrite +// operation will fail. +func (m *ReadModifyWrite) Increment(family, column string, delta int64) { + m.ops = append(m.ops, &btdpb.ReadModifyWriteRule{ + FamilyName: family, + ColumnQualifier: []byte(column), + Rule: &btdpb.ReadModifyWriteRule_IncrementAmount{delta}, + }) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bigtable_test.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bigtable_test.go new file mode 100644 index 0000000000..8bce893631 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bigtable_test.go @@ -0,0 +1,606 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bigtable + +import ( + "flag" + "fmt" + "math/rand" + "reflect" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + "google.golang.org/cloud" + "google.golang.org/cloud/bigtable/bttest" + btspb "google.golang.org/cloud/bigtable/internal/service_proto" + "google.golang.org/grpc" +) + +func dataChunk(fam, col string, ts int64, data string) string { + return fmt.Sprintf("chunks:>>>", fam, col, ts, data) +} + +func commit() string { return "chunks:" } +func reset() string { return "chunks:" } + +var chunkTests = []struct { + desc string + chunks []string // sequence of ReadRowsResponse protos in text format + want map[string]Row +}{ + { + desc: "single row single chunk", + chunks: []string{ + `row_key: "row1" ` + dataChunk("fam", "col1", 1428382701000000, "data") + commit(), + }, + want: map[string]Row{ + "row1": Row{ + "fam": []ReadItem{{ + Row: "row1", + Column: "fam:col1", + Timestamp: 1428382701000000, + Value: []byte("data"), + }}, + }, + }, + }, + { + desc: "single row multiple chunks", + chunks: []string{ + `row_key: "row1" ` + dataChunk("fam", "col1", 1428382701000000, "data"), + `row_key: "row1" ` + dataChunk("fam", "col2", 1428382702000000, "more data"), + `row_key: "row1" ` + commit(), + }, + want: map[string]Row{ + "row1": Row{ + "fam": []ReadItem{ + { + Row: "row1", + Column: "fam:col1", + Timestamp: 1428382701000000, + Value: []byte("data"), + }, + { + Row: "row1", + Column: "fam:col2", + Timestamp: 1428382702000000, + Value: []byte("more data"), + }, + }, + }, + }, + }, + { + desc: "chunk, reset, chunk, commit", + chunks: []string{ + `row_key: "row1" ` + dataChunk("fam", "col1", 1428382701000000, "data"), + `row_key: "row1" ` + reset(), + `row_key: "row1" ` + dataChunk("fam", "col1", 1428382702000000, "data") + commit(), + }, + want: map[string]Row{ + "row1": Row{ + "fam": []ReadItem{{ + Row: "row1", + Column: "fam:col1", + Timestamp: 1428382702000000, + Value: []byte("data"), + }}, + }, + }, + }, + { + desc: "chunk, reset, commit", + chunks: []string{ + `row_key: "row1" ` + dataChunk("fam", "col1", 1428382701000000, "data"), + `row_key: "row1" ` + reset(), + `row_key: "row1" ` + commit(), + }, + want: map[string]Row{}, + }, + // TODO(dsymonds): More test cases, including + // - multiple rows +} + +func TestChunkReader(t *testing.T) { + for _, tc := range chunkTests { + cr := new(chunkReader) + got := make(map[string]Row) + for i, txt := range tc.chunks { + rrr := new(btspb.ReadRowsResponse) + if err := proto.UnmarshalText(txt, rrr); err != nil { + t.Fatalf("%s: internal error: bad #%d test text: %v", tc.desc, i, err) + } + if row := cr.process(rrr); row != nil { + got[row.Key()] = row + } + } + // TODO(dsymonds): check for partial rows? + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("%s: processed response mismatch.\n got %+v\nwant %+v", tc.desc, got, tc.want) + } + } +} + +func TestPrefix(t *testing.T) { + tests := []struct { + prefix, succ string + }{ + {"", ""}, + {"\xff", ""}, // when used, "" means Infinity + {"x\xff", "y"}, + {"\xfe", "\xff"}, + } + for _, tc := range tests { + got := prefixSuccessor(tc.prefix) + if got != tc.succ { + t.Errorf("prefixSuccessor(%q) = %q, want %s", tc.prefix, got, tc.succ) + continue + } + r := PrefixRange(tc.prefix) + if tc.succ == "" && r.limit != "" { + t.Errorf("PrefixRange(%q) got limit %q", tc.prefix, r.limit) + } + if tc.succ != "" && r.limit != tc.succ { + t.Errorf("PrefixRange(%q) got limit %q, want %q", tc.prefix, r.limit, tc.succ) + } + } +} + +var useProd = flag.String("use_prod", "", `if set to "proj,zone,cluster,table", run integration test against production`) + +func TestClientIntegration(t *testing.T) { + start := time.Now() + lastCheckpoint := start + checkpoint := func(s string) { + n := time.Now() + t.Logf("[%s] %v since start, %v since last checkpoint", s, n.Sub(start), n.Sub(lastCheckpoint)) + lastCheckpoint = n + } + + proj, zone, cluster, table := "proj", "zone", "cluster", "mytable" + var clientOpts []cloud.ClientOption + timeout := 10 * time.Second + if *useProd == "" { + srv, err := bttest.NewServer() + if err != nil { + t.Fatal(err) + } + defer srv.Close() + t.Logf("bttest.Server running on %s", srv.Addr) + conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure()) + if err != nil { + t.Fatalf("grpc.Dial: %v", err) + } + clientOpts = []cloud.ClientOption{cloud.WithBaseGRPC(conn)} + } else { + t.Logf("Running test against production") + a := strings.Split(*useProd, ",") + proj, zone, cluster, table = a[0], a[1], a[2], a[3] + timeout = 5 * time.Minute + } + + ctx, _ := context.WithTimeout(context.Background(), timeout) + + client, err := NewClient(ctx, proj, zone, cluster, clientOpts...) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + defer client.Close() + checkpoint("dialed Client") + + adminClient, err := NewAdminClient(ctx, proj, zone, cluster, clientOpts...) + if err != nil { + t.Fatalf("NewAdminClient: %v", err) + } + defer adminClient.Close() + checkpoint("dialed AdminClient") + + // Delete the table at the end of the test. + // Do this even before creating the table so that if this is running + // against production and CreateTable fails there's a chance of cleaning it up. + defer adminClient.DeleteTable(ctx, table) + + if err := adminClient.CreateTable(ctx, table); err != nil { + t.Fatalf("Creating table: %v", err) + } + checkpoint("created table") + if err := adminClient.CreateColumnFamily(ctx, table, "follows"); err != nil { + t.Fatalf("Creating column family: %v", err) + } + checkpoint(`created "follows" column family`) + + tbl := client.Open(table) + + // Insert some data. + initialData := map[string][]string{ + "wmckinley": []string{"tjefferson"}, + "gwashington": []string{"jadams"}, + "tjefferson": []string{"gwashington", "jadams"}, // wmckinley set conditionally below + "jadams": []string{"gwashington", "tjefferson"}, + } + for row, ss := range initialData { + mut := NewMutation() + for _, name := range ss { + mut.Set("follows", name, 0, []byte("1")) + } + if err := tbl.Apply(ctx, row, mut); err != nil { + t.Errorf("Mutating row %q: %v", row, err) + } + } + checkpoint("inserted initial data") + + // Do a conditional mutation with a complex filter. + mutTrue := NewMutation() + mutTrue.Set("follows", "wmckinley", 0, []byte("1")) + filter := ChainFilters(ColumnFilter("gwash[iz].*"), ValueFilter(".")) + mut := NewCondMutation(filter, mutTrue, nil) + if err := tbl.Apply(ctx, "tjefferson", mut); err != nil { + t.Errorf("Conditionally mutating row: %v", err) + } + // Do a second condition mutation with a filter that does not match, + // and thus no changes should be made. + mutTrue = NewMutation() + mutTrue.DeleteRow() + filter = ColumnFilter("snoop.dogg") + mut = NewCondMutation(filter, mutTrue, nil) + if err := tbl.Apply(ctx, "tjefferson", mut); err != nil { + t.Errorf("Conditionally mutating row: %v", err) + } + checkpoint("did two conditional mutations") + + // Fetch a row. + row, err := tbl.ReadRow(ctx, "jadams") + if err != nil { + t.Fatalf("Reading a row: %v", err) + } + wantRow := Row{ + "follows": []ReadItem{ + {Row: "jadams", Column: "follows:gwashington", Value: []byte("1")}, + {Row: "jadams", Column: "follows:tjefferson", Value: []byte("1")}, + }, + } + for _, ris := range row { + sort.Sort(byColumn(ris)) + } + if !reflect.DeepEqual(row, wantRow) { + t.Errorf("Read row mismatch.\n got %#v\nwant %#v", row, wantRow) + } + checkpoint("tested ReadRow") + + // Do a bunch of reads with filters. + readTests := []struct { + desc string + rr RowRange + filter Filter // may be nil + + // We do the read, grab all the cells, turn them into "--", + // sort that list, and join with a comma. + want string + }{ + { + desc: "read all, unfiltered", + rr: RowRange{}, + want: "gwashington-jadams-1,jadams-gwashington-1,jadams-tjefferson-1,tjefferson-gwashington-1,tjefferson-jadams-1,tjefferson-wmckinley-1,wmckinley-tjefferson-1", + }, + { + desc: "read with InfiniteRange, unfiltered", + rr: InfiniteRange("tjefferson"), + want: "tjefferson-gwashington-1,tjefferson-jadams-1,tjefferson-wmckinley-1,wmckinley-tjefferson-1", + }, + { + desc: "read with NewRange, unfiltered", + rr: NewRange("gargamel", "hubbard"), + want: "gwashington-jadams-1", + }, + { + desc: "read with PrefixRange, unfiltered", + rr: PrefixRange("jad"), + want: "jadams-gwashington-1,jadams-tjefferson-1", + }, + { + desc: "read with SingleRow, unfiltered", + rr: SingleRow("wmckinley"), + want: "wmckinley-tjefferson-1", + }, + { + desc: "read all, with ColumnFilter", + rr: RowRange{}, + filter: ColumnFilter(".*j.*"), // matches "jadams" and "tjefferson" + want: "gwashington-jadams-1,jadams-tjefferson-1,tjefferson-jadams-1,wmckinley-tjefferson-1", + }, + } + for _, tc := range readTests { + var opts []ReadOption + if tc.filter != nil { + opts = append(opts, RowFilter(tc.filter)) + } + var elt []string + err := tbl.ReadRows(context.Background(), tc.rr, func(r Row) bool { + for _, ris := range r { + for _, ri := range ris { + // Use the column qualifier only to make the test data briefer. + col := ri.Column[strings.Index(ri.Column, ":")+1:] + x := fmt.Sprintf("%s-%s-%s", ri.Row, col, ri.Value) + elt = append(elt, x) + } + } + return true + }, opts...) + if err != nil { + t.Errorf("%s: %v", tc.desc, err) + continue + } + sort.Strings(elt) + if got := strings.Join(elt, ","); got != tc.want { + t.Errorf("%s: wrong reads.\n got %q\nwant %q", tc.desc, got, tc.want) + } + } + checkpoint("tested ReadRows in a few ways") + + // Do a scan and stop part way through. + // Verify that the ReadRows callback doesn't keep running. + stopped := false + err = tbl.ReadRows(ctx, InfiniteRange(""), func(r Row) bool { + if r.Key() < "h" { + return true + } + if !stopped { + stopped = true + return false + } + t.Errorf("ReadRows kept scanning to row %q after being told to stop", r.Key()) + return false + }) + if err != nil { + t.Errorf("Partial ReadRows: %v", err) + } + checkpoint("did partial ReadRows test") + + // Delete a row and check it goes away. + mut = NewMutation() + mut.DeleteRow() + if err := tbl.Apply(ctx, "wmckinley", mut); err != nil { + t.Errorf("Apply DeleteRow: %v", err) + } + row, err = tbl.ReadRow(ctx, "wmckinley") + if err != nil { + t.Fatalf("Reading a row after DeleteRow: %v", err) + } + if len(row) != 0 { + t.Fatalf("Read non-zero row after DeleteRow: %v", row) + } + checkpoint("exercised DeleteRow") + + // Check ReadModifyWrite. + + if err := adminClient.CreateColumnFamily(ctx, table, "counter"); err != nil { + t.Fatalf("Creating column family: %v", err) + } + + appendRMW := func(b []byte) *ReadModifyWrite { + rmw := NewReadModifyWrite() + rmw.AppendValue("counter", "likes", b) + return rmw + } + incRMW := func(n int64) *ReadModifyWrite { + rmw := NewReadModifyWrite() + rmw.Increment("counter", "likes", n) + return rmw + } + rmwSeq := []struct { + desc string + rmw *ReadModifyWrite + want []byte + }{ + { + desc: "append #1", + rmw: appendRMW([]byte{0, 0, 0}), + want: []byte{0, 0, 0}, + }, + { + desc: "append #2", + rmw: appendRMW([]byte{0, 0, 0, 0, 17}), // the remaining 40 bits to make a big-endian 17 + want: []byte{0, 0, 0, 0, 0, 0, 0, 17}, + }, + { + desc: "increment", + rmw: incRMW(8), + want: []byte{0, 0, 0, 0, 0, 0, 0, 25}, + }, + } + for _, step := range rmwSeq { + row, err := tbl.ApplyReadModifyWrite(ctx, "gwashington", step.rmw) + if err != nil { + t.Fatalf("ApplyReadModifyWrite %+v: %v", step.rmw, err) + } + clearTimestamps(row) + wantRow := Row{"counter": []ReadItem{{Row: "gwashington", Column: "counter:likes", Value: step.want}}} + if !reflect.DeepEqual(row, wantRow) { + t.Fatalf("After %s,\n got %v\nwant %v", step.desc, row, wantRow) + } + } + checkpoint("tested ReadModifyWrite") + + // Test arbitrary timestamps more thoroughly. + if err := adminClient.CreateColumnFamily(ctx, table, "ts"); err != nil { + t.Fatalf("Creating column family: %v", err) + } + const numVersions = 4 + mut = NewMutation() + for i := 0; i < numVersions; i++ { + // Timestamps are used in thousands because the server + // only permits that granularity. + mut.Set("ts", "col", Timestamp(i*1000), []byte(fmt.Sprintf("val-%d", i))) + } + if err := tbl.Apply(ctx, "testrow", mut); err != nil { + t.Fatalf("Mutating row: %v", err) + } + r, err := tbl.ReadRow(ctx, "testrow") + if err != nil { + t.Fatalf("Reading row: %v", err) + } + wantRow = Row{"ts": []ReadItem{ + // These should be returned in descending timestamp order. + {Row: "testrow", Column: "ts:col", Timestamp: 3000, Value: []byte("val-3")}, + {Row: "testrow", Column: "ts:col", Timestamp: 2000, Value: []byte("val-2")}, + {Row: "testrow", Column: "ts:col", Timestamp: 1000, Value: []byte("val-1")}, + {Row: "testrow", Column: "ts:col", Timestamp: 0, Value: []byte("val-0")}, + }} + if !reflect.DeepEqual(r, wantRow) { + t.Errorf("Cell with multiple versions,\n got %v\nwant %v", r, wantRow) + } + // Do the same read, but filter to the latest two versions. + r, err = tbl.ReadRow(ctx, "testrow", RowFilter(LatestNFilter(2))) + if err != nil { + t.Fatalf("Reading row: %v", err) + } + wantRow = Row{"ts": []ReadItem{ + {Row: "testrow", Column: "ts:col", Timestamp: 3000, Value: []byte("val-3")}, + {Row: "testrow", Column: "ts:col", Timestamp: 2000, Value: []byte("val-2")}, + }} + if !reflect.DeepEqual(r, wantRow) { + t.Errorf("Cell with multiple versions and LatestNFilter(2),\n got %v\nwant %v", r, wantRow) + } + // Delete the cell with timestamp 2000 and repeat the last read, + // checking that we get ts 3000 and ts 1000. + mut = NewMutation() + mut.DeleteTimestampRange("ts", "col", 2000, 3000) // half-open interval + if err := tbl.Apply(ctx, "testrow", mut); err != nil { + t.Fatalf("Mutating row: %v", err) + } + r, err = tbl.ReadRow(ctx, "testrow", RowFilter(LatestNFilter(2))) + if err != nil { + t.Fatalf("Reading row: %v", err) + } + wantRow = Row{"ts": []ReadItem{ + {Row: "testrow", Column: "ts:col", Timestamp: 3000, Value: []byte("val-3")}, + {Row: "testrow", Column: "ts:col", Timestamp: 1000, Value: []byte("val-1")}, + }} + if !reflect.DeepEqual(r, wantRow) { + t.Errorf("Cell with multiple versions and LatestNFilter(2), after deleting timestamp 2000,\n got %v\nwant %v", r, wantRow) + } + checkpoint("tested multiple versions in a cell") + + // Do highly concurrent reads/writes. + // TODO(dsymonds): Raise this to 1000 when https://github.com/grpc/grpc-go/issues/205 is resolved. + const maxConcurrency = 100 + var wg sync.WaitGroup + for i := 0; i < maxConcurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + switch r := rand.Intn(100); { // r ∈ [0,100) + case 0 <= r && r < 30: + // Do a read. + _, err := tbl.ReadRow(ctx, "testrow", RowFilter(LatestNFilter(1))) + if err != nil { + t.Errorf("Concurrent read: %v", err) + } + case 30 <= r && r < 100: + // Do a write. + mut := NewMutation() + mut.Set("ts", "col", 0, []byte("data")) + if err := tbl.Apply(ctx, "testrow", mut); err != nil { + t.Errorf("Concurrent write: %v", err) + } + } + }() + } + wg.Wait() + checkpoint("tested high concurrency") + + // Large reads, writes and scans. + bigBytes := make([]byte, 15<<20) // 15 MB is large + nonsense := []byte("lorem ipsum dolor sit amet, ") + fill(bigBytes, nonsense) + mut = NewMutation() + mut.Set("ts", "col", 0, bigBytes) + if err := tbl.Apply(ctx, "bigrow", mut); err != nil { + t.Errorf("Big write: %v", err) + } + r, err = tbl.ReadRow(ctx, "bigrow") + if err != nil { + t.Errorf("Big read: %v", err) + } + wantRow = Row{"ts": []ReadItem{ + {Row: "bigrow", Column: "ts:col", Value: bigBytes}, + }} + if !reflect.DeepEqual(r, wantRow) { + t.Errorf("Big read returned incorrect bytes: %v", r) + } + // Now write 1000 rows, each with 82 KB values, then scan them all. + medBytes := make([]byte, 82<<10) + fill(medBytes, nonsense) + sem := make(chan int, 50) // do up to 50 mutations at a time. + for i := 0; i < 1000; i++ { + mut := NewMutation() + mut.Set("ts", "big-scan", 0, medBytes) + row := fmt.Sprintf("row-%d", i) + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-sem }() + sem <- 1 + if err := tbl.Apply(ctx, row, mut); err != nil { + t.Errorf("Preparing large scan: %v", err) + } + }() + } + wg.Wait() + n := 0 + err = tbl.ReadRows(ctx, PrefixRange("row-"), func(r Row) bool { + for _, ris := range r { + for _, ri := range ris { + n += len(ri.Value) + } + } + return true + }, RowFilter(ColumnFilter("big-scan"))) + if err != nil { + t.Errorf("Doing large scan: %v", err) + } + if want := 1000 * len(medBytes); n != want { + t.Errorf("Large scan returned %d bytes, want %d", n, want) + } + checkpoint("tested big read/write/scan") +} + +func fill(b, sub []byte) { + for len(b) > len(sub) { + n := copy(b, sub) + b = b[n:] + } +} + +type byColumn []ReadItem + +func (b byColumn) Len() int { return len(b) } +func (b byColumn) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byColumn) Less(i, j int) bool { return b[i].Column < b[j].Column } + +func clearTimestamps(r Row) { + for _, ris := range r { + for i := range ris { + ris[i].Timestamp = 0 + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bttest/inmem.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bttest/inmem.go new file mode 100644 index 0000000000..e771873895 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/bttest/inmem.go @@ -0,0 +1,839 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package bttest contains test helpers for working with the bigtable package. + +To use a Server, create it, and then connect to it with no security: +(The project/zone/cluster values are ignored.) + srv, err := bttest.NewServer() + ... + conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure()) + ... + client, err := bigtable.NewClient(ctx, proj, zone, cluster, + bigtable.WithBaseGRPC(conn)) + ... +*/ +package bttest // import "google.golang.org/cloud/bigtable/bttest" + +import ( + "encoding/binary" + "fmt" + "log" + "math/rand" + "net" + "regexp" + "sort" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + btdpb "google.golang.org/cloud/bigtable/internal/data_proto" + emptypb "google.golang.org/cloud/bigtable/internal/empty" + btspb "google.golang.org/cloud/bigtable/internal/service_proto" + bttdpb "google.golang.org/cloud/bigtable/internal/table_data_proto" + bttspb "google.golang.org/cloud/bigtable/internal/table_service_proto" + "google.golang.org/grpc" +) + +// Server is an in-memory Cloud Bigtable fake. +// It is unauthenticated, and only a rough approximation. +type Server struct { + Addr string + + l net.Listener + srv *grpc.Server + s *server +} + +// server is the real implementation of the fake. +// It is a separate and unexported type so the API won't be cluttered with +// methods that are only relevant to the fake's implementation. +type server struct { + mu sync.Mutex + tables map[string]*table // keyed by fully qualified name + gcc chan int // set when gcloop starts, closed when server shuts down + + // Any unimplemented methods will cause a panic. + bttspb.BigtableTableServiceServer + btspb.BigtableServiceServer +} + +// NewServer creates a new Server. The Server will be listening for gRPC connections +// at the address named by the Addr field, without TLS. +func NewServer() (*Server, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + + s := &Server{ + Addr: l.Addr().String(), + l: l, + srv: grpc.NewServer(), + s: &server{ + tables: make(map[string]*table), + }, + } + bttspb.RegisterBigtableTableServiceServer(s.srv, s.s) + btspb.RegisterBigtableServiceServer(s.srv, s.s) + + go s.srv.Serve(s.l) + + return s, nil +} + +// Close shuts down the server. +func (s *Server) Close() { + s.s.mu.Lock() + if s.s.gcc != nil { + close(s.s.gcc) + } + s.s.mu.Unlock() + + s.srv.Stop() + s.l.Close() +} + +func (s *server) CreateTable(ctx context.Context, req *bttspb.CreateTableRequest) (*bttdpb.Table, error) { + tbl := req.Name + "/tables/" + req.TableId + + s.mu.Lock() + if _, ok := s.tables[tbl]; ok { + s.mu.Unlock() + return nil, fmt.Errorf("table %q already exists", tbl) + } + s.tables[tbl] = newTable() + s.mu.Unlock() + + return &bttdpb.Table{Name: tbl}, nil +} + +func (s *server) ListTables(ctx context.Context, req *bttspb.ListTablesRequest) (*bttspb.ListTablesResponse, error) { + res := &bttspb.ListTablesResponse{} + prefix := req.Name + "/tables/" + + s.mu.Lock() + for tbl := range s.tables { + if strings.HasPrefix(tbl, prefix) { + res.Tables = append(res.Tables, &bttdpb.Table{Name: tbl}) + } + } + s.mu.Unlock() + + return res, nil +} + +func (s *server) GetTable(ctx context.Context, req *bttspb.GetTableRequest) (*bttdpb.Table, error) { + tbl := req.Name + + s.mu.Lock() + tblIns, ok := s.tables[tbl] + s.mu.Unlock() + if !ok { + return nil, fmt.Errorf("table %q not found", tbl) + } + + return &bttdpb.Table{ + Name: tbl, + ColumnFamilies: toColumnFamilies(tblIns.families), + }, nil +} + +func (s *server) DeleteTable(ctx context.Context, req *bttspb.DeleteTableRequest) (*emptypb.Empty, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.tables[req.Name]; !ok { + return nil, fmt.Errorf("no such table %q", req.Name) + } + delete(s.tables, req.Name) + return &emptypb.Empty{}, nil +} + +func (s *server) CreateColumnFamily(ctx context.Context, req *bttspb.CreateColumnFamilyRequest) (*bttdpb.ColumnFamily, error) { + s.mu.Lock() + tbl, ok := s.tables[req.Name] + s.mu.Unlock() + if !ok { + return nil, fmt.Errorf("no such table %q", req.Name) + } + + // Check it is unique and record it. + fam := req.ColumnFamilyId + tbl.mu.Lock() + defer tbl.mu.Unlock() + if _, ok := tbl.families[fam]; ok { + return nil, fmt.Errorf("family %q already exists", fam) + } + newcf := &columnFamily{ + name: req.Name + "/columnFamilies/" + fam, + } + tbl.families[fam] = newcf + return newcf.proto(), nil +} + +func (s *server) UpdateColumnFamily(ctx context.Context, req *bttdpb.ColumnFamily) (*bttdpb.ColumnFamily, error) { + index := strings.Index(req.Name, "/columnFamilies/") + if index == -1 { + return nil, fmt.Errorf("bad family name %q", req.Name) + } + tblName := req.Name[:index] + fam := req.Name[index+len("/columnFamilies/"):] + + s.mu.Lock() + tbl, ok := s.tables[tblName] + s.mu.Unlock() + if !ok { + return nil, fmt.Errorf("no such table %q", req.Name) + } + + tbl.mu.Lock() + defer tbl.mu.Unlock() + + // Check it is unique and record it. + if _, ok := tbl.families[fam]; !ok { + return nil, fmt.Errorf("no such family %q", fam) + } + + newcf := &columnFamily{ + name: req.Name, + gcRule: req.GcRule, + } + // assume that we ALWAYS want to replace by the new setting + // we may need partial update through + tbl.families[fam] = newcf + s.needGC() + return newcf.proto(), nil +} + +func (s *server) ReadRows(req *btspb.ReadRowsRequest, stream btspb.BigtableService_ReadRowsServer) error { + s.mu.Lock() + tbl, ok := s.tables[req.TableName] + s.mu.Unlock() + if !ok { + return fmt.Errorf("no such table %q", req.TableName) + } + + var start, end string // half-open interval + switch targ := req.Target.(type) { + case *btspb.ReadRowsRequest_RowRange: + start, end = string(targ.RowRange.StartKey), string(targ.RowRange.EndKey) + case *btspb.ReadRowsRequest_RowKey: + // A single row read is simply an edge case. + start = string(targ.RowKey) + end = start + "\x00" + default: + return fmt.Errorf("unknown ReadRowsRequest.Target oneof %T", targ) + } + + // Get rows to stream back. + tbl.mu.RLock() + si, ei := 0, len(tbl.rows) // half-open interval + if start != "" { + si = sort.Search(len(tbl.rows), func(i int) bool { return tbl.rows[i].key >= start }) + } + if end != "" { + ei = sort.Search(len(tbl.rows), func(i int) bool { return tbl.rows[i].key >= end }) + } + if si >= ei { + tbl.mu.RUnlock() + return nil + } + rows := make([]*row, ei-si) + copy(rows, tbl.rows[si:ei]) + tbl.mu.RUnlock() + + for _, r := range rows { + if err := streamRow(stream, r, req.Filter); err != nil { + return err + } + } + + return nil +} + +func streamRow(stream btspb.BigtableService_ReadRowsServer, r *row, f *btdpb.RowFilter) error { + r.mu.Lock() + nr := r.copy() + r.mu.Unlock() + r = nr + + filterRow(f, r) + + rrr := &btspb.ReadRowsResponse{ + RowKey: []byte(r.key), + } + for col, cells := range r.cells { + i := strings.Index(col, ":") // guaranteed to exist + fam, col := col[:i], col[i+1:] + if len(cells) == 0 { + continue + } + // TODO(dsymonds): Apply transformers. + colm := &btdpb.Column{ + Qualifier: []byte(col), + // Cells is populated below. + } + for _, cell := range cells { + colm.Cells = append(colm.Cells, &btdpb.Cell{ + TimestampMicros: cell.ts, + Value: cell.value, + }) + } + rrr.Chunks = append(rrr.Chunks, &btspb.ReadRowsResponse_Chunk{ + Chunk: &btspb.ReadRowsResponse_Chunk_RowContents{&btdpb.Family{ + Name: fam, + Columns: []*btdpb.Column{colm}, + }}, + }) + } + rrr.Chunks = append(rrr.Chunks, &btspb.ReadRowsResponse_Chunk{Chunk: &btspb.ReadRowsResponse_Chunk_CommitRow{true}}) + return stream.Send(rrr) +} + +// filterRow modifies a row with the given filter. +func filterRow(f *btdpb.RowFilter, r *row) { + if f == nil { + return + } + // Handle filters that apply beyond just including/excluding cells. + switch f := f.Filter.(type) { + case *btdpb.RowFilter_Chain_: + for _, sub := range f.Chain.Filters { + filterRow(sub, r) + } + return + case *btdpb.RowFilter_Interleave_: + srs := make([]*row, 0, len(f.Interleave.Filters)) + for _, sub := range f.Interleave.Filters { + sr := r.copy() + filterRow(sub, sr) + srs = append(srs, sr) + } + // merge + // TODO(dsymonds): is this correct? + r.cells = make(map[string][]cell) + for _, sr := range srs { + for col, cs := range sr.cells { + r.cells[col] = append(r.cells[col], cs...) + } + } + for _, cs := range r.cells { + sort.Sort(byDescTS(cs)) + } + return + case *btdpb.RowFilter_CellsPerColumnLimitFilter: + lim := int(f.CellsPerColumnLimitFilter) + for col, cs := range r.cells { + if len(cs) > lim { + r.cells[col] = cs[:lim] + } + } + return + } + + // Any other case, operate on a per-cell basis. + for key, cs := range r.cells { + i := strings.Index(key, ":") // guaranteed to exist + fam, col := key[:i], key[i+1:] + r.cells[key] = filterCells(f, fam, col, cs) + } +} + +func filterCells(f *btdpb.RowFilter, fam, col string, cs []cell) []cell { + var ret []cell + for _, cell := range cs { + if includeCell(f, fam, col, cell) { + ret = append(ret, cell) + } + } + return ret +} + +func includeCell(f *btdpb.RowFilter, fam, col string, cell cell) bool { + if f == nil { + return true + } + // TODO(dsymonds): Implement many more filters. + switch f := f.Filter.(type) { + default: + log.Printf("WARNING: don't know how to handle filter of type %T (ignoring it)", f) + return true + case *btdpb.RowFilter_ColumnQualifierRegexFilter: + pat := string(f.ColumnQualifierRegexFilter) + rx, err := regexp.Compile(pat) + if err != nil { + log.Printf("Bad column_qualifier_regex_filter pattern %q: %v", pat, err) + return false + } + return rx.MatchString(col) + case *btdpb.RowFilter_ValueRegexFilter: + pat := string(f.ValueRegexFilter) + rx, err := regexp.Compile(pat) + if err != nil { + log.Printf("Bad value_regex_filter pattern %q: %v", pat, err) + return false + } + return rx.Match(cell.value) + } +} + +func (s *server) MutateRow(ctx context.Context, req *btspb.MutateRowRequest) (*emptypb.Empty, error) { + s.mu.Lock() + tbl, ok := s.tables[req.TableName] + s.mu.Unlock() + if !ok { + return nil, fmt.Errorf("no such table %q", req.TableName) + } + + r := tbl.mutableRow(string(req.RowKey)) + r.mu.Lock() + defer r.mu.Unlock() + + if err := applyMutations(tbl, r, req.Mutations); err != nil { + return nil, err + } + return &emptypb.Empty{}, nil +} + +func (s *server) CheckAndMutateRow(ctx context.Context, req *btspb.CheckAndMutateRowRequest) (*btspb.CheckAndMutateRowResponse, error) { + s.mu.Lock() + tbl, ok := s.tables[req.TableName] + s.mu.Unlock() + if !ok { + return nil, fmt.Errorf("no such table %q", req.TableName) + } + + res := &btspb.CheckAndMutateRowResponse{} + + r := tbl.mutableRow(string(req.RowKey)) + r.mu.Lock() + defer r.mu.Unlock() + + // Figure out which mutation to apply. + whichMut := false + if req.PredicateFilter == nil { + // Use true_mutations iff row contains any cells. + whichMut = len(r.cells) > 0 + } else { + // Use true_mutations iff any cells in the row match the filter. + // TODO(dsymonds): This could be cheaper. + nr := r.copy() + filterRow(req.PredicateFilter, nr) + for _, cs := range nr.cells { + if len(cs) > 0 { + whichMut = true + break + } + } + // TODO(dsymonds): Figure out if this is supposed to be set + // even when there's no predicate filter. + res.PredicateMatched = whichMut + } + muts := req.FalseMutations + if whichMut { + muts = req.TrueMutations + } + + if err := applyMutations(tbl, r, muts); err != nil { + return nil, err + } + return res, nil +} + +// applyMutations applies a sequence of mutations to a row. +// It assumes r.mu is locked. +func applyMutations(tbl *table, r *row, muts []*btdpb.Mutation) error { + for _, mut := range muts { + switch mut := mut.Mutation.(type) { + default: + return fmt.Errorf("can't handle mutation type %T", mut) + case *btdpb.Mutation_SetCell_: + set := mut.SetCell + tbl.mu.RLock() + _, famOK := tbl.families[set.FamilyName] + tbl.mu.RUnlock() + if !famOK { + return fmt.Errorf("unknown family %q", set.FamilyName) + } + ts := set.TimestampMicros + if ts == -1 { // bigtable.ServerTime + ts = time.Now().UnixNano() / 1e3 + ts -= ts % 1000 // round to millisecond granularity + } + if !tbl.validTimestamp(ts) { + return fmt.Errorf("invalid timestamp %d", ts) + } + col := fmt.Sprintf("%s:%s", set.FamilyName, set.ColumnQualifier) + + cs := r.cells[col] + newCell := cell{ts: ts, value: set.Value} + replaced := false + for i, cell := range cs { + if cell.ts == newCell.ts { + cs[i] = newCell + replaced = true + break + } + } + if !replaced { + cs = append(cs, newCell) + } + sort.Sort(byDescTS(cs)) + r.cells[col] = cs + case *btdpb.Mutation_DeleteFromColumn_: + del := mut.DeleteFromColumn + col := fmt.Sprintf("%s:%s", del.FamilyName, del.ColumnQualifier) + + cs := r.cells[col] + if del.TimeRange != nil { + tsr := del.TimeRange + if !tbl.validTimestamp(tsr.StartTimestampMicros) { + return fmt.Errorf("invalid timestamp %d", tsr.StartTimestampMicros) + } + if !tbl.validTimestamp(tsr.EndTimestampMicros) { + return fmt.Errorf("invalid timestamp %d", tsr.EndTimestampMicros) + } + // Find half-open interval to remove. + // Cells are in descending timestamp order, + // so the predicates to sort.Search are inverted. + si, ei := 0, len(cs) + if tsr.StartTimestampMicros > 0 { + ei = sort.Search(len(cs), func(i int) bool { return cs[i].ts < tsr.StartTimestampMicros }) + } + if tsr.EndTimestampMicros > 0 { + si = sort.Search(len(cs), func(i int) bool { return cs[i].ts < tsr.EndTimestampMicros }) + } + if si < ei { + copy(cs[si:], cs[ei:]) + cs = cs[:len(cs)-(ei-si)] + } + } else { + cs = nil + } + if len(cs) == 0 { + delete(r.cells, col) + } else { + r.cells[col] = cs + } + case *btdpb.Mutation_DeleteFromRow_: + r.cells = make(map[string][]cell) + } + } + return nil +} + +func (s *server) ReadModifyWriteRow(ctx context.Context, req *btspb.ReadModifyWriteRowRequest) (*btdpb.Row, error) { + s.mu.Lock() + tbl, ok := s.tables[req.TableName] + s.mu.Unlock() + if !ok { + return nil, fmt.Errorf("no such table %q", req.TableName) + } + + updates := make(map[string]cell) // copy of updated cells; keyed by full column name + + r := tbl.mutableRow(string(req.RowKey)) + r.mu.Lock() + defer r.mu.Unlock() + // Assume all mutations apply to the most recent version of the cell. + // TODO(dsymonds): Verify this assumption and document it in the proto. + for _, rule := range req.Rules { + tbl.mu.RLock() + _, famOK := tbl.families[rule.FamilyName] + tbl.mu.RUnlock() + if !famOK { + return nil, fmt.Errorf("unknown family %q", rule.FamilyName) + } + + key := fmt.Sprintf("%s:%s", rule.FamilyName, rule.ColumnQualifier) + + newCell := false + if len(r.cells[key]) == 0 { + r.cells[key] = []cell{{ + // TODO(dsymonds): should this set a timestamp? + }} + newCell = true + } + cell := &r.cells[key][0] + + switch rule := rule.Rule.(type) { + default: + return nil, fmt.Errorf("unknown RMW rule oneof %T", rule) + case *btdpb.ReadModifyWriteRule_AppendValue: + cell.value = append(cell.value, rule.AppendValue...) + case *btdpb.ReadModifyWriteRule_IncrementAmount: + var v int64 + if !newCell { + if len(cell.value) != 8 { + return nil, fmt.Errorf("increment on non-64-bit value") + } + v = int64(binary.BigEndian.Uint64(cell.value)) + } + v += rule.IncrementAmount + var val [8]byte + binary.BigEndian.PutUint64(val[:], uint64(v)) + cell.value = val[:] + } + updates[key] = *cell + } + + res := &btdpb.Row{ + Key: req.RowKey, + } + for col, cell := range updates { + i := strings.Index(col, ":") + fam, qual := col[:i], col[i+1:] + var f *btdpb.Family + for _, ff := range res.Families { + if ff.Name == fam { + f = ff + break + } + } + if f == nil { + f = &btdpb.Family{Name: fam} + res.Families = append(res.Families, f) + } + f.Columns = append(f.Columns, &btdpb.Column{ + Qualifier: []byte(qual), + Cells: []*btdpb.Cell{{ + Value: cell.value, + }}, + }) + } + return res, nil +} + +// needGC is invoked whenever the server needs gcloop running. +func (s *server) needGC() { + s.mu.Lock() + if s.gcc == nil { + s.gcc = make(chan int) + go s.gcloop(s.gcc) + } + s.mu.Unlock() +} + +func (s *server) gcloop(done <-chan int) { + const ( + minWait = 500 // ms + maxWait = 1500 // ms + ) + + for { + // Wait for a random time interval. + d := time.Duration(minWait+rand.Intn(maxWait-minWait)) * time.Millisecond + select { + case <-time.After(d): + case <-done: + return // server has been closed + } + + // Do a GC pass over all tables. + var tables []*table + s.mu.Lock() + for _, tbl := range s.tables { + tables = append(tables, tbl) + } + s.mu.Unlock() + for _, tbl := range tables { + tbl.gc() + } + } +} + +type table struct { + mu sync.RWMutex + families map[string]*columnFamily // keyed by plain family name + rows []*row // sorted by row key + rowIndex map[string]*row // indexed by row key +} + +func newTable() *table { + return &table{ + families: make(map[string]*columnFamily), + rowIndex: make(map[string]*row), + } +} + +func (t *table) validTimestamp(ts int64) bool { + // Assume millisecond granularity is required. + return ts%1000 == 0 +} + +func (t *table) mutableRow(row string) *row { + // Try fast path first. + t.mu.RLock() + r := t.rowIndex[row] + t.mu.RUnlock() + if r != nil { + return r + } + + // We probably need to create the row. + t.mu.Lock() + r = t.rowIndex[row] + if r == nil { + r = newRow(row) + t.rowIndex[row] = r + t.rows = append(t.rows, r) + sort.Sort(byRowKey(t.rows)) // yay, inefficient! + } + t.mu.Unlock() + return r +} + +func (t *table) gc() { + // This method doesn't add or remove rows, so we only need a read lock for the table. + t.mu.RLock() + defer t.mu.RUnlock() + + // Gather GC rules we'll apply. + rules := make(map[string]*bttdpb.GcRule) // keyed by "fam" + for fam, cf := range t.families { + if cf.gcRule != nil { + rules[fam] = cf.gcRule + } + } + if len(rules) == 0 { + return + } + + for _, r := range t.rows { + r.mu.Lock() + r.gc(rules) + r.mu.Unlock() + } +} + +type byRowKey []*row + +func (b byRowKey) Len() int { return len(b) } +func (b byRowKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byRowKey) Less(i, j int) bool { return b[i].key < b[j].key } + +type row struct { + key string + + mu sync.Mutex + cells map[string][]cell // keyed by full column name; cells are in descending timestamp order +} + +func newRow(key string) *row { + return &row{ + key: key, + cells: make(map[string][]cell), + } +} + +// copy returns a copy of the row. +// Cell values are aliased. +// r.mu should be held. +func (r *row) copy() *row { + nr := &row{ + key: r.key, + cells: make(map[string][]cell, len(r.cells)), + } + for col, cs := range r.cells { + // Copy the []cell slice, but not the []byte inside each cell. + nr.cells[col] = append([]cell(nil), cs...) + } + return nr +} + +// gc applies the given GC rules to the row. +// r.mu should be held. +func (r *row) gc(rules map[string]*bttdpb.GcRule) { + for col, cs := range r.cells { + fam := col[:strings.Index(col, ":")] + rule, ok := rules[fam] + if !ok { + continue + } + r.cells[col] = applyGC(cs, rule) + } +} + +var gcTypeWarn sync.Once + +// applyGC applies the given GC rule to the cells. +func applyGC(cells []cell, rule *bttdpb.GcRule) []cell { + switch rule := rule.Rule.(type) { + default: + // TODO(dsymonds): Support GcRule_Intersection_ + gcTypeWarn.Do(func() { + log.Printf("Unsupported GC rule type %T", rule) + }) + case *bttdpb.GcRule_Union_: + for _, sub := range rule.Union.Rules { + cells = applyGC(cells, sub) + } + return cells + case *bttdpb.GcRule_MaxAge: + // Timestamps are in microseconds. + cutoff := time.Now().UnixNano() / 1e3 + cutoff -= rule.MaxAge.Seconds * 1e6 + cutoff -= int64(rule.MaxAge.Nanos) / 1e3 + // The slice of cells in in descending timestamp order. + // This sort.Search will return the index of the first cell whose timestamp is chronologically before the cutoff. + si := sort.Search(len(cells), func(i int) bool { return cells[i].ts < cutoff }) + if si < len(cells) { + log.Printf("bttest: GC MaxAge(%v) deleted %d cells.", rule.MaxAge, len(cells)-si) + } + return cells[:si] + case *bttdpb.GcRule_MaxNumVersions: + n := int(rule.MaxNumVersions) + if len(cells) > n { + log.Printf("bttest: GC MaxNumVersions(%d) deleted %d cells.", n, len(cells)-n) + cells = cells[:n] + } + return cells + } + return cells +} + +type cell struct { + ts int64 + value []byte +} + +type byDescTS []cell + +func (b byDescTS) Len() int { return len(b) } +func (b byDescTS) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byDescTS) Less(i, j int) bool { return b[i].ts > b[j].ts } + +type columnFamily struct { + name string + gcRule *bttdpb.GcRule +} + +func (c *columnFamily) proto() *bttdpb.ColumnFamily { + return &bttdpb.ColumnFamily{ + Name: c.name, + GcRule: c.gcRule, + } +} + +func toColumnFamilies(families map[string]*columnFamily) map[string]*bttdpb.ColumnFamily { + f := make(map[string]*bttdpb.ColumnFamily) + for k, v := range families { + f[k] = v.proto() + } + return f +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/cbt/cbt.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/cbt/cbt.go new file mode 100644 index 0000000000..504686b709 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/cbt/cbt.go @@ -0,0 +1,580 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// Command docs are in cbtdoc.go. + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "log" + "os" + "regexp" + "sort" + "strconv" + "strings" + "text/tabwriter" + "text/template" + "time" + + "golang.org/x/net/context" + "google.golang.org/cloud/bigtable" + "google.golang.org/cloud/bigtable/internal/cbtrc" +) + +var ( + oFlag = flag.String("o", "", "if set, redirect stdout to this file") + + config *cbtrc.Config + client *bigtable.Client + adminClient *bigtable.AdminClient + clusterAdminClient *bigtable.ClusterAdminClient +) + +func getClient() *bigtable.Client { + if client == nil { + var err error + client, err = bigtable.NewClient(context.Background(), config.Project, config.Zone, config.Cluster) + if err != nil { + log.Fatalf("Making bigtable.Client: %v", err) + } + } + return client +} + +func getAdminClient() *bigtable.AdminClient { + if adminClient == nil { + var err error + adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Zone, config.Cluster) + if err != nil { + log.Fatalf("Making bigtable.AdminClient: %v", err) + } + } + return adminClient +} + +func getClusterAdminClient() *bigtable.ClusterAdminClient { + if clusterAdminClient == nil { + var err error + clusterAdminClient, err = bigtable.NewClusterAdminClient(context.Background(), config.Project) + if err != nil { + log.Fatalf("Making bigtable.ClusterAdminClient: %v", err) + } + } + return clusterAdminClient +} + +func main() { + var err error + config, err = cbtrc.Load() + if err != nil { + log.Fatal(err) + } + config.RegisterFlags() + + flag.Usage = usage + flag.Parse() + if err := config.CheckFlags(); err != nil { + log.Fatal(err) + } + if config.Creds != "" { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds) + } + if flag.NArg() == 0 { + usage() + os.Exit(1) + } + + if *oFlag != "" { + f, err := os.Create(*oFlag) + if err != nil { + log.Fatal(err) + } + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) + } + }() + os.Stdout = f + } + + ctx := context.Background() + for _, cmd := range commands { + if cmd.Name == flag.Arg(0) { + cmd.do(ctx, flag.Args()[1:]...) + return + } + } + log.Fatalf("Unknown command %q", flag.Arg(0)) +} + +func usage() { + fmt.Fprintf(os.Stderr, "Usage: %s [flags] ...\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "\n%s", cmdSummary) +} + +var cmdSummary string // generated in init, below + +func init() { + var buf bytes.Buffer + tw := tabwriter.NewWriter(&buf, 10, 8, 4, '\t', 0) + for _, cmd := range commands { + fmt.Fprintf(tw, "cbt %s\t%s\n", cmd.Name, cmd.Desc) + } + tw.Flush() + buf.WriteString(configHelp) + cmdSummary = buf.String() +} + +var configHelp = ` +For convenience, values of the -project, -zone, -cluster and -creds flags +may be specified in ` + cbtrc.Filename() + ` in this format: + project = my-project-123 + zone = us-central1-b + cluster = my-cluster + creds = path-to-account-key.json +All values are optional, and all will be overridden by flags. +` + +var commands = []struct { + Name, Desc string + do func(context.Context, ...string) + Usage string +}{ + { + Name: "count", + Desc: "Count rows in a table", + do: doCount, + Usage: "cbt count ", + }, + { + Name: "createfamily", + Desc: "Create a column family", + do: doCreateFamily, + Usage: "cbt createfamily
", + }, + { + Name: "createtable", + Desc: "Create a table", + do: doCreateTable, + Usage: "cbt createtable
", + }, + { + Name: "deletefamily", + Desc: "Delete a column family", + do: doDeleteFamily, + Usage: "cbt deletefamily
", + }, + { + Name: "deleterow", + Desc: "Delete a row", + do: doDeleteRow, + Usage: "cbt deleterow
", + }, + { + Name: "deletetable", + Desc: "Delete a table", + do: doDeleteTable, + Usage: "cbt deletetable
", + }, + { + Name: "doc", + Desc: "Print documentation for cbt", + do: doDoc, + Usage: "cbt doc", + }, + { + Name: "help", + Desc: "Print help text", + do: doHelp, + Usage: "cbt help [command]", + }, + { + Name: "listclusters", + Desc: "List clusters in a project", + do: doListClusters, + Usage: "cbt listclusters", + }, + { + Name: "lookup", + Desc: "Read from a single row", + do: doLookup, + Usage: "cbt lookup
", + }, + { + Name: "ls", + Desc: "List tables and column families", + do: doLS, + Usage: "cbt ls List tables\n" + + "cbt ls
List column families in
", + }, + { + Name: "read", + Desc: "Read rows", + do: doRead, + Usage: "cbt read
[start=] [limit=] [prefix=]\n" + + " start= Start reading at this row\n" + + " limit= Stop reading before this row\n" + + " prefix= Read rows with this prefix\n", + }, + { + Name: "set", + Desc: "Set value of a cell", + do: doSet, + Usage: "cbt set
family:column=val[@ts] ...\n" + + " family:column=val[@ts] may be repeated to set multiple cells.\n" + + "\n" + + " ts is an optional integer timestamp.\n" + + " If it cannot be parsed, the `@ts` part will be\n" + + " interpreted as part of the value.", + }, + /* TODO(dsymonds): Re-enable when there's a ClusterAdmin API. + { + Name: "setclustersize", + Desc: "Set size of a cluster", + do: doSetClusterSize, + Usage: "cbt setclustersize ", + }, + */ +} + +func doCount(ctx context.Context, args ...string) { + if len(args) != 1 { + log.Fatal("usage: cbt count
") + } + tbl := getClient().Open(args[0]) + + n := 0 + err := tbl.ReadRows(ctx, bigtable.InfiniteRange(""), func(_ bigtable.Row) bool { + n++ + return true + }, bigtable.RowFilter(bigtable.StripValueFilter())) + if err != nil { + log.Fatalf("Reading rows: %v", err) + } + fmt.Println(n) +} + +func doCreateFamily(ctx context.Context, args ...string) { + if len(args) != 2 { + log.Fatal("usage: cbt createfamily
") + } + err := getAdminClient().CreateColumnFamily(ctx, args[0], args[1]) + if err != nil { + log.Fatalf("Creating column family: %v", err) + } +} + +func doCreateTable(ctx context.Context, args ...string) { + if len(args) != 1 { + log.Fatal("usage: cbt createtable
") + } + err := getAdminClient().CreateTable(ctx, args[0]) + if err != nil { + log.Fatalf("Creating table: %v", err) + } +} + +func doDeleteFamily(ctx context.Context, args ...string) { + if len(args) != 2 { + log.Fatal("usage: cbt deletefamily
") + } + err := getAdminClient().DeleteColumnFamily(ctx, args[0], args[1]) + if err != nil { + log.Fatalf("Deleting column family: %v", err) + } +} + +func doDeleteRow(ctx context.Context, args ...string) { + if len(args) != 2 { + log.Fatal("usage: cbt deleterow
") + } + tbl := getClient().Open(args[0]) + mut := bigtable.NewMutation() + mut.DeleteRow() + if err := tbl.Apply(ctx, args[1], mut); err != nil { + log.Fatalf("Deleting row: %v", err) + } +} + +func doDeleteTable(ctx context.Context, args ...string) { + if len(args) != 1 { + log.Fatalf("Can't do `cbt deletetable %s`", args) + } + err := getAdminClient().DeleteTable(ctx, args[0]) + if err != nil { + log.Fatalf("Deleting table: %v", err) + } +} + +// to break circular dependencies +var ( + doDocFn func(ctx context.Context, args ...string) + doHelpFn func(ctx context.Context, args ...string) +) + +func init() { + doDocFn = doDocReal + doHelpFn = doHelpReal +} + +func doDoc(ctx context.Context, args ...string) { doDocFn(ctx, args...) } +func doHelp(ctx context.Context, args ...string) { doHelpFn(ctx, args...) } + +func doDocReal(ctx context.Context, args ...string) { + data := map[string]interface{}{ + "Commands": commands, + } + var buf bytes.Buffer + if err := docTemplate.Execute(&buf, data); err != nil { + log.Fatalf("Bad doc template: %v", err) + } + out, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatalf("Bad doc output: %v", err) + } + os.Stdout.Write(out) +} + +var docTemplate = template.Must(template.New("doc").Funcs(template.FuncMap{ + "indent": func(s, ind string) string { + ss := strings.Split(s, "\n") + for i, p := range ss { + ss[i] = ind + p + } + return strings.Join(ss, "\n") + }, +}). + Parse(` +// DO NOT EDIT. THIS IS AUTOMATICALLY GENERATED. +// Run "go generate" to regenerate. +//go:generate go run cbt.go -o cbtdoc.go doc + +/* +Cbt is a tool for doing basic interactions with Cloud Bigtable. + +Usage: + + cbt [options] command [arguments] + +The commands are: +{{range .Commands}} + {{printf "%-25s %s" .Name .Desc}}{{end}} + +Use "cbt help " for more information about a command. + +{{range .Commands}} +{{.Desc}} + +Usage: +{{indent .Usage "\t"}} + + + +{{end}} +*/ +package main +`)) + +func doHelpReal(ctx context.Context, args ...string) { + if len(args) == 0 { + fmt.Print(cmdSummary) + return + } + for _, cmd := range commands { + if cmd.Name == args[0] { + fmt.Println(cmd.Usage) + return + } + } + log.Fatalf("Don't know command %q", args[0]) +} + +func doListClusters(ctx context.Context, args ...string) { + if len(args) != 0 { + log.Fatalf("usage: cbt listclusters") + } + cis, err := getClusterAdminClient().Clusters(ctx) + if err != nil { + log.Fatalf("Getting list of clusters: %v", err) + } + tw := tabwriter.NewWriter(os.Stdout, 10, 8, 4, '\t', 0) + fmt.Fprintf(tw, "Cluster Name\tZone\tInfo\n") + fmt.Fprintf(tw, "------------\t----\t----\n") + for _, ci := range cis { + fmt.Fprintf(tw, "%s\t%s\t%s (%d serve nodes)\n", ci.Name, ci.Zone, ci.DisplayName, ci.ServeNodes) + } + tw.Flush() +} + +func doLookup(ctx context.Context, args ...string) { + if len(args) != 2 { + log.Fatalf("usage: cbt lookup
") + } + table, row := args[0], args[1] + tbl := getClient().Open(table) + r, err := tbl.ReadRow(ctx, row) + if err != nil { + log.Fatalf("Reading row: %v", err) + } + printRow(r) +} + +func printRow(r bigtable.Row) { + fmt.Println(strings.Repeat("-", 40)) + fmt.Println(r.Key()) + + var fams []string + for fam := range r { + fams = append(fams, fam) + } + sort.Strings(fams) + for _, fam := range fams { + ris := r[fam] + sort.Sort(byColumn(ris)) + for _, ri := range ris { + ts := time.Unix(0, int64(ri.Timestamp)*1e3) + fmt.Printf(" %-40s @ %s\n", ri.Column, ts.Format("2006/01/02-15:04:05.000000")) + fmt.Printf(" %q\n", ri.Value) + } + } +} + +type byColumn []bigtable.ReadItem + +func (b byColumn) Len() int { return len(b) } +func (b byColumn) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byColumn) Less(i, j int) bool { return b[i].Column < b[j].Column } + +func doLS(ctx context.Context, args ...string) { + switch len(args) { + default: + log.Fatalf("Can't do `cbt ls %s`", args) + case 0: + tables, err := getAdminClient().Tables(ctx) + if err != nil { + log.Fatalf("Getting list of tables: %v", err) + } + sort.Strings(tables) + for _, table := range tables { + fmt.Println(table) + } + case 1: + table := args[0] + ti, err := getAdminClient().TableInfo(ctx, table) + if err != nil { + log.Fatalf("Getting table info: %v", err) + } + sort.Strings(ti.Families) + for _, fam := range ti.Families { + fmt.Println(fam) + } + } +} + +func doRead(ctx context.Context, args ...string) { + if len(args) < 1 { + log.Fatalf("usage: cbt read
[args ...]") + } + tbl := getClient().Open(args[0]) + + parsed := make(map[string]string) + for _, arg := range args[1:] { + i := strings.Index(arg, "=") + if i < 0 { + log.Fatalf("Bad arg %q", arg) + } + key, val := arg[:i], arg[i+1:] + switch key { + default: + log.Fatalf("Unknown arg key %q", key) + case "start", "limit", "prefix": + parsed[key] = val + } + } + if (parsed["start"] != "" || parsed["limit"] != "") && parsed["prefix"] != "" { + log.Fatal(`"start"/"limit" may not be mixed with "prefix"`) + } + + var rr bigtable.RowRange + if start, limit := parsed["start"], parsed["limit"]; limit != "" { + rr = bigtable.NewRange(start, limit) + } else if start != "" { + rr = bigtable.InfiniteRange(start) + } + if prefix := parsed["prefix"]; prefix != "" { + rr = bigtable.PrefixRange(prefix) + } + + // TODO(dsymonds): Support filters. + err := tbl.ReadRows(ctx, rr, func(r bigtable.Row) bool { + printRow(r) + return true + }) + if err != nil { + log.Fatalf("Reading rows: %v", err) + } +} + +var setArg = regexp.MustCompile(`([^:]+):([^=]*)=(.*)`) + +func doSet(ctx context.Context, args ...string) { + if len(args) < 3 { + log.Fatalf("usage: cbt set
family:[column]=val[@ts] ...") + } + tbl := getClient().Open(args[0]) + row := args[1] + mut := bigtable.NewMutation() + for _, arg := range args[2:] { + m := setArg.FindStringSubmatch(arg) + if m == nil { + log.Fatalf("Bad set arg %q", arg) + } + val := m[3] + ts := bigtable.Now() + if i := strings.LastIndex(val, "@"); i >= 0 { + // Try parsing a timestamp. + n, err := strconv.ParseInt(val[i+1:], 0, 64) + if err == nil { + val = val[:i] + ts = bigtable.Timestamp(n) + } + } + mut.Set(m[1], m[2], ts, []byte(val)) + } + if err := tbl.Apply(ctx, row, mut); err != nil { + log.Fatalf("Applying mutation: %v", err) + } +} + +/* TODO(dsymonds): Re-enable when there's a ClusterAdmin API. +func doSetClusterSize(ctx context.Context, args ...string) { + if len(args) != 1 { + log.Fatalf("usage: cbt setclustersize ") + } + n, err := strconv.ParseInt(args[0], 0, 32) + if err != nil { + log.Fatalf("Bad num_nodes value %q: %v", args[0], err) + } + if err := getAdminClient().SetClusterSize(ctx, int(n)); err != nil { + log.Fatalf("Setting cluster size: %v", err) + } +} +*/ diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/cbt/cbtdoc.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/cbt/cbtdoc.go new file mode 100644 index 0000000000..0e00367225 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/cbt/cbtdoc.go @@ -0,0 +1,146 @@ +// DO NOT EDIT. THIS IS AUTOMATICALLY GENERATED. +// Run "go generate" to regenerate. +//go:generate go run cbt.go -o cbtdoc.go doc + +/* +Cbt is a tool for doing basic interactions with Cloud Bigtable. + +Usage: + + cbt [options] command [arguments] + +The commands are: + + count Count rows in a table + createfamily Create a column family + createtable Create a table + deletefamily Delete a column family + deleterow Delete a row + deletetable Delete a table + doc Print documentation for cbt + help Print help text + listclusters List clusters in a project + lookup Read from a single row + ls List tables and column families + read Read rows + set Set value of a cell + +Use "cbt help " for more information about a command. + + +Count rows in a table + +Usage: + cbt count
+ + + + +Create a column family + +Usage: + cbt createfamily
+ + + + +Create a table + +Usage: + cbt createtable
+ + + + +Delete a column family + +Usage: + cbt deletefamily
+ + + + +Delete a row + +Usage: + cbt deleterow
+ + + + +Delete a table + +Usage: + cbt deletetable
+ + + + +Print documentation for cbt + +Usage: + cbt doc + + + + +Print help text + +Usage: + cbt help [command] + + + + +List clusters in a project + +Usage: + cbt listclusters + + + + +Read from a single row + +Usage: + cbt lookup
+ + + + +List tables and column families + +Usage: + cbt ls List tables + cbt ls
List column families in
+ + + + +Read rows + +Usage: + cbt read
[start=] [limit=] [prefix=] + start= Start reading at this row + limit= Stop reading before this row + prefix= Read rows with this prefix + + + + + +Set value of a cell + +Usage: + cbt set
family:column=val[@ts] ... + family:column=val[@ts] may be repeated to set multiple cells. + + ts is an optional integer timestamp. + If it cannot be parsed, the `@ts` part will be + interpreted as part of the value. + + + + +*/ +package main diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/loadtest/loadtest.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/loadtest/loadtest.go new file mode 100644 index 0000000000..ff5c5a1510 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/loadtest/loadtest.go @@ -0,0 +1,159 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Loadtest does some load testing through the Go client library for Cloud Bigtable. +*/ +package main + +import ( + "bytes" + "flag" + "fmt" + "log" + "math/rand" + "os" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/context" + "google.golang.org/cloud/bigtable" + "google.golang.org/cloud/bigtable/internal/cbtrc" +) + +var ( + runFor = flag.Duration("run_for", 5*time.Second, "how long to run the load test for") + scratchTable = flag.String("scratch_table", "loadtest-scratch", "name of table to use; should not already exist") + + config *cbtrc.Config + client *bigtable.Client + adminClient *bigtable.AdminClient +) + +func main() { + var err error + config, err = cbtrc.Load() + if err != nil { + log.Fatal(err) + } + config.RegisterFlags() + + flag.Parse() + if err := config.CheckFlags(); err != nil { + log.Fatal(err) + } + if config.Creds != "" { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds) + } + if flag.NArg() != 0 { + flag.Usage() + os.Exit(1) + } + + log.Printf("Dialing connections...") + client, err = bigtable.NewClient(context.Background(), config.Project, config.Zone, config.Cluster) + if err != nil { + log.Fatalf("Making bigtable.Client: %v", err) + } + defer client.Close() + adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Zone, config.Cluster) + if err != nil { + log.Fatalf("Making bigtable.AdminClient: %v", err) + } + defer adminClient.Close() + + // Create a scratch table. + log.Printf("Setting up scratch table...") + if err := adminClient.CreateTable(context.Background(), *scratchTable); err != nil { + log.Fatalf("Making scratch table %q: %v", *scratchTable, err) + } + if err := adminClient.CreateColumnFamily(context.Background(), *scratchTable, "f"); err != nil { + log.Fatalf("Making scratch table column family: %v", err) + } + // Upon a successful run, delete the table. Don't bother checking for errors. + defer adminClient.DeleteTable(context.Background(), *scratchTable) + + log.Printf("Starting load test... (run for %v)", *runFor) + tbl := client.Open(*scratchTable) + sem := make(chan int, 100) // limit the number of requests happening at once + var reads, writes stats + stopTime := time.Now().Add(*runFor) + var wg sync.WaitGroup + for time.Now().Before(stopTime) { + sem <- 1 + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-sem }() + + ok := true + opStart := time.Now() + var stats *stats + defer func() { + stats.Record(ok, time.Since(opStart)) + }() + + row := fmt.Sprintf("row%d", rand.Intn(100)) // operate on 1 of 100 rows + + switch rand.Intn(10) { + default: + // read + stats = &reads + _, err := tbl.ReadRow(context.Background(), row, bigtable.RowFilter(bigtable.LatestNFilter(1))) + if err != nil { + log.Printf("Error doing read: %v", err) + ok = false + } + case 0, 1, 2, 3, 4: + // write + stats = &writes + mut := bigtable.NewMutation() + mut.Set("f", "col", bigtable.Now(), bytes.Repeat([]byte("0"), 1<<10)) // 1 KB write + if err := tbl.Apply(context.Background(), row, mut); err != nil { + log.Printf("Error doing mutation: %v", err) + ok = false + } + } + }() + } + wg.Wait() + + log.Printf("Reads (%d ok / %d tries):\n%v", reads.ok, reads.tries, newAggregate(reads.ds)) + log.Printf("Writes (%d ok / %d tries):\n%v", writes.ok, writes.tries, newAggregate(writes.ds)) +} + +var allStats int64 // atomic + +type stats struct { + mu sync.Mutex + tries, ok int + ds []time.Duration +} + +func (s *stats) Record(ok bool, d time.Duration) { + s.mu.Lock() + s.tries++ + if ok { + s.ok++ + } + s.ds = append(s.ds, d) + s.mu.Unlock() + + if n := atomic.AddInt64(&allStats, 1); n%1000 == 0 { + log.Printf("Progress: done %d ops", n) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/loadtest/stats.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/loadtest/stats.go new file mode 100644 index 0000000000..57c12da1e7 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/cmd/loadtest/stats.go @@ -0,0 +1,97 @@ +package main + +import ( + "bytes" + "fmt" + "math" + "sort" + "text/tabwriter" + "time" +) + +type byDuration []time.Duration + +func (data byDuration) Len() int { return len(data) } +func (data byDuration) Swap(i, j int) { data[i], data[j] = data[j], data[i] } +func (data byDuration) Less(i, j int) bool { return data[i] < data[j] } + +// quantile returns a value representing the kth of q quantiles. +// May alter the order of data. +func quantile(data []time.Duration, k, q int) (quantile time.Duration, ok bool) { + if len(data) < 1 { + return 0, false + } + if k > q { + return 0, false + } + if k < 0 || q < 1 { + return 0, false + } + + sort.Sort(byDuration(data)) + + if k == 0 { + return data[0], true + } + if k == q { + return data[len(data)-1], true + } + + bucketSize := float64(len(data)-1) / float64(q) + i := float64(k) * bucketSize + + lower := int(math.Trunc(i)) + var upper int + if i > float64(lower) && lower+1 < len(data) { + // If the quantile lies between two elements + upper = lower + 1 + } else { + upper = lower + } + weightUpper := i - float64(lower) + weightLower := 1 - weightUpper + return time.Duration(weightLower*float64(data[lower]) + weightUpper*float64(data[upper])), true +} + +type aggregate struct { + min, median, max time.Duration + p95, p99 time.Duration // percentiles +} + +// newAggregate constructs an aggregate from latencies. Returns nil if latencies does not contain aggregateable data. +func newAggregate(latencies []time.Duration) *aggregate { + var agg aggregate + + if len(latencies) == 0 { + return nil + } + var ok bool + if agg.min, ok = quantile(latencies, 0, 2); !ok { + return nil + } + if agg.median, ok = quantile(latencies, 1, 2); !ok { + return nil + } + if agg.max, ok = quantile(latencies, 2, 2); !ok { + return nil + } + if agg.p95, ok = quantile(latencies, 95, 100); !ok { + return nil + } + if agg.p99, ok = quantile(latencies, 99, 100); !ok { + return nil + } + return &agg +} + +func (agg *aggregate) String() string { + if agg == nil { + return "no data" + } + var buf bytes.Buffer + tw := tabwriter.NewWriter(&buf, 0, 0, 1, ' ', 0) // one-space padding + fmt.Fprintf(tw, "min:\t%v\nmedian:\t%v\nmax:\t%v\n95th percentile:\t%v\n99th percentile:\t%v\n", + agg.min, agg.median, agg.max, agg.p95, agg.p99) + tw.Flush() + return buf.String() +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/doc.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/doc.go new file mode 100644 index 0000000000..032e501dd4 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/doc.go @@ -0,0 +1,108 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package bigtable is an API to Google Cloud Bigtable. + +See https://cloud.google.com/bigtable/docs/ for general product documentation. + +Setup and Credentials + +Use NewClient or NewAdminClient to create a client that can be used to access +the data or admin APIs respectively. Both require credentials that have permission +to access the Cloud Bigtable API. + +If your program is run on Google App Engine or Google Compute Engine, using the Application Default Credentials +(https://developers.google.com/accounts/docs/application-default-credentials) +is the simplest option. Those credentials will be used by default when NewClient or NewAdminClient are called. + +To use alternate credentials, pass them to NewClient or NewAdminClient using cloud.WithTokenSource. +For instance, you can use service account credentials by visiting +https://cloud.google.com/console/project/MYPROJECT/apiui/credential, +creating a new OAuth "Client ID", storing the JSON key somewhere accessible, and writing + jsonKey, err := ioutil.ReadFile(pathToKeyFile) + ... + config, err := google.JWTConfigFromJSON(jsonKey, bigtable.Scope) // or bigtable.AdminScope, etc. + ... + client, err := bigtable.NewClient(ctx, project, zone, cluster, cloud.WithTokenSource(config.TokenSource(ctx))) + ... +Here, `google` means the golang.org/x/oauth2/google package +and `cloud` means the google.golang.org/cloud package. + +Reading + +The principal way to read from a Bigtable is to use the ReadRows method on *Table. +A RowRange specifies a contiguous portion of a table. A Filter may be provided through +RowFilter to limit or transform the data that is returned. + tbl := client.Open("mytable") + ... + // Read all the rows starting with "com.google.", + // but only fetch the columns in the "links" family. + rr := bigtable.PrefixRange("com.google.") + err := tbl.ReadRows(ctx, rr, func(r Row) bool { + // do something with r + return true // keep going + }, bigtable.RowFilter(bigtable.FamilyFilter("links"))) + ... + +To read a single row, use the ReadRow helper method. + r, err := tbl.ReadRow(ctx, "com.google.cloud") // "com.google.cloud" is the entire row key + ... + +Writing + +This API exposes two distinct forms of writing to a Bigtable: a Mutation and a ReadModifyWrite. +The former expresses idempotent operations. +The latter expresses non-idempotent operations and returns the new values of updated cells. +These operations are performed by creating a Mutation or ReadModifyWrite (with NewMutation or NewReadModifyWrite), +building up one or more operations on that, and then using the Apply or ApplyReadModifyWrite +methods on a Table. + +For instance, to set a couple of cells in a table, + tbl := client.Open("mytable") + mut := bigtable.NewMutation() + mut.Set("links", "maps.google.com", bigtable.Now(), []byte("1")) + mut.Set("links", "golang.org", bigtable.Now(), []byte("1")) + err := tbl.Apply(ctx, "com.google.cloud", mut) + ... + +To increment an encoded value in one cell, + tbl := client.Open("mytable") + rmw := bigtable.NewReadModifyWrite() + rmw.Increment("links", "golang.org", 12) // add 12 to the cell in column "links:golang.org" + r, err := tbl.ApplyReadModifyWrite(ctx, "com.google.cloud", rmw) + ... +*/ +package bigtable // import "google.golang.org/cloud/bigtable" + +// Scope constants for authentication credentials. +// These should be used when using credential creation functions such as oauth.NewServiceAccountFromFile. +const ( + // Scope is the OAuth scope for Cloud Bigtable data operations. + Scope = "https://www.googleapis.com/auth/bigtable.data" + // ReadonlyScope is the OAuth scope for Cloud Bigtable read-only data operations. + ReadonlyScope = "https://www.googleapis.com/auth/bigtable.readonly" + + // AdminScope is the OAuth scope for Cloud Bigtable table admin operations. + AdminScope = "https://www.googleapis.com/auth/bigtable.admin.table" + + // ClusterAdminScope is the OAuth scope for Cloud Bigtable cluster admin operations. + ClusterAdminScope = "https://www.googleapis.com/auth/bigtable.admin.cluster" +) + +// clientUserAgent identifies the version of this package. +// It should be bumped upon significant changes only. +const clientUserAgent = "cbt-go/20150727" diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/filter.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/filter.go new file mode 100644 index 0000000000..dbe247b1bc --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/filter.go @@ -0,0 +1,156 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bigtable + +import ( + "fmt" + "strings" + + btdpb "google.golang.org/cloud/bigtable/internal/data_proto" +) + +// A Filter represents a row filter. +type Filter interface { + String() string + proto() *btdpb.RowFilter +} + +// ChainFilters returns a filter that applies a sequence of filters. +func ChainFilters(sub ...Filter) Filter { return chainFilter{sub} } + +type chainFilter struct { + sub []Filter +} + +func (cf chainFilter) String() string { + var ss []string + for _, sf := range cf.sub { + ss = append(ss, sf.String()) + } + return "(" + strings.Join(ss, " | ") + ")" +} + +func (cf chainFilter) proto() *btdpb.RowFilter { + chain := &btdpb.RowFilter_Chain{} + for _, sf := range cf.sub { + chain.Filters = append(chain.Filters, sf.proto()) + } + return &btdpb.RowFilter{ + Filter: &btdpb.RowFilter_Chain_{chain}, + } +} + +// InterleaveFilters returns a filter that applies a set of filters in parallel +// and interleaves the results. +func InterleaveFilters(sub ...Filter) Filter { return interleaveFilter{sub} } + +type interleaveFilter struct { + sub []Filter +} + +func (ilf interleaveFilter) String() string { + var ss []string + for _, sf := range ilf.sub { + ss = append(ss, sf.String()) + } + return "(" + strings.Join(ss, " + ") + ")" +} + +func (ilf interleaveFilter) proto() *btdpb.RowFilter { + inter := &btdpb.RowFilter_Interleave{} + for _, sf := range ilf.sub { + inter.Filters = append(inter.Filters, sf.proto()) + } + return &btdpb.RowFilter{ + Filter: &btdpb.RowFilter_Interleave_{inter}, + } +} + +// RowKeyFilter returns a filter that matches cells from rows whose +// key matches the provided RE2 pattern. +// See https://github.com/google/re2/wiki/Syntax for the accepted syntax. +func RowKeyFilter(pattern string) Filter { return rowKeyFilter(pattern) } + +type rowKeyFilter string + +func (rkf rowKeyFilter) String() string { return fmt.Sprintf("row(%s)", string(rkf)) } + +func (rkf rowKeyFilter) proto() *btdpb.RowFilter { + return &btdpb.RowFilter{Filter: &btdpb.RowFilter_RowKeyRegexFilter{[]byte(rkf)}} +} + +// FamilyFilter returns a filter that matches cells whose family name +// matches the provided RE2 pattern. +// See https://github.com/google/re2/wiki/Syntax for the accepted syntax. +func FamilyFilter(pattern string) Filter { return familyFilter(pattern) } + +type familyFilter string + +func (ff familyFilter) String() string { return fmt.Sprintf("col(%s:)", string(ff)) } + +func (ff familyFilter) proto() *btdpb.RowFilter { + return &btdpb.RowFilter{Filter: &btdpb.RowFilter_FamilyNameRegexFilter{string(ff)}} +} + +// ColumnFilter returns a filter that matches cells whose column name +// matches the provided RE2 pattern. +// See https://github.com/google/re2/wiki/Syntax for the accepted syntax. +func ColumnFilter(pattern string) Filter { return columnFilter(pattern) } + +type columnFilter string + +func (cf columnFilter) String() string { return fmt.Sprintf("col(.*:%s)", string(cf)) } + +func (cf columnFilter) proto() *btdpb.RowFilter { + return &btdpb.RowFilter{Filter: &btdpb.RowFilter_ColumnQualifierRegexFilter{[]byte(cf)}} +} + +// ValueFilter returns a filter that matches cells whose value +// matches the provided RE2 pattern. +// See https://github.com/google/re2/wiki/Syntax for the accepted syntax. +func ValueFilter(pattern string) Filter { return valueFilter(pattern) } + +type valueFilter string + +func (vf valueFilter) String() string { return fmt.Sprintf("value_match(%s)", string(vf)) } + +func (vf valueFilter) proto() *btdpb.RowFilter { + return &btdpb.RowFilter{Filter: &btdpb.RowFilter_ValueRegexFilter{[]byte(vf)}} +} + +// LatestNFilter returns a filter that matches the most recent N cells in each column. +func LatestNFilter(n int) Filter { return latestNFilter(n) } + +type latestNFilter int32 + +func (lnf latestNFilter) String() string { return fmt.Sprintf("col(*,%d)", lnf) } + +func (lnf latestNFilter) proto() *btdpb.RowFilter { + return &btdpb.RowFilter{Filter: &btdpb.RowFilter_CellsPerColumnLimitFilter{int32(lnf)}} +} + +// StripValueFilter returns a filter that replaces each value with the empty string. +func StripValueFilter() Filter { return stripValueFilter{} } + +type stripValueFilter struct{} + +func (stripValueFilter) String() string { return "strip_value()" } +func (stripValueFilter) proto() *btdpb.RowFilter { + return &btdpb.RowFilter{Filter: &btdpb.RowFilter_StripValueTransformer{true}} +} + +// TODO(dsymonds): More filters: cond, col/ts/value range, sampling diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/gc.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/gc.go new file mode 100644 index 0000000000..84499fcd27 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/gc.go @@ -0,0 +1,131 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bigtable + +import ( + "fmt" + "strings" + "time" + + durpb "google.golang.org/cloud/bigtable/internal/duration_proto" + bttdpb "google.golang.org/cloud/bigtable/internal/table_data_proto" +) + +// A GCPolicy represents a rule that determines which cells are eligible for garbage collection. +type GCPolicy interface { + String() string + proto() *bttdpb.GcRule +} + +// IntersectionPolicy returns a GC policy that only applies when all its sub-policies apply. +func IntersectionPolicy(sub ...GCPolicy) GCPolicy { return intersectionPolicy{sub} } + +type intersectionPolicy struct { + sub []GCPolicy +} + +func (ip intersectionPolicy) String() string { + var ss []string + for _, sp := range ip.sub { + ss = append(ss, sp.String()) + } + return "(" + strings.Join(ss, " && ") + ")" +} + +func (ip intersectionPolicy) proto() *bttdpb.GcRule { + inter := &bttdpb.GcRule_Intersection{} + for _, sp := range ip.sub { + inter.Rules = append(inter.Rules, sp.proto()) + } + return &bttdpb.GcRule{ + Rule: &bttdpb.GcRule_Intersection_{inter}, + } +} + +// UnionPolicy returns a GC policy that applies when any of its sub-policies apply. +func UnionPolicy(sub ...GCPolicy) GCPolicy { return unionPolicy{sub} } + +type unionPolicy struct { + sub []GCPolicy +} + +func (up unionPolicy) String() string { + var ss []string + for _, sp := range up.sub { + ss = append(ss, sp.String()) + } + return "(" + strings.Join(ss, " || ") + ")" +} + +func (up unionPolicy) proto() *bttdpb.GcRule { + union := &bttdpb.GcRule_Union{} + for _, sp := range up.sub { + union.Rules = append(union.Rules, sp.proto()) + } + return &bttdpb.GcRule{ + Rule: &bttdpb.GcRule_Union_{union}, + } +} + +// MaxVersionsPolicy returns a GC policy that applies to all versions of a cell +// except for the most recent n. +func MaxVersionsPolicy(n int) GCPolicy { return maxVersionsPolicy(n) } + +type maxVersionsPolicy int + +func (mvp maxVersionsPolicy) String() string { return fmt.Sprintf("versions() > %d", int(mvp)) } + +func (mvp maxVersionsPolicy) proto() *bttdpb.GcRule { + return &bttdpb.GcRule{Rule: &bttdpb.GcRule_MaxNumVersions{int32(mvp)}} +} + +// MaxAgePolicy returns a GC policy that applies to all cells +// older than the given age. +func MaxAgePolicy(d time.Duration) GCPolicy { return maxAgePolicy(d) } + +type maxAgePolicy time.Duration + +var units = []struct { + d time.Duration + suffix string +}{ + {24 * time.Hour, "d"}, + {time.Hour, "h"}, + {time.Minute, "m"}, +} + +func (ma maxAgePolicy) String() string { + d := time.Duration(ma) + for _, u := range units { + if d%u.d == 0 { + return fmt.Sprintf("age() > %d%s", d/u.d, u.suffix) + } + } + return fmt.Sprintf("age() > %d", d/time.Microsecond) +} + +func (ma maxAgePolicy) proto() *bttdpb.GcRule { + // This doesn't handle overflows, etc. + // Fix this if people care about GC policies over 290 years. + ns := time.Duration(ma).Nanoseconds() + return &bttdpb.GcRule{ + Rule: &bttdpb.GcRule_MaxAge{&durpb.Duration{ + Seconds: ns / 1e9, + Nanos: int32(ns % 1e9), + }}, + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cbtrc/cbtrc.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cbtrc/cbtrc.go new file mode 100644 index 0000000000..fec4ab3fd2 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cbtrc/cbtrc.go @@ -0,0 +1,105 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cbtrc encapsulates common code for reading .cbtrc files. +package cbtrc + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +// Config represents a configuration. +type Config struct { + Project, Zone, Cluster string // required + Creds string // optional +} + +// RegisterFlags registers a set of standard flags for this config. +// It should be called before flag.Parse. +func (c *Config) RegisterFlags() { + flag.StringVar(&c.Project, "project", c.Project, "project ID") + flag.StringVar(&c.Zone, "zone", c.Zone, "CBT zone") + flag.StringVar(&c.Cluster, "cluster", c.Cluster, "CBT cluster") + flag.StringVar(&c.Creds, "creds", c.Creds, "if set, use application credentials in this file") +} + +// CheckFlags checks that the required config values are set. +func (c *Config) CheckFlags() error { + var missing []string + if c.Project == "" { + missing = append(missing, "-project") + } + if c.Zone == "" { + missing = append(missing, "-zone") + } + if c.Cluster == "" { + missing = append(missing, "-cluster") + } + if len(missing) > 0 { + return fmt.Errorf("Missing %s", strings.Join(missing, " and ")) + } + return nil +} + +// Filename returns the filename consulted for standard configuration. +func Filename() string { + // TODO(dsymonds): Might need tweaking for Windows. + return filepath.Join(os.Getenv("HOME"), ".cbtrc") +} + +// Load loads a .cbtrc file. +// If the file is not present, an empty config is returned. +func Load() (*Config, error) { + filename := Filename() + data, err := ioutil.ReadFile(filename) + if err != nil { + // silent fail if the file isn't there + if os.IsNotExist(err) { + return &Config{}, nil + } + return nil, fmt.Errorf("Reading %s: %v", filename, err) + } + c := new(Config) + s := bufio.NewScanner(bytes.NewReader(data)) + for s.Scan() { + line := s.Text() + i := strings.Index(line, "=") + if i < 0 { + return nil, fmt.Errorf("Bad line in %s: %q", filename, line) + } + key, val := strings.TrimSpace(line[:i]), strings.TrimSpace(line[i+1:]) + switch key { + default: + return nil, fmt.Errorf("Unknown key in %s: %q", filename, key) + case "project": + c.Project = val + case "zone": + c.Zone = val + case "cluster": + c.Cluster = val + case "creds": + c.Creds = val + } + } + return c, s.Err() +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.pb.go new file mode 100644 index 0000000000..4d5a27c765 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.pb.go @@ -0,0 +1,119 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto +// DO NOT EDIT! + +/* +Package google_bigtable_admin_cluster_v1 is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto + +It has these top-level messages: + Zone + Cluster +*/ +package google_bigtable_admin_cluster_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type StorageType int32 + +const ( + // The storage type used is unspecified. + StorageType_STORAGE_UNSPECIFIED StorageType = 0 + // Data will be stored in SSD, providing low and consistent latencies. + StorageType_STORAGE_SSD StorageType = 1 +) + +var StorageType_name = map[int32]string{ + 0: "STORAGE_UNSPECIFIED", + 1: "STORAGE_SSD", +} +var StorageType_value = map[string]int32{ + "STORAGE_UNSPECIFIED": 0, + "STORAGE_SSD": 1, +} + +func (x StorageType) String() string { + return proto.EnumName(StorageType_name, int32(x)) +} + +// Possible states of a zone. +type Zone_Status int32 + +const ( + // The state of the zone is unknown or unspecified. + Zone_UNKNOWN Zone_Status = 0 + // The zone is in a good state. + Zone_OK Zone_Status = 1 + // The zone is down for planned maintenance. + Zone_PLANNED_MAINTENANCE Zone_Status = 2 + // The zone is down for emergency or unplanned maintenance. + Zone_EMERGENCY_MAINENANCE Zone_Status = 3 +) + +var Zone_Status_name = map[int32]string{ + 0: "UNKNOWN", + 1: "OK", + 2: "PLANNED_MAINTENANCE", + 3: "EMERGENCY_MAINENANCE", +} +var Zone_Status_value = map[string]int32{ + "UNKNOWN": 0, + "OK": 1, + "PLANNED_MAINTENANCE": 2, + "EMERGENCY_MAINENANCE": 3, +} + +func (x Zone_Status) String() string { + return proto.EnumName(Zone_Status_name, int32(x)) +} + +// A physical location in which a particular project can allocate Cloud BigTable +// resources. +type Zone struct { + // A permanent unique identifier for the zone. + // Values are of the form projects//zones/[a-z][-a-z0-9]* + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The name of this zone as it appears in UIs. + DisplayName string `protobuf:"bytes,2,opt,name=display_name" json:"display_name,omitempty"` + // The current state of this zone. + Status Zone_Status `protobuf:"varint,3,opt,name=status,enum=google.bigtable.admin.cluster.v1.Zone_Status" json:"status,omitempty"` +} + +func (m *Zone) Reset() { *m = Zone{} } +func (m *Zone) String() string { return proto.CompactTextString(m) } +func (*Zone) ProtoMessage() {} + +// An isolated set of Cloud BigTable resources on which tables can be hosted. +type Cluster struct { + // A permanent unique identifier for the cluster. For technical reasons, the + // zone in which the cluster resides is included here. + // Values are of the form + // projects//zones//clusters/[a-z][-a-z0-9]* + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The descriptive name for this cluster as it appears in UIs. + // Must be unique per zone. + DisplayName string `protobuf:"bytes,4,opt,name=display_name" json:"display_name,omitempty"` + // The number of serve nodes allocated to this cluster. + ServeNodes int32 `protobuf:"varint,5,opt,name=serve_nodes" json:"serve_nodes,omitempty"` + // What storage type to use for tables in this cluster. Only configurable at + // cluster creation time. If unspecified, STORAGE_SSD will be used. + DefaultStorageType StorageType `protobuf:"varint,8,opt,name=default_storage_type,enum=google.bigtable.admin.cluster.v1.StorageType" json:"default_storage_type,omitempty"` +} + +func (m *Cluster) Reset() { *m = Cluster{} } +func (m *Cluster) String() string { return proto.CompactTextString(m) } +func (*Cluster) ProtoMessage() {} + +func init() { + proto.RegisterEnum("google.bigtable.admin.cluster.v1.StorageType", StorageType_name, StorageType_value) + proto.RegisterEnum("google.bigtable.admin.cluster.v1.Zone_Status", Zone_Status_name, Zone_Status_value) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto new file mode 100644 index 0000000000..af39559b31 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto @@ -0,0 +1,89 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.cluster.v1; + + +option java_multiple_files = true; +option java_outer_classname = "BigtableClusterDataProto"; +option java_package = "com.google.bigtable.admin.cluster.v1"; + + +// A physical location in which a particular project can allocate Cloud BigTable +// resources. +message Zone { + // Possible states of a zone. + enum Status { + // The state of the zone is unknown or unspecified. + UNKNOWN = 0; + + // The zone is in a good state. + OK = 1; + + // The zone is down for planned maintenance. + PLANNED_MAINTENANCE = 2; + + // The zone is down for emergency or unplanned maintenance. + EMERGENCY_MAINENANCE = 3; + } + + // A permanent unique identifier for the zone. + // Values are of the form projects//zones/[a-z][-a-z0-9]* + string name = 1; + + // The name of this zone as it appears in UIs. + string display_name = 2; + + // The current state of this zone. + Status status = 3; +} + +// An isolated set of Cloud BigTable resources on which tables can be hosted. +message Cluster { + // A permanent unique identifier for the cluster. For technical reasons, the + // zone in which the cluster resides is included here. + // Values are of the form + // projects//zones//clusters/[a-z][-a-z0-9]* + string name = 1; + + // If this cluster has been deleted, the time at which its backup will + // be irrevocably destroyed. Omitted otherwise. + // This cannot be set directly, only through DeleteCluster. + + // The operation currently running on the cluster, if any. + // This cannot be set directly, only through CreateCluster, UpdateCluster, + // or UndeleteCluster. Calls to these methods will be rejected if + // "current_operation" is already set. + + // The descriptive name for this cluster as it appears in UIs. + // Must be unique per zone. + string display_name = 4; + + // The number of serve nodes allocated to this cluster. + int32 serve_nodes = 5; + + // What storage type to use for tables in this cluster. Only configurable at + // cluster creation time. If unspecified, STORAGE_SSD will be used. + StorageType default_storage_type = 8; +} + +enum StorageType { + // The storage type used is unspecified. + STORAGE_UNSPECIFIED = 0; + + // Data will be stored in SSD, providing low and consistent latencies. + STORAGE_SSD = 1; +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.pb.go new file mode 100644 index 0000000000..8de4500a38 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.pb.go @@ -0,0 +1,331 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.proto +// DO NOT EDIT! + +package google_bigtable_admin_cluster_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_bigtable_admin_cluster_v11 "google.golang.org/cloud/bigtable/internal/cluster_data_proto" +import google_protobuf "google.golang.org/cloud/bigtable/internal/empty" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// Client API for BigtableClusterService service + +type BigtableClusterServiceClient interface { + // Lists the supported zones for the given project. + ListZones(ctx context.Context, in *ListZonesRequest, opts ...grpc.CallOption) (*ListZonesResponse, error) + // Gets information about a particular cluster. + GetCluster(ctx context.Context, in *GetClusterRequest, opts ...grpc.CallOption) (*google_bigtable_admin_cluster_v11.Cluster, error) + // Lists all clusters in the given project, along with any zones for which + // cluster information could not be retrieved. + ListClusters(ctx context.Context, in *ListClustersRequest, opts ...grpc.CallOption) (*ListClustersResponse, error) + // Creates a cluster and begins preparing it to begin serving. The returned + // cluster embeds as its "current_operation" a long-running operation which + // can be used to track the progress of turning up the new cluster. + // Immediately upon completion of this request: + // * The cluster will be readable via the API, with all requested attributes + // but no allocated resources. + // Until completion of the embedded operation: + // * Cancelling the operation will render the cluster immediately unreadable + // via the API. + // * All other attempts to modify or delete the cluster will be rejected. + // Upon completion of the embedded operation: + // * Billing for all successfully-allocated resources will begin (some types + // may have lower than the requested levels). + // * New tables can be created in the cluster. + // * The cluster's allocated resource levels will be readable via the API. + // The embedded operation's "metadata" field type is + // [CreateClusterMetadata][google.bigtable.admin.cluster.v1.CreateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + CreateCluster(ctx context.Context, in *CreateClusterRequest, opts ...grpc.CallOption) (*google_bigtable_admin_cluster_v11.Cluster, error) + // Updates a cluster, and begins allocating or releasing resources as + // requested. The returned cluster embeds as its "current_operation" a + // long-running operation which can be used to track the progress of updating + // the cluster. + // Immediately upon completion of this request: + // * For resource types where a decrease in the cluster's allocation has been + // requested, billing will be based on the newly-requested level. + // Until completion of the embedded operation: + // * Cancelling the operation will set its metadata's "cancelled_at_time", + // and begin restoring resources to their pre-request values. The operation + // is guaranteed to succeed at undoing all resource changes, after which + // point it will terminate with a CANCELLED status. + // * All other attempts to modify or delete the cluster will be rejected. + // * Reading the cluster via the API will continue to give the pre-request + // resource levels. + // Upon completion of the embedded operation: + // * Billing will begin for all successfully-allocated resources (some types + // may have lower than the requested levels). + // * All newly-reserved resources will be available for serving the cluster's + // tables. + // * The cluster's new resource levels will be readable via the API. + // [UpdateClusterMetadata][google.bigtable.admin.cluster.v1.UpdateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + UpdateCluster(ctx context.Context, in *google_bigtable_admin_cluster_v11.Cluster, opts ...grpc.CallOption) (*google_bigtable_admin_cluster_v11.Cluster, error) + // Marks a cluster and all of its tables for permanent deletion in 7 days. + // Immediately upon completion of the request: + // * Billing will cease for all of the cluster's reserved resources. + // * The cluster's "delete_time" field will be set 7 days in the future. + // Soon afterward: + // * All tables within the cluster will become unavailable. + // Prior to the cluster's "delete_time": + // * The cluster can be recovered with a call to UndeleteCluster. + // * All other attempts to modify or delete the cluster will be rejected. + // At the cluster's "delete_time": + // * The cluster and *all of its tables* will immediately and irrevocably + // disappear from the API, and their data will be permanently deleted. + DeleteCluster(ctx context.Context, in *DeleteClusterRequest, opts ...grpc.CallOption) (*google_protobuf.Empty, error) +} + +type bigtableClusterServiceClient struct { + cc *grpc.ClientConn +} + +func NewBigtableClusterServiceClient(cc *grpc.ClientConn) BigtableClusterServiceClient { + return &bigtableClusterServiceClient{cc} +} + +func (c *bigtableClusterServiceClient) ListZones(ctx context.Context, in *ListZonesRequest, opts ...grpc.CallOption) (*ListZonesResponse, error) { + out := new(ListZonesResponse) + err := grpc.Invoke(ctx, "/google.bigtable.admin.cluster.v1.BigtableClusterService/ListZones", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableClusterServiceClient) GetCluster(ctx context.Context, in *GetClusterRequest, opts ...grpc.CallOption) (*google_bigtable_admin_cluster_v11.Cluster, error) { + out := new(google_bigtable_admin_cluster_v11.Cluster) + err := grpc.Invoke(ctx, "/google.bigtable.admin.cluster.v1.BigtableClusterService/GetCluster", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableClusterServiceClient) ListClusters(ctx context.Context, in *ListClustersRequest, opts ...grpc.CallOption) (*ListClustersResponse, error) { + out := new(ListClustersResponse) + err := grpc.Invoke(ctx, "/google.bigtable.admin.cluster.v1.BigtableClusterService/ListClusters", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableClusterServiceClient) CreateCluster(ctx context.Context, in *CreateClusterRequest, opts ...grpc.CallOption) (*google_bigtable_admin_cluster_v11.Cluster, error) { + out := new(google_bigtable_admin_cluster_v11.Cluster) + err := grpc.Invoke(ctx, "/google.bigtable.admin.cluster.v1.BigtableClusterService/CreateCluster", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableClusterServiceClient) UpdateCluster(ctx context.Context, in *google_bigtable_admin_cluster_v11.Cluster, opts ...grpc.CallOption) (*google_bigtable_admin_cluster_v11.Cluster, error) { + out := new(google_bigtable_admin_cluster_v11.Cluster) + err := grpc.Invoke(ctx, "/google.bigtable.admin.cluster.v1.BigtableClusterService/UpdateCluster", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableClusterServiceClient) DeleteCluster(ctx context.Context, in *DeleteClusterRequest, opts ...grpc.CallOption) (*google_protobuf.Empty, error) { + out := new(google_protobuf.Empty) + err := grpc.Invoke(ctx, "/google.bigtable.admin.cluster.v1.BigtableClusterService/DeleteCluster", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for BigtableClusterService service + +type BigtableClusterServiceServer interface { + // Lists the supported zones for the given project. + ListZones(context.Context, *ListZonesRequest) (*ListZonesResponse, error) + // Gets information about a particular cluster. + GetCluster(context.Context, *GetClusterRequest) (*google_bigtable_admin_cluster_v11.Cluster, error) + // Lists all clusters in the given project, along with any zones for which + // cluster information could not be retrieved. + ListClusters(context.Context, *ListClustersRequest) (*ListClustersResponse, error) + // Creates a cluster and begins preparing it to begin serving. The returned + // cluster embeds as its "current_operation" a long-running operation which + // can be used to track the progress of turning up the new cluster. + // Immediately upon completion of this request: + // * The cluster will be readable via the API, with all requested attributes + // but no allocated resources. + // Until completion of the embedded operation: + // * Cancelling the operation will render the cluster immediately unreadable + // via the API. + // * All other attempts to modify or delete the cluster will be rejected. + // Upon completion of the embedded operation: + // * Billing for all successfully-allocated resources will begin (some types + // may have lower than the requested levels). + // * New tables can be created in the cluster. + // * The cluster's allocated resource levels will be readable via the API. + // The embedded operation's "metadata" field type is + // [CreateClusterMetadata][google.bigtable.admin.cluster.v1.CreateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + CreateCluster(context.Context, *CreateClusterRequest) (*google_bigtable_admin_cluster_v11.Cluster, error) + // Updates a cluster, and begins allocating or releasing resources as + // requested. The returned cluster embeds as its "current_operation" a + // long-running operation which can be used to track the progress of updating + // the cluster. + // Immediately upon completion of this request: + // * For resource types where a decrease in the cluster's allocation has been + // requested, billing will be based on the newly-requested level. + // Until completion of the embedded operation: + // * Cancelling the operation will set its metadata's "cancelled_at_time", + // and begin restoring resources to their pre-request values. The operation + // is guaranteed to succeed at undoing all resource changes, after which + // point it will terminate with a CANCELLED status. + // * All other attempts to modify or delete the cluster will be rejected. + // * Reading the cluster via the API will continue to give the pre-request + // resource levels. + // Upon completion of the embedded operation: + // * Billing will begin for all successfully-allocated resources (some types + // may have lower than the requested levels). + // * All newly-reserved resources will be available for serving the cluster's + // tables. + // * The cluster's new resource levels will be readable via the API. + // [UpdateClusterMetadata][google.bigtable.admin.cluster.v1.UpdateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + UpdateCluster(context.Context, *google_bigtable_admin_cluster_v11.Cluster) (*google_bigtable_admin_cluster_v11.Cluster, error) + // Marks a cluster and all of its tables for permanent deletion in 7 days. + // Immediately upon completion of the request: + // * Billing will cease for all of the cluster's reserved resources. + // * The cluster's "delete_time" field will be set 7 days in the future. + // Soon afterward: + // * All tables within the cluster will become unavailable. + // Prior to the cluster's "delete_time": + // * The cluster can be recovered with a call to UndeleteCluster. + // * All other attempts to modify or delete the cluster will be rejected. + // At the cluster's "delete_time": + // * The cluster and *all of its tables* will immediately and irrevocably + // disappear from the API, and their data will be permanently deleted. + DeleteCluster(context.Context, *DeleteClusterRequest) (*google_protobuf.Empty, error) +} + +func RegisterBigtableClusterServiceServer(s *grpc.Server, srv BigtableClusterServiceServer) { + s.RegisterService(&_BigtableClusterService_serviceDesc, srv) +} + +func _BigtableClusterService_ListZones_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(ListZonesRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableClusterServiceServer).ListZones(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableClusterService_GetCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(GetClusterRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableClusterServiceServer).GetCluster(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableClusterService_ListClusters_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(ListClustersRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableClusterServiceServer).ListClusters(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableClusterService_CreateCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(CreateClusterRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableClusterServiceServer).CreateCluster(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableClusterService_UpdateCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(google_bigtable_admin_cluster_v11.Cluster) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableClusterServiceServer).UpdateCluster(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableClusterService_DeleteCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(DeleteClusterRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableClusterServiceServer).DeleteCluster(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +var _BigtableClusterService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "google.bigtable.admin.cluster.v1.BigtableClusterService", + HandlerType: (*BigtableClusterServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ListZones", + Handler: _BigtableClusterService_ListZones_Handler, + }, + { + MethodName: "GetCluster", + Handler: _BigtableClusterService_GetCluster_Handler, + }, + { + MethodName: "ListClusters", + Handler: _BigtableClusterService_ListClusters_Handler, + }, + { + MethodName: "CreateCluster", + Handler: _BigtableClusterService_CreateCluster_Handler, + }, + { + MethodName: "UpdateCluster", + Handler: _BigtableClusterService_UpdateCluster_Handler, + }, + { + MethodName: "DeleteCluster", + Handler: _BigtableClusterService_DeleteCluster_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.proto new file mode 100644 index 0000000000..6243dcfb58 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.proto @@ -0,0 +1,118 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.cluster.v1; + +import "google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto"; +import "google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.proto"; +import "google.golang.org/cloud/bigtable/internal/empty/empty.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableClusterServicesProto"; +option java_package = "com.google.bigtable.admin.cluster.v1"; + + +// Service for managing zonal Cloud Bigtable resources. +service BigtableClusterService { + // Lists the supported zones for the given project. + rpc ListZones(ListZonesRequest) returns (ListZonesResponse) { + } + + // Gets information about a particular cluster. + rpc GetCluster(GetClusterRequest) returns (Cluster) { + } + + // Lists all clusters in the given project, along with any zones for which + // cluster information could not be retrieved. + rpc ListClusters(ListClustersRequest) returns (ListClustersResponse) { + } + + // Creates a cluster and begins preparing it to begin serving. The returned + // cluster embeds as its "current_operation" a long-running operation which + // can be used to track the progress of turning up the new cluster. + // Immediately upon completion of this request: + // * The cluster will be readable via the API, with all requested attributes + // but no allocated resources. + // Until completion of the embedded operation: + // * Cancelling the operation will render the cluster immediately unreadable + // via the API. + // * All other attempts to modify or delete the cluster will be rejected. + // Upon completion of the embedded operation: + // * Billing for all successfully-allocated resources will begin (some types + // may have lower than the requested levels). + // * New tables can be created in the cluster. + // * The cluster's allocated resource levels will be readable via the API. + // The embedded operation's "metadata" field type is + // [CreateClusterMetadata][google.bigtable.admin.cluster.v1.CreateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + rpc CreateCluster(CreateClusterRequest) returns (Cluster) { + } + + // Updates a cluster, and begins allocating or releasing resources as + // requested. The returned cluster embeds as its "current_operation" a + // long-running operation which can be used to track the progress of updating + // the cluster. + // Immediately upon completion of this request: + // * For resource types where a decrease in the cluster's allocation has been + // requested, billing will be based on the newly-requested level. + // Until completion of the embedded operation: + // * Cancelling the operation will set its metadata's "cancelled_at_time", + // and begin restoring resources to their pre-request values. The operation + // is guaranteed to succeed at undoing all resource changes, after which + // point it will terminate with a CANCELLED status. + // * All other attempts to modify or delete the cluster will be rejected. + // * Reading the cluster via the API will continue to give the pre-request + // resource levels. + // Upon completion of the embedded operation: + // * Billing will begin for all successfully-allocated resources (some types + // may have lower than the requested levels). + // * All newly-reserved resources will be available for serving the cluster's + // tables. + // * The cluster's new resource levels will be readable via the API. + // [UpdateClusterMetadata][google.bigtable.admin.cluster.v1.UpdateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + rpc UpdateCluster(Cluster) returns (Cluster) { + } + + // Marks a cluster and all of its tables for permanent deletion in 7 days. + // Immediately upon completion of the request: + // * Billing will cease for all of the cluster's reserved resources. + // * The cluster's "delete_time" field will be set 7 days in the future. + // Soon afterward: + // * All tables within the cluster will become unavailable. + // Prior to the cluster's "delete_time": + // * The cluster can be recovered with a call to UndeleteCluster. + // * All other attempts to modify or delete the cluster will be rejected. + // At the cluster's "delete_time": + // * The cluster and *all of its tables* will immediately and irrevocably + // disappear from the API, and their data will be permanently deleted. + rpc DeleteCluster(DeleteClusterRequest) returns (google.protobuf.Empty) { + } + + // Cancels the scheduled deletion of an cluster and begins preparing it to + // resume serving. The returned operation will also be embedded as the + // cluster's "current_operation". + // Immediately upon completion of this request: + // * The cluster's "delete_time" field will be unset, protecting it from + // automatic deletion. + // Until completion of the returned operation: + // * The operation cannot be cancelled. + // Upon completion of the returned operation: + // * Billing for the cluster's resources will resume. + // * All tables within the cluster will be available. + // [UndeleteClusterMetadata][google.bigtable.admin.cluster.v1.UndeleteClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.pb.go new file mode 100644 index 0000000000..c60c14a84f --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.pb.go @@ -0,0 +1,205 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.proto +// DO NOT EDIT! + +/* +Package google_bigtable_admin_cluster_v1 is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.proto + google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service.proto + +It has these top-level messages: + ListZonesRequest + ListZonesResponse + GetClusterRequest + ListClustersRequest + ListClustersResponse + CreateClusterRequest + CreateClusterMetadata + UpdateClusterMetadata + DeleteClusterRequest + UndeleteClusterRequest + UndeleteClusterMetadata +*/ +package google_bigtable_admin_cluster_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_bigtable_admin_cluster_v11 "google.golang.org/cloud/bigtable/internal/cluster_data_proto" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// Request message for BigtableClusterService.ListZones. +type ListZonesRequest struct { + // The unique name of the project for which a list of supported zones is + // requested. + // Values are of the form projects/ + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *ListZonesRequest) Reset() { *m = ListZonesRequest{} } +func (m *ListZonesRequest) String() string { return proto.CompactTextString(m) } +func (*ListZonesRequest) ProtoMessage() {} + +// Response message for BigtableClusterService.ListZones. +type ListZonesResponse struct { + // The list of requested zones. + Zones []*google_bigtable_admin_cluster_v11.Zone `protobuf:"bytes,1,rep,name=zones" json:"zones,omitempty"` +} + +func (m *ListZonesResponse) Reset() { *m = ListZonesResponse{} } +func (m *ListZonesResponse) String() string { return proto.CompactTextString(m) } +func (*ListZonesResponse) ProtoMessage() {} + +func (m *ListZonesResponse) GetZones() []*google_bigtable_admin_cluster_v11.Zone { + if m != nil { + return m.Zones + } + return nil +} + +// Request message for BigtableClusterService.GetCluster. +type GetClusterRequest struct { + // The unique name of the requested cluster. + // Values are of the form projects//zones//clusters/ + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *GetClusterRequest) Reset() { *m = GetClusterRequest{} } +func (m *GetClusterRequest) String() string { return proto.CompactTextString(m) } +func (*GetClusterRequest) ProtoMessage() {} + +// Request message for BigtableClusterService.ListClusters. +type ListClustersRequest struct { + // The unique name of the project for which a list of clusters is requested. + // Values are of the form projects/ + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *ListClustersRequest) Reset() { *m = ListClustersRequest{} } +func (m *ListClustersRequest) String() string { return proto.CompactTextString(m) } +func (*ListClustersRequest) ProtoMessage() {} + +// Response message for BigtableClusterService.ListClusters. +type ListClustersResponse struct { + // The list of requested Clusters. + Clusters []*google_bigtable_admin_cluster_v11.Cluster `protobuf:"bytes,1,rep,name=clusters" json:"clusters,omitempty"` + // The zones for which clusters could not be retrieved. + FailedZones []*google_bigtable_admin_cluster_v11.Zone `protobuf:"bytes,2,rep,name=failed_zones" json:"failed_zones,omitempty"` +} + +func (m *ListClustersResponse) Reset() { *m = ListClustersResponse{} } +func (m *ListClustersResponse) String() string { return proto.CompactTextString(m) } +func (*ListClustersResponse) ProtoMessage() {} + +func (m *ListClustersResponse) GetClusters() []*google_bigtable_admin_cluster_v11.Cluster { + if m != nil { + return m.Clusters + } + return nil +} + +func (m *ListClustersResponse) GetFailedZones() []*google_bigtable_admin_cluster_v11.Zone { + if m != nil { + return m.FailedZones + } + return nil +} + +// Request message for BigtableClusterService.CreateCluster. +type CreateClusterRequest struct { + // The unique name of the zone in which to create the cluster. + // Values are of the form projects//zones/ + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The id to be used when referring to the new cluster within its zone, + // e.g. just the "test-cluster" section of the full name + // "projects//zones//clusters/test-cluster". + ClusterId string `protobuf:"bytes,2,opt,name=cluster_id" json:"cluster_id,omitempty"` + // The cluster to create. + // The "name", "delete_time", and "current_operation" fields must be left + // blank. + Cluster *google_bigtable_admin_cluster_v11.Cluster `protobuf:"bytes,3,opt,name=cluster" json:"cluster,omitempty"` +} + +func (m *CreateClusterRequest) Reset() { *m = CreateClusterRequest{} } +func (m *CreateClusterRequest) String() string { return proto.CompactTextString(m) } +func (*CreateClusterRequest) ProtoMessage() {} + +func (m *CreateClusterRequest) GetCluster() *google_bigtable_admin_cluster_v11.Cluster { + if m != nil { + return m.Cluster + } + return nil +} + +// Metadata type for the operation returned by +// BigtableClusterService.CreateCluster. +type CreateClusterMetadata struct { + // The request which prompted the creation of this operation. + OriginalRequest *CreateClusterRequest `protobuf:"bytes,1,opt,name=original_request" json:"original_request,omitempty"` +} + +func (m *CreateClusterMetadata) Reset() { *m = CreateClusterMetadata{} } +func (m *CreateClusterMetadata) String() string { return proto.CompactTextString(m) } +func (*CreateClusterMetadata) ProtoMessage() {} + +func (m *CreateClusterMetadata) GetOriginalRequest() *CreateClusterRequest { + if m != nil { + return m.OriginalRequest + } + return nil +} + +// Metadata type for the operation returned by +// BigtableClusterService.UpdateCluster. +type UpdateClusterMetadata struct { + // The request which prompted the creation of this operation. + OriginalRequest *google_bigtable_admin_cluster_v11.Cluster `protobuf:"bytes,1,opt,name=original_request" json:"original_request,omitempty"` +} + +func (m *UpdateClusterMetadata) Reset() { *m = UpdateClusterMetadata{} } +func (m *UpdateClusterMetadata) String() string { return proto.CompactTextString(m) } +func (*UpdateClusterMetadata) ProtoMessage() {} + +func (m *UpdateClusterMetadata) GetOriginalRequest() *google_bigtable_admin_cluster_v11.Cluster { + if m != nil { + return m.OriginalRequest + } + return nil +} + +// Request message for BigtableClusterService.DeleteCluster. +type DeleteClusterRequest struct { + // The unique name of the cluster to be deleted. + // Values are of the form projects//zones//clusters/ + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *DeleteClusterRequest) Reset() { *m = DeleteClusterRequest{} } +func (m *DeleteClusterRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteClusterRequest) ProtoMessage() {} + +// Request message for BigtableClusterService.UndeleteCluster. +type UndeleteClusterRequest struct { + // The unique name of the cluster to be un-deleted. + // Values are of the form projects//zones//clusters/ + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *UndeleteClusterRequest) Reset() { *m = UndeleteClusterRequest{} } +func (m *UndeleteClusterRequest) String() string { return proto.CompactTextString(m) } +func (*UndeleteClusterRequest) ProtoMessage() {} + +// Metadata type for the operation returned by +// BigtableClusterService.UndeleteCluster. +type UndeleteClusterMetadata struct { +} + +func (m *UndeleteClusterMetadata) Reset() { *m = UndeleteClusterMetadata{} } +func (m *UndeleteClusterMetadata) String() string { return proto.CompactTextString(m) } +func (*UndeleteClusterMetadata) ProtoMessage() {} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.proto new file mode 100644 index 0000000000..2e5d4a72d7 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.proto @@ -0,0 +1,126 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.cluster.v1; + +import "google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableClusterServiceMessagesProto"; +option java_package = "com.google.bigtable.admin.cluster.v1"; + + +// Request message for BigtableClusterService.ListZones. +message ListZonesRequest { + // The unique name of the project for which a list of supported zones is + // requested. + // Values are of the form projects/ + string name = 1; +} + +// Response message for BigtableClusterService.ListZones. +message ListZonesResponse { + // The list of requested zones. + repeated Zone zones = 1; +} + +// Request message for BigtableClusterService.GetCluster. +message GetClusterRequest { + // The unique name of the requested cluster. + // Values are of the form projects//zones//clusters/ + string name = 1; +} + +// Request message for BigtableClusterService.ListClusters. +message ListClustersRequest { + // The unique name of the project for which a list of clusters is requested. + // Values are of the form projects/ + string name = 1; +} + +// Response message for BigtableClusterService.ListClusters. +message ListClustersResponse { + // The list of requested Clusters. + repeated Cluster clusters = 1; + + // The zones for which clusters could not be retrieved. + repeated Zone failed_zones = 2; +} + +// Request message for BigtableClusterService.CreateCluster. +message CreateClusterRequest { + // The unique name of the zone in which to create the cluster. + // Values are of the form projects//zones/ + string name = 1; + + // The id to be used when referring to the new cluster within its zone, + // e.g. just the "test-cluster" section of the full name + // "projects//zones//clusters/test-cluster". + string cluster_id = 2; + + // The cluster to create. + // The "name", "delete_time", and "current_operation" fields must be left + // blank. + Cluster cluster = 3; +} + +// Metadata type for the operation returned by +// BigtableClusterService.CreateCluster. +message CreateClusterMetadata { + // The request which prompted the creation of this operation. + CreateClusterRequest original_request = 1; + + // The time at which original_request was received. + + // The time at which this operation failed or was completed successfully. +} + +// Metadata type for the operation returned by +// BigtableClusterService.UpdateCluster. +message UpdateClusterMetadata { + // The request which prompted the creation of this operation. + Cluster original_request = 1; + + // The time at which original_request was received. + + // The time at which this operation was cancelled. If set, this operation is + // in the process of undoing itself (which is guaranteed to succeed) and + // cannot be cancelled again. + + // The time at which this operation failed or was completed successfully. +} + +// Request message for BigtableClusterService.DeleteCluster. +message DeleteClusterRequest { + // The unique name of the cluster to be deleted. + // Values are of the form projects//zones//clusters/ + string name = 1; +} + +// Request message for BigtableClusterService.UndeleteCluster. +message UndeleteClusterRequest { + // The unique name of the cluster to be un-deleted. + // Values are of the form projects//zones//clusters/ + string name = 1; +} + +// Metadata type for the operation returned by +// BigtableClusterService.UndeleteCluster. +message UndeleteClusterMetadata { + // The time at which the original request was received. + + // The time at which this operation failed or was completed successfully. +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.pb.go new file mode 100644 index 0000000000..c68471ac58 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.pb.go @@ -0,0 +1,1462 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto +// DO NOT EDIT! + +/* +Package google_bigtable_v1 is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto + +It has these top-level messages: + Row + Family + Column + Cell + RowRange + ColumnRange + TimestampRange + ValueRange + RowFilter + Mutation + ReadModifyWriteRule +*/ +package google_bigtable_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// Specifies the complete (requested) contents of a single row of a table. +// Rows which exceed 256MiB in size cannot be read in full. +type Row struct { + // The unique key which identifies this row within its table. This is the same + // key that's used to identify the row in, for example, a MutateRowRequest. + // May contain any non-empty byte string up to 4KiB in length. + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // May be empty, but only if the entire row is empty. + // The mutual ordering of column families is not specified. + Families []*Family `protobuf:"bytes,2,rep,name=families" json:"families,omitempty"` +} + +func (m *Row) Reset() { *m = Row{} } +func (m *Row) String() string { return proto.CompactTextString(m) } +func (*Row) ProtoMessage() {} + +func (m *Row) GetFamilies() []*Family { + if m != nil { + return m.Families + } + return nil +} + +// Specifies (some of) the contents of a single row/column family of a table. +type Family struct { + // The unique key which identifies this family within its row. This is the + // same key that's used to identify the family in, for example, a RowFilter + // which sets its "family_name_regex_filter" field. + // Must match [-_.a-zA-Z0-9]+, except that AggregatingRowProcessors may + // produce cells in a sentinel family with an empty name. + // Must be no greater than 64 characters in length. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Must not be empty. Sorted in order of increasing "qualifier". + Columns []*Column `protobuf:"bytes,2,rep,name=columns" json:"columns,omitempty"` +} + +func (m *Family) Reset() { *m = Family{} } +func (m *Family) String() string { return proto.CompactTextString(m) } +func (*Family) ProtoMessage() {} + +func (m *Family) GetColumns() []*Column { + if m != nil { + return m.Columns + } + return nil +} + +// Specifies (some of) the contents of a single row/column of a table. +type Column struct { + // The unique key which identifies this column within its family. This is the + // same key that's used to identify the column in, for example, a RowFilter + // which sets its "column_qualifier_regex_filter" field. + // May contain any byte string, including the empty string, up to 16kiB in + // length. + Qualifier []byte `protobuf:"bytes,1,opt,name=qualifier,proto3" json:"qualifier,omitempty"` + // Must not be empty. Sorted in order of decreasing "timestamp_micros". + Cells []*Cell `protobuf:"bytes,2,rep,name=cells" json:"cells,omitempty"` +} + +func (m *Column) Reset() { *m = Column{} } +func (m *Column) String() string { return proto.CompactTextString(m) } +func (*Column) ProtoMessage() {} + +func (m *Column) GetCells() []*Cell { + if m != nil { + return m.Cells + } + return nil +} + +// Specifies (some of) the contents of a single row/column/timestamp of a table. +type Cell struct { + // The cell's stored timestamp, which also uniquely identifies it within + // its column. + // Values are always expressed in microseconds, but individual tables may set + // a coarser "granularity" to further restrict the allowed values. For + // example, a table which specifies millisecond granularity will only allow + // values of "timestamp_micros" which are multiples of 1000. + TimestampMicros int64 `protobuf:"varint,1,opt,name=timestamp_micros" json:"timestamp_micros,omitempty"` + // The value stored in the cell. + // May contain any byte string, including the empty string, up to 100MiB in + // length. + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // Labels applied to the cell by a [RowFilter][google.bigtable.v1.RowFilter]. + Labels []string `protobuf:"bytes,3,rep,name=labels" json:"labels,omitempty"` +} + +func (m *Cell) Reset() { *m = Cell{} } +func (m *Cell) String() string { return proto.CompactTextString(m) } +func (*Cell) ProtoMessage() {} + +// Specifies a contiguous range of rows. +type RowRange struct { + // Inclusive lower bound. If left empty, interpreted as the empty string. + StartKey []byte `protobuf:"bytes,2,opt,name=start_key,proto3" json:"start_key,omitempty"` + // Exclusive upper bound. If left empty, interpreted as infinity. + EndKey []byte `protobuf:"bytes,3,opt,name=end_key,proto3" json:"end_key,omitempty"` +} + +func (m *RowRange) Reset() { *m = RowRange{} } +func (m *RowRange) String() string { return proto.CompactTextString(m) } +func (*RowRange) ProtoMessage() {} + +// Specifies a contiguous range of columns within a single column family. +// The range spans from : to +// :, where both bounds can be either inclusive or +// exclusive. +type ColumnRange struct { + // The name of the column family within which this range falls. + FamilyName string `protobuf:"bytes,1,opt,name=family_name" json:"family_name,omitempty"` + // The column qualifier at which to start the range (within 'column_family'). + // If neither field is set, interpreted as the empty string, inclusive. + // + // Types that are valid to be assigned to StartQualifier: + // *ColumnRange_StartQualifierInclusive + // *ColumnRange_StartQualifierExclusive + StartQualifier isColumnRange_StartQualifier `protobuf_oneof:"start_qualifier"` + // The column qualifier at which to end the range (within 'column_family'). + // If neither field is set, interpreted as the infinite string, exclusive. + // + // Types that are valid to be assigned to EndQualifier: + // *ColumnRange_EndQualifierInclusive + // *ColumnRange_EndQualifierExclusive + EndQualifier isColumnRange_EndQualifier `protobuf_oneof:"end_qualifier"` +} + +func (m *ColumnRange) Reset() { *m = ColumnRange{} } +func (m *ColumnRange) String() string { return proto.CompactTextString(m) } +func (*ColumnRange) ProtoMessage() {} + +type isColumnRange_StartQualifier interface { + isColumnRange_StartQualifier() +} +type isColumnRange_EndQualifier interface { + isColumnRange_EndQualifier() +} + +type ColumnRange_StartQualifierInclusive struct { + StartQualifierInclusive []byte `protobuf:"bytes,2,opt,name=start_qualifier_inclusive,proto3,oneof"` +} +type ColumnRange_StartQualifierExclusive struct { + StartQualifierExclusive []byte `protobuf:"bytes,3,opt,name=start_qualifier_exclusive,proto3,oneof"` +} +type ColumnRange_EndQualifierInclusive struct { + EndQualifierInclusive []byte `protobuf:"bytes,4,opt,name=end_qualifier_inclusive,proto3,oneof"` +} +type ColumnRange_EndQualifierExclusive struct { + EndQualifierExclusive []byte `protobuf:"bytes,5,opt,name=end_qualifier_exclusive,proto3,oneof"` +} + +func (*ColumnRange_StartQualifierInclusive) isColumnRange_StartQualifier() {} +func (*ColumnRange_StartQualifierExclusive) isColumnRange_StartQualifier() {} +func (*ColumnRange_EndQualifierInclusive) isColumnRange_EndQualifier() {} +func (*ColumnRange_EndQualifierExclusive) isColumnRange_EndQualifier() {} + +func (m *ColumnRange) GetStartQualifier() isColumnRange_StartQualifier { + if m != nil { + return m.StartQualifier + } + return nil +} +func (m *ColumnRange) GetEndQualifier() isColumnRange_EndQualifier { + if m != nil { + return m.EndQualifier + } + return nil +} + +func (m *ColumnRange) GetStartQualifierInclusive() []byte { + if x, ok := m.GetStartQualifier().(*ColumnRange_StartQualifierInclusive); ok { + return x.StartQualifierInclusive + } + return nil +} + +func (m *ColumnRange) GetStartQualifierExclusive() []byte { + if x, ok := m.GetStartQualifier().(*ColumnRange_StartQualifierExclusive); ok { + return x.StartQualifierExclusive + } + return nil +} + +func (m *ColumnRange) GetEndQualifierInclusive() []byte { + if x, ok := m.GetEndQualifier().(*ColumnRange_EndQualifierInclusive); ok { + return x.EndQualifierInclusive + } + return nil +} + +func (m *ColumnRange) GetEndQualifierExclusive() []byte { + if x, ok := m.GetEndQualifier().(*ColumnRange_EndQualifierExclusive); ok { + return x.EndQualifierExclusive + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ColumnRange) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _ColumnRange_OneofMarshaler, _ColumnRange_OneofUnmarshaler, []interface{}{ + (*ColumnRange_StartQualifierInclusive)(nil), + (*ColumnRange_StartQualifierExclusive)(nil), + (*ColumnRange_EndQualifierInclusive)(nil), + (*ColumnRange_EndQualifierExclusive)(nil), + } +} + +func _ColumnRange_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ColumnRange) + // start_qualifier + switch x := m.StartQualifier.(type) { + case *ColumnRange_StartQualifierInclusive: + b.EncodeVarint(2<<3 | proto.WireBytes) + b.EncodeRawBytes(x.StartQualifierInclusive) + case *ColumnRange_StartQualifierExclusive: + b.EncodeVarint(3<<3 | proto.WireBytes) + b.EncodeRawBytes(x.StartQualifierExclusive) + case nil: + default: + return fmt.Errorf("ColumnRange.StartQualifier has unexpected type %T", x) + } + // end_qualifier + switch x := m.EndQualifier.(type) { + case *ColumnRange_EndQualifierInclusive: + b.EncodeVarint(4<<3 | proto.WireBytes) + b.EncodeRawBytes(x.EndQualifierInclusive) + case *ColumnRange_EndQualifierExclusive: + b.EncodeVarint(5<<3 | proto.WireBytes) + b.EncodeRawBytes(x.EndQualifierExclusive) + case nil: + default: + return fmt.Errorf("ColumnRange.EndQualifier has unexpected type %T", x) + } + return nil +} + +func _ColumnRange_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ColumnRange) + switch tag { + case 2: // start_qualifier.start_qualifier_inclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.StartQualifier = &ColumnRange_StartQualifierInclusive{x} + return true, err + case 3: // start_qualifier.start_qualifier_exclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.StartQualifier = &ColumnRange_StartQualifierExclusive{x} + return true, err + case 4: // end_qualifier.end_qualifier_inclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.EndQualifier = &ColumnRange_EndQualifierInclusive{x} + return true, err + case 5: // end_qualifier.end_qualifier_exclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.EndQualifier = &ColumnRange_EndQualifierExclusive{x} + return true, err + default: + return false, nil + } +} + +// Specified a contiguous range of microsecond timestamps. +type TimestampRange struct { + // Inclusive lower bound. If left empty, interpreted as 0. + StartTimestampMicros int64 `protobuf:"varint,1,opt,name=start_timestamp_micros" json:"start_timestamp_micros,omitempty"` + // Exclusive upper bound. If left empty, interpreted as infinity. + EndTimestampMicros int64 `protobuf:"varint,2,opt,name=end_timestamp_micros" json:"end_timestamp_micros,omitempty"` +} + +func (m *TimestampRange) Reset() { *m = TimestampRange{} } +func (m *TimestampRange) String() string { return proto.CompactTextString(m) } +func (*TimestampRange) ProtoMessage() {} + +// Specifies a contiguous range of raw byte values. +type ValueRange struct { + // The value at which to start the range. + // If neither field is set, interpreted as the empty string, inclusive. + // + // Types that are valid to be assigned to StartValue: + // *ValueRange_StartValueInclusive + // *ValueRange_StartValueExclusive + StartValue isValueRange_StartValue `protobuf_oneof:"start_value"` + // The value at which to end the range. + // If neither field is set, interpreted as the infinite string, exclusive. + // + // Types that are valid to be assigned to EndValue: + // *ValueRange_EndValueInclusive + // *ValueRange_EndValueExclusive + EndValue isValueRange_EndValue `protobuf_oneof:"end_value"` +} + +func (m *ValueRange) Reset() { *m = ValueRange{} } +func (m *ValueRange) String() string { return proto.CompactTextString(m) } +func (*ValueRange) ProtoMessage() {} + +type isValueRange_StartValue interface { + isValueRange_StartValue() +} +type isValueRange_EndValue interface { + isValueRange_EndValue() +} + +type ValueRange_StartValueInclusive struct { + StartValueInclusive []byte `protobuf:"bytes,1,opt,name=start_value_inclusive,proto3,oneof"` +} +type ValueRange_StartValueExclusive struct { + StartValueExclusive []byte `protobuf:"bytes,2,opt,name=start_value_exclusive,proto3,oneof"` +} +type ValueRange_EndValueInclusive struct { + EndValueInclusive []byte `protobuf:"bytes,3,opt,name=end_value_inclusive,proto3,oneof"` +} +type ValueRange_EndValueExclusive struct { + EndValueExclusive []byte `protobuf:"bytes,4,opt,name=end_value_exclusive,proto3,oneof"` +} + +func (*ValueRange_StartValueInclusive) isValueRange_StartValue() {} +func (*ValueRange_StartValueExclusive) isValueRange_StartValue() {} +func (*ValueRange_EndValueInclusive) isValueRange_EndValue() {} +func (*ValueRange_EndValueExclusive) isValueRange_EndValue() {} + +func (m *ValueRange) GetStartValue() isValueRange_StartValue { + if m != nil { + return m.StartValue + } + return nil +} +func (m *ValueRange) GetEndValue() isValueRange_EndValue { + if m != nil { + return m.EndValue + } + return nil +} + +func (m *ValueRange) GetStartValueInclusive() []byte { + if x, ok := m.GetStartValue().(*ValueRange_StartValueInclusive); ok { + return x.StartValueInclusive + } + return nil +} + +func (m *ValueRange) GetStartValueExclusive() []byte { + if x, ok := m.GetStartValue().(*ValueRange_StartValueExclusive); ok { + return x.StartValueExclusive + } + return nil +} + +func (m *ValueRange) GetEndValueInclusive() []byte { + if x, ok := m.GetEndValue().(*ValueRange_EndValueInclusive); ok { + return x.EndValueInclusive + } + return nil +} + +func (m *ValueRange) GetEndValueExclusive() []byte { + if x, ok := m.GetEndValue().(*ValueRange_EndValueExclusive); ok { + return x.EndValueExclusive + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ValueRange) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _ValueRange_OneofMarshaler, _ValueRange_OneofUnmarshaler, []interface{}{ + (*ValueRange_StartValueInclusive)(nil), + (*ValueRange_StartValueExclusive)(nil), + (*ValueRange_EndValueInclusive)(nil), + (*ValueRange_EndValueExclusive)(nil), + } +} + +func _ValueRange_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ValueRange) + // start_value + switch x := m.StartValue.(type) { + case *ValueRange_StartValueInclusive: + b.EncodeVarint(1<<3 | proto.WireBytes) + b.EncodeRawBytes(x.StartValueInclusive) + case *ValueRange_StartValueExclusive: + b.EncodeVarint(2<<3 | proto.WireBytes) + b.EncodeRawBytes(x.StartValueExclusive) + case nil: + default: + return fmt.Errorf("ValueRange.StartValue has unexpected type %T", x) + } + // end_value + switch x := m.EndValue.(type) { + case *ValueRange_EndValueInclusive: + b.EncodeVarint(3<<3 | proto.WireBytes) + b.EncodeRawBytes(x.EndValueInclusive) + case *ValueRange_EndValueExclusive: + b.EncodeVarint(4<<3 | proto.WireBytes) + b.EncodeRawBytes(x.EndValueExclusive) + case nil: + default: + return fmt.Errorf("ValueRange.EndValue has unexpected type %T", x) + } + return nil +} + +func _ValueRange_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ValueRange) + switch tag { + case 1: // start_value.start_value_inclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.StartValue = &ValueRange_StartValueInclusive{x} + return true, err + case 2: // start_value.start_value_exclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.StartValue = &ValueRange_StartValueExclusive{x} + return true, err + case 3: // end_value.end_value_inclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.EndValue = &ValueRange_EndValueInclusive{x} + return true, err + case 4: // end_value.end_value_exclusive + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.EndValue = &ValueRange_EndValueExclusive{x} + return true, err + default: + return false, nil + } +} + +// Takes a row as input and produces an alternate view of the row based on +// specified rules. For example, a RowFilter might trim down a row to include +// just the cells from columns matching a given regular expression, or might +// return all the cells of a row but not their values. More complicated filters +// can be composed out of these components to express requests such as, "within +// every column of a particular family, give just the two most recent cells +// which are older than timestamp X." +// +// There are two broad categories of RowFilters (true filters and transformers), +// as well as two ways to compose simple filters into more complex ones +// (chains and interleaves). They work as follows: +// +// * True filters alter the input row by excluding some of its cells wholesale +// from the output row. An example of a true filter is the "value_regex_filter", +// which excludes cells whose values don't match the specified pattern. All +// regex true filters use RE2 syntax (https://github.com/google/re2/wiki/Syntax) +// in raw byte mode (RE2::Latin1), and are evaluated as full matches. An +// important point to keep in mind is that RE2(.) is equivalent by default to +// RE2([^\n]), meaning that it does not match newlines. When attempting to match +// an arbitrary byte, you should therefore use the escape sequence '\C', which +// may need to be further escaped as '\\C' in your client language. +// +// * Transformers alter the input row by changing the values of some of its +// cells in the output, without excluding them completely. Currently, the only +// supported transformer is the "strip_value_transformer", which replaces every +// cell's value with the empty string. +// +// * Chains and interleaves are described in more detail in the +// RowFilter.Chain and RowFilter.Interleave documentation. +// +// The total serialized size of a RowFilter message must not +// exceed 4096 bytes, and RowFilters may not be nested within each other +// (in Chains or Interleaves) to a depth of more than 20. +type RowFilter struct { + // Which of the possible RowFilter types to apply. If none are set, this + // RowFilter returns all cells in the input row. + // + // Types that are valid to be assigned to Filter: + // *RowFilter_Chain_ + // *RowFilter_Interleave_ + // *RowFilter_Condition_ + // *RowFilter_Sink + // *RowFilter_PassAllFilter + // *RowFilter_BlockAllFilter + // *RowFilter_RowKeyRegexFilter + // *RowFilter_RowSampleFilter + // *RowFilter_FamilyNameRegexFilter + // *RowFilter_ColumnQualifierRegexFilter + // *RowFilter_ColumnRangeFilter + // *RowFilter_TimestampRangeFilter + // *RowFilter_ValueRegexFilter + // *RowFilter_ValueRangeFilter + // *RowFilter_CellsPerRowOffsetFilter + // *RowFilter_CellsPerRowLimitFilter + // *RowFilter_CellsPerColumnLimitFilter + // *RowFilter_StripValueTransformer + // *RowFilter_ApplyLabelTransformer + Filter isRowFilter_Filter `protobuf_oneof:"filter"` +} + +func (m *RowFilter) Reset() { *m = RowFilter{} } +func (m *RowFilter) String() string { return proto.CompactTextString(m) } +func (*RowFilter) ProtoMessage() {} + +type isRowFilter_Filter interface { + isRowFilter_Filter() +} + +type RowFilter_Chain_ struct { + Chain *RowFilter_Chain `protobuf:"bytes,1,opt,name=chain,oneof"` +} +type RowFilter_Interleave_ struct { + Interleave *RowFilter_Interleave `protobuf:"bytes,2,opt,name=interleave,oneof"` +} +type RowFilter_Condition_ struct { + Condition *RowFilter_Condition `protobuf:"bytes,3,opt,name=condition,oneof"` +} +type RowFilter_Sink struct { + Sink bool `protobuf:"varint,16,opt,name=sink,oneof"` +} +type RowFilter_PassAllFilter struct { + PassAllFilter bool `protobuf:"varint,17,opt,name=pass_all_filter,oneof"` +} +type RowFilter_BlockAllFilter struct { + BlockAllFilter bool `protobuf:"varint,18,opt,name=block_all_filter,oneof"` +} +type RowFilter_RowKeyRegexFilter struct { + RowKeyRegexFilter []byte `protobuf:"bytes,4,opt,name=row_key_regex_filter,proto3,oneof"` +} +type RowFilter_RowSampleFilter struct { + RowSampleFilter float64 `protobuf:"fixed64,14,opt,name=row_sample_filter,oneof"` +} +type RowFilter_FamilyNameRegexFilter struct { + FamilyNameRegexFilter string `protobuf:"bytes,5,opt,name=family_name_regex_filter,oneof"` +} +type RowFilter_ColumnQualifierRegexFilter struct { + ColumnQualifierRegexFilter []byte `protobuf:"bytes,6,opt,name=column_qualifier_regex_filter,proto3,oneof"` +} +type RowFilter_ColumnRangeFilter struct { + ColumnRangeFilter *ColumnRange `protobuf:"bytes,7,opt,name=column_range_filter,oneof"` +} +type RowFilter_TimestampRangeFilter struct { + TimestampRangeFilter *TimestampRange `protobuf:"bytes,8,opt,name=timestamp_range_filter,oneof"` +} +type RowFilter_ValueRegexFilter struct { + ValueRegexFilter []byte `protobuf:"bytes,9,opt,name=value_regex_filter,proto3,oneof"` +} +type RowFilter_ValueRangeFilter struct { + ValueRangeFilter *ValueRange `protobuf:"bytes,15,opt,name=value_range_filter,oneof"` +} +type RowFilter_CellsPerRowOffsetFilter struct { + CellsPerRowOffsetFilter int32 `protobuf:"varint,10,opt,name=cells_per_row_offset_filter,oneof"` +} +type RowFilter_CellsPerRowLimitFilter struct { + CellsPerRowLimitFilter int32 `protobuf:"varint,11,opt,name=cells_per_row_limit_filter,oneof"` +} +type RowFilter_CellsPerColumnLimitFilter struct { + CellsPerColumnLimitFilter int32 `protobuf:"varint,12,opt,name=cells_per_column_limit_filter,oneof"` +} +type RowFilter_StripValueTransformer struct { + StripValueTransformer bool `protobuf:"varint,13,opt,name=strip_value_transformer,oneof"` +} +type RowFilter_ApplyLabelTransformer struct { + ApplyLabelTransformer string `protobuf:"bytes,19,opt,name=apply_label_transformer,oneof"` +} + +func (*RowFilter_Chain_) isRowFilter_Filter() {} +func (*RowFilter_Interleave_) isRowFilter_Filter() {} +func (*RowFilter_Condition_) isRowFilter_Filter() {} +func (*RowFilter_Sink) isRowFilter_Filter() {} +func (*RowFilter_PassAllFilter) isRowFilter_Filter() {} +func (*RowFilter_BlockAllFilter) isRowFilter_Filter() {} +func (*RowFilter_RowKeyRegexFilter) isRowFilter_Filter() {} +func (*RowFilter_RowSampleFilter) isRowFilter_Filter() {} +func (*RowFilter_FamilyNameRegexFilter) isRowFilter_Filter() {} +func (*RowFilter_ColumnQualifierRegexFilter) isRowFilter_Filter() {} +func (*RowFilter_ColumnRangeFilter) isRowFilter_Filter() {} +func (*RowFilter_TimestampRangeFilter) isRowFilter_Filter() {} +func (*RowFilter_ValueRegexFilter) isRowFilter_Filter() {} +func (*RowFilter_ValueRangeFilter) isRowFilter_Filter() {} +func (*RowFilter_CellsPerRowOffsetFilter) isRowFilter_Filter() {} +func (*RowFilter_CellsPerRowLimitFilter) isRowFilter_Filter() {} +func (*RowFilter_CellsPerColumnLimitFilter) isRowFilter_Filter() {} +func (*RowFilter_StripValueTransformer) isRowFilter_Filter() {} +func (*RowFilter_ApplyLabelTransformer) isRowFilter_Filter() {} + +func (m *RowFilter) GetFilter() isRowFilter_Filter { + if m != nil { + return m.Filter + } + return nil +} + +func (m *RowFilter) GetChain() *RowFilter_Chain { + if x, ok := m.GetFilter().(*RowFilter_Chain_); ok { + return x.Chain + } + return nil +} + +func (m *RowFilter) GetInterleave() *RowFilter_Interleave { + if x, ok := m.GetFilter().(*RowFilter_Interleave_); ok { + return x.Interleave + } + return nil +} + +func (m *RowFilter) GetCondition() *RowFilter_Condition { + if x, ok := m.GetFilter().(*RowFilter_Condition_); ok { + return x.Condition + } + return nil +} + +func (m *RowFilter) GetSink() bool { + if x, ok := m.GetFilter().(*RowFilter_Sink); ok { + return x.Sink + } + return false +} + +func (m *RowFilter) GetPassAllFilter() bool { + if x, ok := m.GetFilter().(*RowFilter_PassAllFilter); ok { + return x.PassAllFilter + } + return false +} + +func (m *RowFilter) GetBlockAllFilter() bool { + if x, ok := m.GetFilter().(*RowFilter_BlockAllFilter); ok { + return x.BlockAllFilter + } + return false +} + +func (m *RowFilter) GetRowKeyRegexFilter() []byte { + if x, ok := m.GetFilter().(*RowFilter_RowKeyRegexFilter); ok { + return x.RowKeyRegexFilter + } + return nil +} + +func (m *RowFilter) GetRowSampleFilter() float64 { + if x, ok := m.GetFilter().(*RowFilter_RowSampleFilter); ok { + return x.RowSampleFilter + } + return 0 +} + +func (m *RowFilter) GetFamilyNameRegexFilter() string { + if x, ok := m.GetFilter().(*RowFilter_FamilyNameRegexFilter); ok { + return x.FamilyNameRegexFilter + } + return "" +} + +func (m *RowFilter) GetColumnQualifierRegexFilter() []byte { + if x, ok := m.GetFilter().(*RowFilter_ColumnQualifierRegexFilter); ok { + return x.ColumnQualifierRegexFilter + } + return nil +} + +func (m *RowFilter) GetColumnRangeFilter() *ColumnRange { + if x, ok := m.GetFilter().(*RowFilter_ColumnRangeFilter); ok { + return x.ColumnRangeFilter + } + return nil +} + +func (m *RowFilter) GetTimestampRangeFilter() *TimestampRange { + if x, ok := m.GetFilter().(*RowFilter_TimestampRangeFilter); ok { + return x.TimestampRangeFilter + } + return nil +} + +func (m *RowFilter) GetValueRegexFilter() []byte { + if x, ok := m.GetFilter().(*RowFilter_ValueRegexFilter); ok { + return x.ValueRegexFilter + } + return nil +} + +func (m *RowFilter) GetValueRangeFilter() *ValueRange { + if x, ok := m.GetFilter().(*RowFilter_ValueRangeFilter); ok { + return x.ValueRangeFilter + } + return nil +} + +func (m *RowFilter) GetCellsPerRowOffsetFilter() int32 { + if x, ok := m.GetFilter().(*RowFilter_CellsPerRowOffsetFilter); ok { + return x.CellsPerRowOffsetFilter + } + return 0 +} + +func (m *RowFilter) GetCellsPerRowLimitFilter() int32 { + if x, ok := m.GetFilter().(*RowFilter_CellsPerRowLimitFilter); ok { + return x.CellsPerRowLimitFilter + } + return 0 +} + +func (m *RowFilter) GetCellsPerColumnLimitFilter() int32 { + if x, ok := m.GetFilter().(*RowFilter_CellsPerColumnLimitFilter); ok { + return x.CellsPerColumnLimitFilter + } + return 0 +} + +func (m *RowFilter) GetStripValueTransformer() bool { + if x, ok := m.GetFilter().(*RowFilter_StripValueTransformer); ok { + return x.StripValueTransformer + } + return false +} + +func (m *RowFilter) GetApplyLabelTransformer() string { + if x, ok := m.GetFilter().(*RowFilter_ApplyLabelTransformer); ok { + return x.ApplyLabelTransformer + } + return "" +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*RowFilter) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _RowFilter_OneofMarshaler, _RowFilter_OneofUnmarshaler, []interface{}{ + (*RowFilter_Chain_)(nil), + (*RowFilter_Interleave_)(nil), + (*RowFilter_Condition_)(nil), + (*RowFilter_Sink)(nil), + (*RowFilter_PassAllFilter)(nil), + (*RowFilter_BlockAllFilter)(nil), + (*RowFilter_RowKeyRegexFilter)(nil), + (*RowFilter_RowSampleFilter)(nil), + (*RowFilter_FamilyNameRegexFilter)(nil), + (*RowFilter_ColumnQualifierRegexFilter)(nil), + (*RowFilter_ColumnRangeFilter)(nil), + (*RowFilter_TimestampRangeFilter)(nil), + (*RowFilter_ValueRegexFilter)(nil), + (*RowFilter_ValueRangeFilter)(nil), + (*RowFilter_CellsPerRowOffsetFilter)(nil), + (*RowFilter_CellsPerRowLimitFilter)(nil), + (*RowFilter_CellsPerColumnLimitFilter)(nil), + (*RowFilter_StripValueTransformer)(nil), + (*RowFilter_ApplyLabelTransformer)(nil), + } +} + +func _RowFilter_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*RowFilter) + // filter + switch x := m.Filter.(type) { + case *RowFilter_Chain_: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Chain); err != nil { + return err + } + case *RowFilter_Interleave_: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Interleave); err != nil { + return err + } + case *RowFilter_Condition_: + b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Condition); err != nil { + return err + } + case *RowFilter_Sink: + t := uint64(0) + if x.Sink { + t = 1 + } + b.EncodeVarint(16<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *RowFilter_PassAllFilter: + t := uint64(0) + if x.PassAllFilter { + t = 1 + } + b.EncodeVarint(17<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *RowFilter_BlockAllFilter: + t := uint64(0) + if x.BlockAllFilter { + t = 1 + } + b.EncodeVarint(18<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *RowFilter_RowKeyRegexFilter: + b.EncodeVarint(4<<3 | proto.WireBytes) + b.EncodeRawBytes(x.RowKeyRegexFilter) + case *RowFilter_RowSampleFilter: + b.EncodeVarint(14<<3 | proto.WireFixed64) + b.EncodeFixed64(math.Float64bits(x.RowSampleFilter)) + case *RowFilter_FamilyNameRegexFilter: + b.EncodeVarint(5<<3 | proto.WireBytes) + b.EncodeStringBytes(x.FamilyNameRegexFilter) + case *RowFilter_ColumnQualifierRegexFilter: + b.EncodeVarint(6<<3 | proto.WireBytes) + b.EncodeRawBytes(x.ColumnQualifierRegexFilter) + case *RowFilter_ColumnRangeFilter: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ColumnRangeFilter); err != nil { + return err + } + case *RowFilter_TimestampRangeFilter: + b.EncodeVarint(8<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.TimestampRangeFilter); err != nil { + return err + } + case *RowFilter_ValueRegexFilter: + b.EncodeVarint(9<<3 | proto.WireBytes) + b.EncodeRawBytes(x.ValueRegexFilter) + case *RowFilter_ValueRangeFilter: + b.EncodeVarint(15<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ValueRangeFilter); err != nil { + return err + } + case *RowFilter_CellsPerRowOffsetFilter: + b.EncodeVarint(10<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.CellsPerRowOffsetFilter)) + case *RowFilter_CellsPerRowLimitFilter: + b.EncodeVarint(11<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.CellsPerRowLimitFilter)) + case *RowFilter_CellsPerColumnLimitFilter: + b.EncodeVarint(12<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.CellsPerColumnLimitFilter)) + case *RowFilter_StripValueTransformer: + t := uint64(0) + if x.StripValueTransformer { + t = 1 + } + b.EncodeVarint(13<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *RowFilter_ApplyLabelTransformer: + b.EncodeVarint(19<<3 | proto.WireBytes) + b.EncodeStringBytes(x.ApplyLabelTransformer) + case nil: + default: + return fmt.Errorf("RowFilter.Filter has unexpected type %T", x) + } + return nil +} + +func _RowFilter_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*RowFilter) + switch tag { + case 1: // filter.chain + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RowFilter_Chain) + err := b.DecodeMessage(msg) + m.Filter = &RowFilter_Chain_{msg} + return true, err + case 2: // filter.interleave + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RowFilter_Interleave) + err := b.DecodeMessage(msg) + m.Filter = &RowFilter_Interleave_{msg} + return true, err + case 3: // filter.condition + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RowFilter_Condition) + err := b.DecodeMessage(msg) + m.Filter = &RowFilter_Condition_{msg} + return true, err + case 16: // filter.sink + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Filter = &RowFilter_Sink{x != 0} + return true, err + case 17: // filter.pass_all_filter + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Filter = &RowFilter_PassAllFilter{x != 0} + return true, err + case 18: // filter.block_all_filter + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Filter = &RowFilter_BlockAllFilter{x != 0} + return true, err + case 4: // filter.row_key_regex_filter + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.Filter = &RowFilter_RowKeyRegexFilter{x} + return true, err + case 14: // filter.row_sample_filter + if wire != proto.WireFixed64 { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeFixed64() + m.Filter = &RowFilter_RowSampleFilter{math.Float64frombits(x)} + return true, err + case 5: // filter.family_name_regex_filter + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.Filter = &RowFilter_FamilyNameRegexFilter{x} + return true, err + case 6: // filter.column_qualifier_regex_filter + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.Filter = &RowFilter_ColumnQualifierRegexFilter{x} + return true, err + case 7: // filter.column_range_filter + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ColumnRange) + err := b.DecodeMessage(msg) + m.Filter = &RowFilter_ColumnRangeFilter{msg} + return true, err + case 8: // filter.timestamp_range_filter + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(TimestampRange) + err := b.DecodeMessage(msg) + m.Filter = &RowFilter_TimestampRangeFilter{msg} + return true, err + case 9: // filter.value_regex_filter + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.Filter = &RowFilter_ValueRegexFilter{x} + return true, err + case 15: // filter.value_range_filter + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ValueRange) + err := b.DecodeMessage(msg) + m.Filter = &RowFilter_ValueRangeFilter{msg} + return true, err + case 10: // filter.cells_per_row_offset_filter + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Filter = &RowFilter_CellsPerRowOffsetFilter{int32(x)} + return true, err + case 11: // filter.cells_per_row_limit_filter + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Filter = &RowFilter_CellsPerRowLimitFilter{int32(x)} + return true, err + case 12: // filter.cells_per_column_limit_filter + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Filter = &RowFilter_CellsPerColumnLimitFilter{int32(x)} + return true, err + case 13: // filter.strip_value_transformer + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Filter = &RowFilter_StripValueTransformer{x != 0} + return true, err + case 19: // filter.apply_label_transformer + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.Filter = &RowFilter_ApplyLabelTransformer{x} + return true, err + default: + return false, nil + } +} + +// A RowFilter which sends rows through several RowFilters in sequence. +type RowFilter_Chain struct { + // The elements of "filters" are chained together to process the input row: + // in row -> f(0) -> intermediate row -> f(1) -> ... -> f(N) -> out row + // The full chain is executed atomically. + Filters []*RowFilter `protobuf:"bytes,1,rep,name=filters" json:"filters,omitempty"` +} + +func (m *RowFilter_Chain) Reset() { *m = RowFilter_Chain{} } +func (m *RowFilter_Chain) String() string { return proto.CompactTextString(m) } +func (*RowFilter_Chain) ProtoMessage() {} + +func (m *RowFilter_Chain) GetFilters() []*RowFilter { + if m != nil { + return m.Filters + } + return nil +} + +// A RowFilter which sends each row to each of several component +// RowFilters and interleaves the results. +type RowFilter_Interleave struct { + // The elements of "filters" all process a copy of the input row, and the + // results are pooled, sorted, and combined into a single output row. + // If multiple cells are produced with the same column and timestamp, + // they will all appear in the output row in an unspecified mutual order. + // Consider the following example, with three filters: + // + // input row + // | + // ----------------------------------------------------- + // | | | + // f(0) f(1) f(2) + // | | | + // 1: foo,bar,10,x foo,bar,10,z far,bar,7,a + // 2: foo,blah,11,z far,blah,5,x far,blah,5,x + // | | | + // ----------------------------------------------------- + // | + // 1: foo,bar,10,z // could have switched with #2 + // 2: foo,bar,10,x // could have switched with #1 + // 3: foo,blah,11,z + // 4: far,bar,7,a + // 5: far,blah,5,x // identical to #6 + // 6: far,blah,5,x // identical to #5 + // All interleaved filters are executed atomically. + Filters []*RowFilter `protobuf:"bytes,1,rep,name=filters" json:"filters,omitempty"` +} + +func (m *RowFilter_Interleave) Reset() { *m = RowFilter_Interleave{} } +func (m *RowFilter_Interleave) String() string { return proto.CompactTextString(m) } +func (*RowFilter_Interleave) ProtoMessage() {} + +func (m *RowFilter_Interleave) GetFilters() []*RowFilter { + if m != nil { + return m.Filters + } + return nil +} + +// A RowFilter which evaluates one of two possible RowFilters, depending on +// whether or not a predicate RowFilter outputs any cells from the input row. +// +// IMPORTANT NOTE: The predicate filter does not execute atomically with the +// true and false filters, which may lead to inconsistent or unexpected +// results. Additionally, Condition filters have poor performance, especially +// when filters are set for the false condition. +type RowFilter_Condition struct { + // If "predicate_filter" outputs any cells, then "true_filter" will be + // evaluated on the input row. Otherwise, "false_filter" will be evaluated. + PredicateFilter *RowFilter `protobuf:"bytes,1,opt,name=predicate_filter" json:"predicate_filter,omitempty"` + // The filter to apply to the input row if "predicate_filter" returns any + // results. If not provided, no results will be returned in the true case. + TrueFilter *RowFilter `protobuf:"bytes,2,opt,name=true_filter" json:"true_filter,omitempty"` + // The filter to apply to the input row if "predicate_filter" does not + // return any results. If not provided, no results will be returned in the + // false case. + FalseFilter *RowFilter `protobuf:"bytes,3,opt,name=false_filter" json:"false_filter,omitempty"` +} + +func (m *RowFilter_Condition) Reset() { *m = RowFilter_Condition{} } +func (m *RowFilter_Condition) String() string { return proto.CompactTextString(m) } +func (*RowFilter_Condition) ProtoMessage() {} + +func (m *RowFilter_Condition) GetPredicateFilter() *RowFilter { + if m != nil { + return m.PredicateFilter + } + return nil +} + +func (m *RowFilter_Condition) GetTrueFilter() *RowFilter { + if m != nil { + return m.TrueFilter + } + return nil +} + +func (m *RowFilter_Condition) GetFalseFilter() *RowFilter { + if m != nil { + return m.FalseFilter + } + return nil +} + +// Specifies a particular change to be made to the contents of a row. +type Mutation struct { + // Which of the possible Mutation types to apply. + // + // Types that are valid to be assigned to Mutation: + // *Mutation_SetCell_ + // *Mutation_DeleteFromColumn_ + // *Mutation_DeleteFromFamily_ + // *Mutation_DeleteFromRow_ + Mutation isMutation_Mutation `protobuf_oneof:"mutation"` +} + +func (m *Mutation) Reset() { *m = Mutation{} } +func (m *Mutation) String() string { return proto.CompactTextString(m) } +func (*Mutation) ProtoMessage() {} + +type isMutation_Mutation interface { + isMutation_Mutation() +} + +type Mutation_SetCell_ struct { + SetCell *Mutation_SetCell `protobuf:"bytes,1,opt,name=set_cell,oneof"` +} +type Mutation_DeleteFromColumn_ struct { + DeleteFromColumn *Mutation_DeleteFromColumn `protobuf:"bytes,2,opt,name=delete_from_column,oneof"` +} +type Mutation_DeleteFromFamily_ struct { + DeleteFromFamily *Mutation_DeleteFromFamily `protobuf:"bytes,3,opt,name=delete_from_family,oneof"` +} +type Mutation_DeleteFromRow_ struct { + DeleteFromRow *Mutation_DeleteFromRow `protobuf:"bytes,4,opt,name=delete_from_row,oneof"` +} + +func (*Mutation_SetCell_) isMutation_Mutation() {} +func (*Mutation_DeleteFromColumn_) isMutation_Mutation() {} +func (*Mutation_DeleteFromFamily_) isMutation_Mutation() {} +func (*Mutation_DeleteFromRow_) isMutation_Mutation() {} + +func (m *Mutation) GetMutation() isMutation_Mutation { + if m != nil { + return m.Mutation + } + return nil +} + +func (m *Mutation) GetSetCell() *Mutation_SetCell { + if x, ok := m.GetMutation().(*Mutation_SetCell_); ok { + return x.SetCell + } + return nil +} + +func (m *Mutation) GetDeleteFromColumn() *Mutation_DeleteFromColumn { + if x, ok := m.GetMutation().(*Mutation_DeleteFromColumn_); ok { + return x.DeleteFromColumn + } + return nil +} + +func (m *Mutation) GetDeleteFromFamily() *Mutation_DeleteFromFamily { + if x, ok := m.GetMutation().(*Mutation_DeleteFromFamily_); ok { + return x.DeleteFromFamily + } + return nil +} + +func (m *Mutation) GetDeleteFromRow() *Mutation_DeleteFromRow { + if x, ok := m.GetMutation().(*Mutation_DeleteFromRow_); ok { + return x.DeleteFromRow + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Mutation) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _Mutation_OneofMarshaler, _Mutation_OneofUnmarshaler, []interface{}{ + (*Mutation_SetCell_)(nil), + (*Mutation_DeleteFromColumn_)(nil), + (*Mutation_DeleteFromFamily_)(nil), + (*Mutation_DeleteFromRow_)(nil), + } +} + +func _Mutation_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Mutation) + // mutation + switch x := m.Mutation.(type) { + case *Mutation_SetCell_: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.SetCell); err != nil { + return err + } + case *Mutation_DeleteFromColumn_: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.DeleteFromColumn); err != nil { + return err + } + case *Mutation_DeleteFromFamily_: + b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.DeleteFromFamily); err != nil { + return err + } + case *Mutation_DeleteFromRow_: + b.EncodeVarint(4<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.DeleteFromRow); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("Mutation.Mutation has unexpected type %T", x) + } + return nil +} + +func _Mutation_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Mutation) + switch tag { + case 1: // mutation.set_cell + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Mutation_SetCell) + err := b.DecodeMessage(msg) + m.Mutation = &Mutation_SetCell_{msg} + return true, err + case 2: // mutation.delete_from_column + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Mutation_DeleteFromColumn) + err := b.DecodeMessage(msg) + m.Mutation = &Mutation_DeleteFromColumn_{msg} + return true, err + case 3: // mutation.delete_from_family + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Mutation_DeleteFromFamily) + err := b.DecodeMessage(msg) + m.Mutation = &Mutation_DeleteFromFamily_{msg} + return true, err + case 4: // mutation.delete_from_row + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Mutation_DeleteFromRow) + err := b.DecodeMessage(msg) + m.Mutation = &Mutation_DeleteFromRow_{msg} + return true, err + default: + return false, nil + } +} + +// A Mutation which sets the value of the specified cell. +type Mutation_SetCell struct { + // The name of the family into which new data should be written. + // Must match [-_.a-zA-Z0-9]+ + FamilyName string `protobuf:"bytes,1,opt,name=family_name" json:"family_name,omitempty"` + // The qualifier of the column into which new data should be written. + // Can be any byte string, including the empty string. + ColumnQualifier []byte `protobuf:"bytes,2,opt,name=column_qualifier,proto3" json:"column_qualifier,omitempty"` + // The timestamp of the cell into which new data should be written. + // Use -1 for current Bigtable server time. + // Otherwise, the client should set this value itself, noting that the + // default value is a timestamp of zero if the field is left unspecified. + // Values must match the "granularity" of the table (e.g. micros, millis). + TimestampMicros int64 `protobuf:"varint,3,opt,name=timestamp_micros" json:"timestamp_micros,omitempty"` + // The value to be written into the specified cell. + Value []byte `protobuf:"bytes,4,opt,name=value,proto3" json:"value,omitempty"` +} + +func (m *Mutation_SetCell) Reset() { *m = Mutation_SetCell{} } +func (m *Mutation_SetCell) String() string { return proto.CompactTextString(m) } +func (*Mutation_SetCell) ProtoMessage() {} + +// A Mutation which deletes cells from the specified column, optionally +// restricting the deletions to a given timestamp range. +type Mutation_DeleteFromColumn struct { + // The name of the family from which cells should be deleted. + // Must match [-_.a-zA-Z0-9]+ + FamilyName string `protobuf:"bytes,1,opt,name=family_name" json:"family_name,omitempty"` + // The qualifier of the column from which cells should be deleted. + // Can be any byte string, including the empty string. + ColumnQualifier []byte `protobuf:"bytes,2,opt,name=column_qualifier,proto3" json:"column_qualifier,omitempty"` + // The range of timestamps within which cells should be deleted. + TimeRange *TimestampRange `protobuf:"bytes,3,opt,name=time_range" json:"time_range,omitempty"` +} + +func (m *Mutation_DeleteFromColumn) Reset() { *m = Mutation_DeleteFromColumn{} } +func (m *Mutation_DeleteFromColumn) String() string { return proto.CompactTextString(m) } +func (*Mutation_DeleteFromColumn) ProtoMessage() {} + +func (m *Mutation_DeleteFromColumn) GetTimeRange() *TimestampRange { + if m != nil { + return m.TimeRange + } + return nil +} + +// A Mutation which deletes all cells from the specified column family. +type Mutation_DeleteFromFamily struct { + // The name of the family from which cells should be deleted. + // Must match [-_.a-zA-Z0-9]+ + FamilyName string `protobuf:"bytes,1,opt,name=family_name" json:"family_name,omitempty"` +} + +func (m *Mutation_DeleteFromFamily) Reset() { *m = Mutation_DeleteFromFamily{} } +func (m *Mutation_DeleteFromFamily) String() string { return proto.CompactTextString(m) } +func (*Mutation_DeleteFromFamily) ProtoMessage() {} + +// A Mutation which deletes all cells from the containing row. +type Mutation_DeleteFromRow struct { +} + +func (m *Mutation_DeleteFromRow) Reset() { *m = Mutation_DeleteFromRow{} } +func (m *Mutation_DeleteFromRow) String() string { return proto.CompactTextString(m) } +func (*Mutation_DeleteFromRow) ProtoMessage() {} + +// Specifies an atomic read/modify/write operation on the latest value of the +// specified column. +type ReadModifyWriteRule struct { + // The name of the family to which the read/modify/write should be applied. + // Must match [-_.a-zA-Z0-9]+ + FamilyName string `protobuf:"bytes,1,opt,name=family_name" json:"family_name,omitempty"` + // The qualifier of the column to which the read/modify/write should be + // applied. + // Can be any byte string, including the empty string. + ColumnQualifier []byte `protobuf:"bytes,2,opt,name=column_qualifier,proto3" json:"column_qualifier,omitempty"` + // The rule used to determine the column's new latest value from its current + // latest value. + // + // Types that are valid to be assigned to Rule: + // *ReadModifyWriteRule_AppendValue + // *ReadModifyWriteRule_IncrementAmount + Rule isReadModifyWriteRule_Rule `protobuf_oneof:"rule"` +} + +func (m *ReadModifyWriteRule) Reset() { *m = ReadModifyWriteRule{} } +func (m *ReadModifyWriteRule) String() string { return proto.CompactTextString(m) } +func (*ReadModifyWriteRule) ProtoMessage() {} + +type isReadModifyWriteRule_Rule interface { + isReadModifyWriteRule_Rule() +} + +type ReadModifyWriteRule_AppendValue struct { + AppendValue []byte `protobuf:"bytes,3,opt,name=append_value,proto3,oneof"` +} +type ReadModifyWriteRule_IncrementAmount struct { + IncrementAmount int64 `protobuf:"varint,4,opt,name=increment_amount,oneof"` +} + +func (*ReadModifyWriteRule_AppendValue) isReadModifyWriteRule_Rule() {} +func (*ReadModifyWriteRule_IncrementAmount) isReadModifyWriteRule_Rule() {} + +func (m *ReadModifyWriteRule) GetRule() isReadModifyWriteRule_Rule { + if m != nil { + return m.Rule + } + return nil +} + +func (m *ReadModifyWriteRule) GetAppendValue() []byte { + if x, ok := m.GetRule().(*ReadModifyWriteRule_AppendValue); ok { + return x.AppendValue + } + return nil +} + +func (m *ReadModifyWriteRule) GetIncrementAmount() int64 { + if x, ok := m.GetRule().(*ReadModifyWriteRule_IncrementAmount); ok { + return x.IncrementAmount + } + return 0 +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ReadModifyWriteRule) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _ReadModifyWriteRule_OneofMarshaler, _ReadModifyWriteRule_OneofUnmarshaler, []interface{}{ + (*ReadModifyWriteRule_AppendValue)(nil), + (*ReadModifyWriteRule_IncrementAmount)(nil), + } +} + +func _ReadModifyWriteRule_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ReadModifyWriteRule) + // rule + switch x := m.Rule.(type) { + case *ReadModifyWriteRule_AppendValue: + b.EncodeVarint(3<<3 | proto.WireBytes) + b.EncodeRawBytes(x.AppendValue) + case *ReadModifyWriteRule_IncrementAmount: + b.EncodeVarint(4<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.IncrementAmount)) + case nil: + default: + return fmt.Errorf("ReadModifyWriteRule.Rule has unexpected type %T", x) + } + return nil +} + +func _ReadModifyWriteRule_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ReadModifyWriteRule) + switch tag { + case 3: // rule.append_value + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.Rule = &ReadModifyWriteRule_AppendValue{x} + return true, err + case 4: // rule.increment_amount + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Rule = &ReadModifyWriteRule_IncrementAmount{int64(x)} + return true, err + default: + return false, nil + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto new file mode 100644 index 0000000000..86234d22d5 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto @@ -0,0 +1,500 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.v1; + +option java_multiple_files = true; +option java_outer_classname = "BigtableDataProto"; +option java_package = "com.google.bigtable.v1"; + + +// Specifies the complete (requested) contents of a single row of a table. +// Rows which exceed 256MiB in size cannot be read in full. +message Row { + // The unique key which identifies this row within its table. This is the same + // key that's used to identify the row in, for example, a MutateRowRequest. + // May contain any non-empty byte string up to 4KiB in length. + bytes key = 1; + + // May be empty, but only if the entire row is empty. + // The mutual ordering of column families is not specified. + repeated Family families = 2; +} + +// Specifies (some of) the contents of a single row/column family of a table. +message Family { + // The unique key which identifies this family within its row. This is the + // same key that's used to identify the family in, for example, a RowFilter + // which sets its "family_name_regex_filter" field. + // Must match [-_.a-zA-Z0-9]+, except that AggregatingRowProcessors may + // produce cells in a sentinel family with an empty name. + // Must be no greater than 64 characters in length. + string name = 1; + + // Must not be empty. Sorted in order of increasing "qualifier". + repeated Column columns = 2; +} + +// Specifies (some of) the contents of a single row/column of a table. +message Column { + // The unique key which identifies this column within its family. This is the + // same key that's used to identify the column in, for example, a RowFilter + // which sets its "column_qualifier_regex_filter" field. + // May contain any byte string, including the empty string, up to 16kiB in + // length. + bytes qualifier = 1; + + // Must not be empty. Sorted in order of decreasing "timestamp_micros". + repeated Cell cells = 2; +} + +// Specifies (some of) the contents of a single row/column/timestamp of a table. +message Cell { + // The cell's stored timestamp, which also uniquely identifies it within + // its column. + // Values are always expressed in microseconds, but individual tables may set + // a coarser "granularity" to further restrict the allowed values. For + // example, a table which specifies millisecond granularity will only allow + // values of "timestamp_micros" which are multiples of 1000. + int64 timestamp_micros = 1; + + // The value stored in the cell. + // May contain any byte string, including the empty string, up to 100MiB in + // length. + bytes value = 2; + + // Labels applied to the cell by a [RowFilter][google.bigtable.v1.RowFilter]. + repeated string labels = 3; +} + +// Specifies a contiguous range of rows. +message RowRange { + // Inclusive lower bound. If left empty, interpreted as the empty string. + bytes start_key = 2; + + // Exclusive upper bound. If left empty, interpreted as infinity. + bytes end_key = 3; +} + +// Specifies a contiguous range of columns within a single column family. +// The range spans from : to +// :, where both bounds can be either inclusive or +// exclusive. +message ColumnRange { + // The name of the column family within which this range falls. + string family_name = 1; + + // The column qualifier at which to start the range (within 'column_family'). + // If neither field is set, interpreted as the empty string, inclusive. + oneof start_qualifier { + // Used when giving an inclusive lower bound for the range. + bytes start_qualifier_inclusive = 2; + + // Used when giving an exclusive lower bound for the range. + bytes start_qualifier_exclusive = 3; + } + + // The column qualifier at which to end the range (within 'column_family'). + // If neither field is set, interpreted as the infinite string, exclusive. + oneof end_qualifier { + // Used when giving an inclusive upper bound for the range. + bytes end_qualifier_inclusive = 4; + + // Used when giving an exclusive upper bound for the range. + bytes end_qualifier_exclusive = 5; + } +} + +// Specified a contiguous range of microsecond timestamps. +message TimestampRange { + // Inclusive lower bound. If left empty, interpreted as 0. + int64 start_timestamp_micros = 1; + + // Exclusive upper bound. If left empty, interpreted as infinity. + int64 end_timestamp_micros = 2; +} + +// Specifies a contiguous range of raw byte values. +message ValueRange { + // The value at which to start the range. + // If neither field is set, interpreted as the empty string, inclusive. + oneof start_value { + // Used when giving an inclusive lower bound for the range. + bytes start_value_inclusive = 1; + + // Used when giving an exclusive lower bound for the range. + bytes start_value_exclusive = 2; + } + + // The value at which to end the range. + // If neither field is set, interpreted as the infinite string, exclusive. + oneof end_value { + // Used when giving an inclusive upper bound for the range. + bytes end_value_inclusive = 3; + + // Used when giving an exclusive upper bound for the range. + bytes end_value_exclusive = 4; + } +} + +// Takes a row as input and produces an alternate view of the row based on +// specified rules. For example, a RowFilter might trim down a row to include +// just the cells from columns matching a given regular expression, or might +// return all the cells of a row but not their values. More complicated filters +// can be composed out of these components to express requests such as, "within +// every column of a particular family, give just the two most recent cells +// which are older than timestamp X." +// +// There are two broad categories of RowFilters (true filters and transformers), +// as well as two ways to compose simple filters into more complex ones +// (chains and interleaves). They work as follows: +// +// * True filters alter the input row by excluding some of its cells wholesale +// from the output row. An example of a true filter is the "value_regex_filter", +// which excludes cells whose values don't match the specified pattern. All +// regex true filters use RE2 syntax (https://github.com/google/re2/wiki/Syntax) +// in raw byte mode (RE2::Latin1), and are evaluated as full matches. An +// important point to keep in mind is that RE2(.) is equivalent by default to +// RE2([^\n]), meaning that it does not match newlines. When attempting to match +// an arbitrary byte, you should therefore use the escape sequence '\C', which +// may need to be further escaped as '\\C' in your client language. +// +// * Transformers alter the input row by changing the values of some of its +// cells in the output, without excluding them completely. Currently, the only +// supported transformer is the "strip_value_transformer", which replaces every +// cell's value with the empty string. +// +// * Chains and interleaves are described in more detail in the +// RowFilter.Chain and RowFilter.Interleave documentation. +// +// The total serialized size of a RowFilter message must not +// exceed 4096 bytes, and RowFilters may not be nested within each other +// (in Chains or Interleaves) to a depth of more than 20. +message RowFilter { + // A RowFilter which sends rows through several RowFilters in sequence. + message Chain { + // The elements of "filters" are chained together to process the input row: + // in row -> f(0) -> intermediate row -> f(1) -> ... -> f(N) -> out row + // The full chain is executed atomically. + repeated RowFilter filters = 1; + } + + // A RowFilter which sends each row to each of several component + // RowFilters and interleaves the results. + message Interleave { + // The elements of "filters" all process a copy of the input row, and the + // results are pooled, sorted, and combined into a single output row. + // If multiple cells are produced with the same column and timestamp, + // they will all appear in the output row in an unspecified mutual order. + // Consider the following example, with three filters: + // + // input row + // | + // ----------------------------------------------------- + // | | | + // f(0) f(1) f(2) + // | | | + // 1: foo,bar,10,x foo,bar,10,z far,bar,7,a + // 2: foo,blah,11,z far,blah,5,x far,blah,5,x + // | | | + // ----------------------------------------------------- + // | + // 1: foo,bar,10,z // could have switched with #2 + // 2: foo,bar,10,x // could have switched with #1 + // 3: foo,blah,11,z + // 4: far,bar,7,a + // 5: far,blah,5,x // identical to #6 + // 6: far,blah,5,x // identical to #5 + // All interleaved filters are executed atomically. + repeated RowFilter filters = 1; + } + + // A RowFilter which evaluates one of two possible RowFilters, depending on + // whether or not a predicate RowFilter outputs any cells from the input row. + // + // IMPORTANT NOTE: The predicate filter does not execute atomically with the + // true and false filters, which may lead to inconsistent or unexpected + // results. Additionally, Condition filters have poor performance, especially + // when filters are set for the false condition. + message Condition { + // If "predicate_filter" outputs any cells, then "true_filter" will be + // evaluated on the input row. Otherwise, "false_filter" will be evaluated. + RowFilter predicate_filter = 1; + + // The filter to apply to the input row if "predicate_filter" returns any + // results. If not provided, no results will be returned in the true case. + RowFilter true_filter = 2; + + // The filter to apply to the input row if "predicate_filter" does not + // return any results. If not provided, no results will be returned in the + // false case. + RowFilter false_filter = 3; + } + + // Which of the possible RowFilter types to apply. If none are set, this + // RowFilter returns all cells in the input row. + oneof filter { + // Applies several RowFilters to the data in sequence, progressively + // narrowing the results. + Chain chain = 1; + + // Applies several RowFilters to the data in parallel and combines the + // results. + Interleave interleave = 2; + + // Applies one of two possible RowFilters to the data based on the output of + // a predicate RowFilter. + Condition condition = 3; + + // ADVANCED USE ONLY. + // Hook for introspection into the RowFilter. Outputs all cells directly to + // the output of the read rather than to any parent filter. Consider the + // following example: + // + // Chain( + // FamilyRegex("A"), + // Interleave( + // All(), + // Chain(Label("foo"), Sink()) + // ), + // QualifierRegex("B") + // ) + // + // A,A,1,w + // A,B,2,x + // B,B,4,z + // | + // FamilyRegex("A") + // | + // A,A,1,w + // A,B,2,x + // | + // +------------+-------------+ + // | | + // All() Label(foo) + // | | + // A,A,1,w A,A,1,w,labels:[foo] + // A,B,2,x A,B,2,x,labels:[foo] + // | | + // | Sink() --------------+ + // | | | + // +------------+ x------+ A,A,1,w,labels:[foo] + // | A,B,2,x,labels:[foo] + // A,A,1,w | + // A,B,2,x | + // | | + // QualifierRegex("B") | + // | | + // A,B,2,x | + // | | + // +--------------------------------+ + // | + // A,A,1,w,labels:[foo] + // A,B,2,x,labels:[foo] // could be switched + // A,B,2,x // could be switched + // + // Despite being excluded by the qualifier filter, a copy of every cell + // that reaches the sink is present in the final result. + // + // As with an [Interleave][google.bigtable.v1.RowFilter.Interleave], + // duplicate cells are possible, and appear in an unspecified mutual order. + // In this case we have a duplicate with column "A:B" and timestamp 2, + // because one copy passed through the all filter while the other was + // passed through the label and sink. Note that one copy has label "foo", + // while the other does not. + // + // Cannot be used within the `predicate_filter`, `true_filter`, or + // `false_filter` of a [Condition][google.bigtable.v1.RowFilter.Condition]. + bool sink = 16; + + // Matches all cells, regardless of input. Functionally equivalent to + // leaving `filter` unset, but included for completeness. + bool pass_all_filter = 17; + + // Does not match any cells, regardless of input. Useful for temporarily + // disabling just part of a filter. + bool block_all_filter = 18; + + // Matches only cells from rows whose keys satisfy the given RE2 regex. In + // other words, passes through the entire row when the key matches, and + // otherwise produces an empty row. + // Note that, since row keys can contain arbitrary bytes, the '\C' escape + // sequence must be used if a true wildcard is desired. The '.' character + // will not match the new line character '\n', which may be present in a + // binary key. + bytes row_key_regex_filter = 4; + + // Matches all cells from a row with probability p, and matches no cells + // from the row with probability 1-p. + double row_sample_filter = 14; + + // Matches only cells from columns whose families satisfy the given RE2 + // regex. For technical reasons, the regex must not contain the ':' + // character, even if it is not being used as a literal. + // Note that, since column families cannot contain the new line character + // '\n', it is sufficient to use '.' as a full wildcard when matching + // column family names. + string family_name_regex_filter = 5; + + // Matches only cells from columns whose qualifiers satisfy the given RE2 + // regex. + // Note that, since column qualifiers can contain arbitrary bytes, the '\C' + // escape sequence must be used if a true wildcard is desired. The '.' + // character will not match the new line character '\n', which may be + // present in a binary qualifier. + bytes column_qualifier_regex_filter = 6; + + // Matches only cells from columns within the given range. + ColumnRange column_range_filter = 7; + + // Matches only cells with timestamps within the given range. + TimestampRange timestamp_range_filter = 8; + + // Matches only cells with values that satisfy the given regular expression. + // Note that, since cell values can contain arbitrary bytes, the '\C' escape + // sequence must be used if a true wildcard is desired. The '.' character + // will not match the new line character '\n', which may be present in a + // binary value. + bytes value_regex_filter = 9; + + // Matches only cells with values that fall within the given range. + ValueRange value_range_filter = 15; + + // Skips the first N cells of each row, matching all subsequent cells. + int32 cells_per_row_offset_filter = 10; + + // Matches only the first N cells of each row. + int32 cells_per_row_limit_filter = 11; + + // Matches only the most recent N cells within each column. For example, + // if N=2, this filter would match column "foo:bar" at timestamps 10 and 9, + // skip all earlier cells in "foo:bar", and then begin matching again in + // column "foo:bar2". + int32 cells_per_column_limit_filter = 12; + + // Replaces each cell's value with the empty string. + bool strip_value_transformer = 13; + + // Applies the given label to all cells in the output row. This allows + // the client to determine which results were produced from which part of + // the filter. + // + // Values must be at most 15 characters in length, and match the RE2 + // pattern [a-z0-9\\-]+ + // + // Due to a technical limitation, it is not currently possible to apply + // multiple labels to a cell. As a result, a Chain may have no more than + // one sub-filter which contains a apply_label_transformer. It is okay for + // an Interleave to contain multiple apply_label_transformers, as they will + // be applied to separate copies of the input. This may be relaxed in the + // future. + string apply_label_transformer = 19; + } +} + +// Specifies a particular change to be made to the contents of a row. +message Mutation { + // A Mutation which sets the value of the specified cell. + message SetCell { + // The name of the family into which new data should be written. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + + // The qualifier of the column into which new data should be written. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The timestamp of the cell into which new data should be written. + // Use -1 for current Bigtable server time. + // Otherwise, the client should set this value itself, noting that the + // default value is a timestamp of zero if the field is left unspecified. + // Values must match the "granularity" of the table (e.g. micros, millis). + int64 timestamp_micros = 3; + + // The value to be written into the specified cell. + bytes value = 4; + } + + // A Mutation which deletes cells from the specified column, optionally + // restricting the deletions to a given timestamp range. + message DeleteFromColumn { + // The name of the family from which cells should be deleted. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + + // The qualifier of the column from which cells should be deleted. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The range of timestamps within which cells should be deleted. + TimestampRange time_range = 3; + } + + // A Mutation which deletes all cells from the specified column family. + message DeleteFromFamily { + // The name of the family from which cells should be deleted. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + } + + // A Mutation which deletes all cells from the containing row. + message DeleteFromRow { + + } + + // Which of the possible Mutation types to apply. + oneof mutation { + // Set a cell's value. + SetCell set_cell = 1; + + // Deletes cells from a column. + DeleteFromColumn delete_from_column = 2; + + // Deletes cells from a column family. + DeleteFromFamily delete_from_family = 3; + + // Deletes cells from the entire row. + DeleteFromRow delete_from_row = 4; + } +} + +// Specifies an atomic read/modify/write operation on the latest value of the +// specified column. +message ReadModifyWriteRule { + // The name of the family to which the read/modify/write should be applied. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + + // The qualifier of the column to which the read/modify/write should be + // applied. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The rule used to determine the column's new latest value from its current + // latest value. + oneof rule { + // Rule specifying that "append_value" be appended to the existing value. + // If the targeted cell is unset, it will be treated as containing the + // empty string. + bytes append_value = 3; + + // Rule specifying that "increment_amount" be added to the existing value. + // If the targeted cell is unset, it will be treated as containing a zero. + // Otherwise, the targeted cell must contain an 8-byte value (interpreted + // as a 64-bit big-endian signed integer), or the entire request will fail. + int64 increment_amount = 4; + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/duration_proto/duration.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/duration_proto/duration.pb.go new file mode 100644 index 0000000000..63d6727e36 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/duration_proto/duration.pb.go @@ -0,0 +1,81 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/duration_proto/duration.proto +// DO NOT EDIT! + +/* +Package google_protobuf is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/duration_proto/duration.proto + +It has these top-level messages: + Duration +*/ +package google_protobuf + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (durations.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +type Duration struct { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"` + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"` +} + +func (m *Duration) Reset() { *m = Duration{} } +func (m *Duration) String() string { return proto.CompactTextString(m) } +func (*Duration) ProtoMessage() {} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/duration_proto/duration.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/duration_proto/duration.proto new file mode 100644 index 0000000000..15e9d44d3b --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/duration_proto/duration.proto @@ -0,0 +1,78 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "DurationProto"; +option java_package = "com.google.protobuf"; + + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (durations.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/empty/empty.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/empty/empty.pb.go new file mode 100644 index 0000000000..89bb423481 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/empty/empty.pb.go @@ -0,0 +1,38 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/empty/empty.proto +// DO NOT EDIT! + +/* +Package google_protobuf is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/empty/empty.proto + +It has these top-level messages: + Empty +*/ +package google_protobuf + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +type Empty struct { +} + +func (m *Empty) Reset() { *m = Empty{} } +func (m *Empty) String() string { return proto.CompactTextString(m) } +func (*Empty) ProtoMessage() {} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/empty/empty.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/empty/empty.proto new file mode 100644 index 0000000000..43b06e87bd --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/empty/empty.proto @@ -0,0 +1,34 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.protobuf; + +option java_multiple_files = true; +option java_outer_classname = "EmptyProto"; +option java_package = "com.google.protobuf"; + + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +message Empty { + +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/regen.sh b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/regen.sh new file mode 100755 index 0000000000..6cfa1688b4 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/regen.sh @@ -0,0 +1,76 @@ +#!/bin/bash -e +# +# This script rebuilds the generated code for the protocol buffers. +# To run this you will need protoc and goprotobuf installed; +# see https://github.com/golang/protobuf for instructions. +# You also need Go and Git installed. + +PKG=google.golang.org/cloud/bigtable +UPSTREAM=https://github.com/GoogleCloudPlatform/cloud-bigtable-client +UPSTREAM_SUBDIR=bigtable-protos/src/main/proto + +function die() { + echo 1>&2 $* + exit 1 +} + +# Sanity check that the right tools are accessible. +for tool in go git protoc protoc-gen-go; do + q=$(which $tool) || die "didn't find $tool" + echo 1>&2 "$tool: $q" +done + +tmpdir=$(mktemp -d -t regen-cbt.XXXXXX) +trap 'rm -rf $tmpdir' EXIT + +echo -n 1>&2 "finding package dir... " +pkgdir=$(go list -f '{{.Dir}}' $PKG) +echo 1>&2 $pkgdir +base=$(echo $pkgdir | sed "s,/$PKG\$,,") +echo 1>&2 "base: $base" +cd $base + +echo 1>&2 "fetching latest protos... " +git clone -q $UPSTREAM $tmpdir +# Pass 1: build mapping from upstream filename to our filename. +declare -A filename_map +for f in $(cd $PKG && find internal -name '*.proto'); do + echo -n 1>&2 "looking for latest version of $f... " + up=$(cd $tmpdir/$UPSTREAM_SUBDIR && find * -name $(basename $f)) + echo 1>&2 $up + if [ $(echo $up | wc -w) != "1" ]; then + die "not exactly one match" + fi + filename_map[$up]=$f +done +# Pass 2: build sed script for fixing imports. +import_fixes=$tmpdir/fix_imports.sed +for up in "${!filename_map[@]}"; do + f=${filename_map[$up]} + echo >>$import_fixes "s,\"$up\",\"$PKG/$f\"," +done +cat $import_fixes | sed 's,^,### ,' 1>&2 +# Pass 3: copy files, making necessary adjustments. +for up in "${!filename_map[@]}"; do + f=${filename_map[$up]} + cat $tmpdir/$UPSTREAM_SUBDIR/$up | + # Adjust proto imports. + sed -f $import_fixes | + # Drop the UndeleteCluster RPC method. It returns a google.longrunning.Operation. + sed '/^ rpc UndeleteCluster(/,/^ }$/d' | + # Drop annotations, long-running operations and timestamps. They aren't supported (yet). + sed '/"google\/longrunning\/operations.proto"/d' | + sed '/google.longrunning.Operation/d' | + sed '/"google\/protobuf\/timestamp.proto"/d' | + sed '/google\.protobuf\.Timestamp/d' | + sed '/"google\/api\/annotations.proto"/d' | + sed '/option.*google\.api\.http.*{.*};$/d' | + cat > $PKG/$f +done + +# Run protoc once per package. +for dir in $(find $PKG/internal -name '*.proto' | xargs dirname | sort | uniq); do + echo 1>&2 "* $dir" + protoc --go_out=plugins=grpc:. $dir/*.proto +done +echo 1>&2 "All OK" diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.pb.go new file mode 100644 index 0000000000..74b3837669 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.pb.go @@ -0,0 +1,287 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.proto +// DO NOT EDIT! + +package google_bigtable_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_bigtable_v11 "google.golang.org/cloud/bigtable/internal/data_proto" +import google_protobuf "google.golang.org/cloud/bigtable/internal/empty" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// Client API for BigtableService service + +type BigtableServiceClient interface { + // Streams back the contents of all requested rows, optionally applying + // the same Reader filter to each. Depending on their size, rows may be + // broken up across multiple responses, but atomicity of each row will still + // be preserved. + ReadRows(ctx context.Context, in *ReadRowsRequest, opts ...grpc.CallOption) (BigtableService_ReadRowsClient, error) + // Returns a sample of row keys in the table. The returned row keys will + // delimit contiguous sections of the table of approximately equal size, + // which can be used to break up the data for distributed tasks like + // mapreduces. + SampleRowKeys(ctx context.Context, in *SampleRowKeysRequest, opts ...grpc.CallOption) (BigtableService_SampleRowKeysClient, error) + // Mutates a row atomically. Cells already present in the row are left + // unchanged unless explicitly changed by 'mutation'. + MutateRow(ctx context.Context, in *MutateRowRequest, opts ...grpc.CallOption) (*google_protobuf.Empty, error) + // Mutates a row atomically based on the output of a predicate Reader filter. + CheckAndMutateRow(ctx context.Context, in *CheckAndMutateRowRequest, opts ...grpc.CallOption) (*CheckAndMutateRowResponse, error) + // Modifies a row atomically, reading the latest existing timestamp/value from + // the specified columns and writing a new value at + // max(existing timestamp, current server time) based on pre-defined + // read/modify/write rules. Returns the new contents of all modified cells. + ReadModifyWriteRow(ctx context.Context, in *ReadModifyWriteRowRequest, opts ...grpc.CallOption) (*google_bigtable_v11.Row, error) +} + +type bigtableServiceClient struct { + cc *grpc.ClientConn +} + +func NewBigtableServiceClient(cc *grpc.ClientConn) BigtableServiceClient { + return &bigtableServiceClient{cc} +} + +func (c *bigtableServiceClient) ReadRows(ctx context.Context, in *ReadRowsRequest, opts ...grpc.CallOption) (BigtableService_ReadRowsClient, error) { + stream, err := grpc.NewClientStream(ctx, &_BigtableService_serviceDesc.Streams[0], c.cc, "/google.bigtable.v1.BigtableService/ReadRows", opts...) + if err != nil { + return nil, err + } + x := &bigtableServiceReadRowsClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type BigtableService_ReadRowsClient interface { + Recv() (*ReadRowsResponse, error) + grpc.ClientStream +} + +type bigtableServiceReadRowsClient struct { + grpc.ClientStream +} + +func (x *bigtableServiceReadRowsClient) Recv() (*ReadRowsResponse, error) { + m := new(ReadRowsResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *bigtableServiceClient) SampleRowKeys(ctx context.Context, in *SampleRowKeysRequest, opts ...grpc.CallOption) (BigtableService_SampleRowKeysClient, error) { + stream, err := grpc.NewClientStream(ctx, &_BigtableService_serviceDesc.Streams[1], c.cc, "/google.bigtable.v1.BigtableService/SampleRowKeys", opts...) + if err != nil { + return nil, err + } + x := &bigtableServiceSampleRowKeysClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type BigtableService_SampleRowKeysClient interface { + Recv() (*SampleRowKeysResponse, error) + grpc.ClientStream +} + +type bigtableServiceSampleRowKeysClient struct { + grpc.ClientStream +} + +func (x *bigtableServiceSampleRowKeysClient) Recv() (*SampleRowKeysResponse, error) { + m := new(SampleRowKeysResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *bigtableServiceClient) MutateRow(ctx context.Context, in *MutateRowRequest, opts ...grpc.CallOption) (*google_protobuf.Empty, error) { + out := new(google_protobuf.Empty) + err := grpc.Invoke(ctx, "/google.bigtable.v1.BigtableService/MutateRow", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableServiceClient) CheckAndMutateRow(ctx context.Context, in *CheckAndMutateRowRequest, opts ...grpc.CallOption) (*CheckAndMutateRowResponse, error) { + out := new(CheckAndMutateRowResponse) + err := grpc.Invoke(ctx, "/google.bigtable.v1.BigtableService/CheckAndMutateRow", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableServiceClient) ReadModifyWriteRow(ctx context.Context, in *ReadModifyWriteRowRequest, opts ...grpc.CallOption) (*google_bigtable_v11.Row, error) { + out := new(google_bigtable_v11.Row) + err := grpc.Invoke(ctx, "/google.bigtable.v1.BigtableService/ReadModifyWriteRow", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for BigtableService service + +type BigtableServiceServer interface { + // Streams back the contents of all requested rows, optionally applying + // the same Reader filter to each. Depending on their size, rows may be + // broken up across multiple responses, but atomicity of each row will still + // be preserved. + ReadRows(*ReadRowsRequest, BigtableService_ReadRowsServer) error + // Returns a sample of row keys in the table. The returned row keys will + // delimit contiguous sections of the table of approximately equal size, + // which can be used to break up the data for distributed tasks like + // mapreduces. + SampleRowKeys(*SampleRowKeysRequest, BigtableService_SampleRowKeysServer) error + // Mutates a row atomically. Cells already present in the row are left + // unchanged unless explicitly changed by 'mutation'. + MutateRow(context.Context, *MutateRowRequest) (*google_protobuf.Empty, error) + // Mutates a row atomically based on the output of a predicate Reader filter. + CheckAndMutateRow(context.Context, *CheckAndMutateRowRequest) (*CheckAndMutateRowResponse, error) + // Modifies a row atomically, reading the latest existing timestamp/value from + // the specified columns and writing a new value at + // max(existing timestamp, current server time) based on pre-defined + // read/modify/write rules. Returns the new contents of all modified cells. + ReadModifyWriteRow(context.Context, *ReadModifyWriteRowRequest) (*google_bigtable_v11.Row, error) +} + +func RegisterBigtableServiceServer(s *grpc.Server, srv BigtableServiceServer) { + s.RegisterService(&_BigtableService_serviceDesc, srv) +} + +func _BigtableService_ReadRows_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ReadRowsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BigtableServiceServer).ReadRows(m, &bigtableServiceReadRowsServer{stream}) +} + +type BigtableService_ReadRowsServer interface { + Send(*ReadRowsResponse) error + grpc.ServerStream +} + +type bigtableServiceReadRowsServer struct { + grpc.ServerStream +} + +func (x *bigtableServiceReadRowsServer) Send(m *ReadRowsResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _BigtableService_SampleRowKeys_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SampleRowKeysRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BigtableServiceServer).SampleRowKeys(m, &bigtableServiceSampleRowKeysServer{stream}) +} + +type BigtableService_SampleRowKeysServer interface { + Send(*SampleRowKeysResponse) error + grpc.ServerStream +} + +type bigtableServiceSampleRowKeysServer struct { + grpc.ServerStream +} + +func (x *bigtableServiceSampleRowKeysServer) Send(m *SampleRowKeysResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _BigtableService_MutateRow_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(MutateRowRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableServiceServer).MutateRow(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableService_CheckAndMutateRow_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(CheckAndMutateRowRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableServiceServer).CheckAndMutateRow(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableService_ReadModifyWriteRow_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(ReadModifyWriteRowRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableServiceServer).ReadModifyWriteRow(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +var _BigtableService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "google.bigtable.v1.BigtableService", + HandlerType: (*BigtableServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "MutateRow", + Handler: _BigtableService_MutateRow_Handler, + }, + { + MethodName: "CheckAndMutateRow", + Handler: _BigtableService_CheckAndMutateRow_Handler, + }, + { + MethodName: "ReadModifyWriteRow", + Handler: _BigtableService_ReadModifyWriteRow_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "ReadRows", + Handler: _BigtableService_ReadRows_Handler, + ServerStreams: true, + }, + { + StreamName: "SampleRowKeys", + Handler: _BigtableService_SampleRowKeys_Handler, + ServerStreams: true, + }, + }, +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.proto new file mode 100644 index 0000000000..814940a18b --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.proto @@ -0,0 +1,60 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.v1; + +import "google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto"; +import "google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.proto"; +import "google.golang.org/cloud/bigtable/internal/empty/empty.proto"; + +option java_generic_services = true; +option java_multiple_files = true; +option java_outer_classname = "BigtableServicesProto"; +option java_package = "com.google.bigtable.v1"; + + +// Service for reading from and writing to existing Bigtables. +service BigtableService { + // Streams back the contents of all requested rows, optionally applying + // the same Reader filter to each. Depending on their size, rows may be + // broken up across multiple responses, but atomicity of each row will still + // be preserved. + rpc ReadRows(ReadRowsRequest) returns (stream ReadRowsResponse) { + } + + // Returns a sample of row keys in the table. The returned row keys will + // delimit contiguous sections of the table of approximately equal size, + // which can be used to break up the data for distributed tasks like + // mapreduces. + rpc SampleRowKeys(SampleRowKeysRequest) returns (stream SampleRowKeysResponse) { + } + + // Mutates a row atomically. Cells already present in the row are left + // unchanged unless explicitly changed by 'mutation'. + rpc MutateRow(MutateRowRequest) returns (google.protobuf.Empty) { + } + + // Mutates a row atomically based on the output of a predicate Reader filter. + rpc CheckAndMutateRow(CheckAndMutateRowRequest) returns (CheckAndMutateRowResponse) { + } + + // Modifies a row atomically, reading the latest existing timestamp/value from + // the specified columns and writing a new value at + // max(existing timestamp, current server time) based on pre-defined + // read/modify/write rules. Returns the new contents of all modified cells. + rpc ReadModifyWriteRow(ReadModifyWriteRowRequest) returns (Row) { + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.pb.go new file mode 100644 index 0000000000..0b870d00a1 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.pb.go @@ -0,0 +1,444 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.proto +// DO NOT EDIT! + +/* +Package google_bigtable_v1 is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.proto + google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service.proto + +It has these top-level messages: + ReadRowsRequest + ReadRowsResponse + SampleRowKeysRequest + SampleRowKeysResponse + MutateRowRequest + CheckAndMutateRowRequest + CheckAndMutateRowResponse + ReadModifyWriteRowRequest +*/ +package google_bigtable_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_bigtable_v11 "google.golang.org/cloud/bigtable/internal/data_proto" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// Request message for BigtableServer.ReadRows. +type ReadRowsRequest struct { + // The unique name of the table from which to read. + TableName string `protobuf:"bytes,1,opt,name=table_name" json:"table_name,omitempty"` + // If neither row_key nor row_range is set, reads from all rows. + // + // Types that are valid to be assigned to Target: + // *ReadRowsRequest_RowKey + // *ReadRowsRequest_RowRange + Target isReadRowsRequest_Target `protobuf_oneof:"target"` + // The filter to apply to the contents of the specified row(s). If unset, + // reads the entire table. + Filter *google_bigtable_v11.RowFilter `protobuf:"bytes,5,opt,name=filter" json:"filter,omitempty"` + // By default, rows are read sequentially, producing results which are + // guaranteed to arrive in increasing row order. Setting + // "allow_row_interleaving" to true allows multiple rows to be interleaved in + // the response stream, which increases throughput but breaks this guarantee, + // and may force the client to use more memory to buffer partially-received + // rows. Cannot be set to true when specifying "num_rows_limit". + AllowRowInterleaving bool `protobuf:"varint,6,opt,name=allow_row_interleaving" json:"allow_row_interleaving,omitempty"` + // The read will terminate after committing to N rows' worth of results. The + // default (zero) is to return all results. + // Note that "allow_row_interleaving" cannot be set to true when this is set. + NumRowsLimit int64 `protobuf:"varint,7,opt,name=num_rows_limit" json:"num_rows_limit,omitempty"` +} + +func (m *ReadRowsRequest) Reset() { *m = ReadRowsRequest{} } +func (m *ReadRowsRequest) String() string { return proto.CompactTextString(m) } +func (*ReadRowsRequest) ProtoMessage() {} + +type isReadRowsRequest_Target interface { + isReadRowsRequest_Target() +} + +type ReadRowsRequest_RowKey struct { + RowKey []byte `protobuf:"bytes,2,opt,name=row_key,proto3,oneof"` +} +type ReadRowsRequest_RowRange struct { + RowRange *google_bigtable_v11.RowRange `protobuf:"bytes,3,opt,name=row_range,oneof"` +} + +func (*ReadRowsRequest_RowKey) isReadRowsRequest_Target() {} +func (*ReadRowsRequest_RowRange) isReadRowsRequest_Target() {} + +func (m *ReadRowsRequest) GetTarget() isReadRowsRequest_Target { + if m != nil { + return m.Target + } + return nil +} + +func (m *ReadRowsRequest) GetRowKey() []byte { + if x, ok := m.GetTarget().(*ReadRowsRequest_RowKey); ok { + return x.RowKey + } + return nil +} + +func (m *ReadRowsRequest) GetRowRange() *google_bigtable_v11.RowRange { + if x, ok := m.GetTarget().(*ReadRowsRequest_RowRange); ok { + return x.RowRange + } + return nil +} + +func (m *ReadRowsRequest) GetFilter() *google_bigtable_v11.RowFilter { + if m != nil { + return m.Filter + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ReadRowsRequest) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _ReadRowsRequest_OneofMarshaler, _ReadRowsRequest_OneofUnmarshaler, []interface{}{ + (*ReadRowsRequest_RowKey)(nil), + (*ReadRowsRequest_RowRange)(nil), + } +} + +func _ReadRowsRequest_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ReadRowsRequest) + // target + switch x := m.Target.(type) { + case *ReadRowsRequest_RowKey: + b.EncodeVarint(2<<3 | proto.WireBytes) + b.EncodeRawBytes(x.RowKey) + case *ReadRowsRequest_RowRange: + b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.RowRange); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("ReadRowsRequest.Target has unexpected type %T", x) + } + return nil +} + +func _ReadRowsRequest_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ReadRowsRequest) + switch tag { + case 2: // target.row_key + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.Target = &ReadRowsRequest_RowKey{x} + return true, err + case 3: // target.row_range + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(google_bigtable_v11.RowRange) + err := b.DecodeMessage(msg) + m.Target = &ReadRowsRequest_RowRange{msg} + return true, err + default: + return false, nil + } +} + +// Response message for BigtableService.ReadRows. +type ReadRowsResponse struct { + // The key of the row for which we're receiving data. + // Results will be received in increasing row key order, unless + // "allow_row_interleaving" was specified in the request. + RowKey []byte `protobuf:"bytes,1,opt,name=row_key,proto3" json:"row_key,omitempty"` + // One or more chunks of the row specified by "row_key". + Chunks []*ReadRowsResponse_Chunk `protobuf:"bytes,2,rep,name=chunks" json:"chunks,omitempty"` +} + +func (m *ReadRowsResponse) Reset() { *m = ReadRowsResponse{} } +func (m *ReadRowsResponse) String() string { return proto.CompactTextString(m) } +func (*ReadRowsResponse) ProtoMessage() {} + +func (m *ReadRowsResponse) GetChunks() []*ReadRowsResponse_Chunk { + if m != nil { + return m.Chunks + } + return nil +} + +// Specifies a piece of a row's contents returned as part of the read +// response stream. +type ReadRowsResponse_Chunk struct { + // Types that are valid to be assigned to Chunk: + // *ReadRowsResponse_Chunk_RowContents + // *ReadRowsResponse_Chunk_ResetRow + // *ReadRowsResponse_Chunk_CommitRow + Chunk isReadRowsResponse_Chunk_Chunk `protobuf_oneof:"chunk"` +} + +func (m *ReadRowsResponse_Chunk) Reset() { *m = ReadRowsResponse_Chunk{} } +func (m *ReadRowsResponse_Chunk) String() string { return proto.CompactTextString(m) } +func (*ReadRowsResponse_Chunk) ProtoMessage() {} + +type isReadRowsResponse_Chunk_Chunk interface { + isReadRowsResponse_Chunk_Chunk() +} + +type ReadRowsResponse_Chunk_RowContents struct { + RowContents *google_bigtable_v11.Family `protobuf:"bytes,1,opt,name=row_contents,oneof"` +} +type ReadRowsResponse_Chunk_ResetRow struct { + ResetRow bool `protobuf:"varint,2,opt,name=reset_row,oneof"` +} +type ReadRowsResponse_Chunk_CommitRow struct { + CommitRow bool `protobuf:"varint,3,opt,name=commit_row,oneof"` +} + +func (*ReadRowsResponse_Chunk_RowContents) isReadRowsResponse_Chunk_Chunk() {} +func (*ReadRowsResponse_Chunk_ResetRow) isReadRowsResponse_Chunk_Chunk() {} +func (*ReadRowsResponse_Chunk_CommitRow) isReadRowsResponse_Chunk_Chunk() {} + +func (m *ReadRowsResponse_Chunk) GetChunk() isReadRowsResponse_Chunk_Chunk { + if m != nil { + return m.Chunk + } + return nil +} + +func (m *ReadRowsResponse_Chunk) GetRowContents() *google_bigtable_v11.Family { + if x, ok := m.GetChunk().(*ReadRowsResponse_Chunk_RowContents); ok { + return x.RowContents + } + return nil +} + +func (m *ReadRowsResponse_Chunk) GetResetRow() bool { + if x, ok := m.GetChunk().(*ReadRowsResponse_Chunk_ResetRow); ok { + return x.ResetRow + } + return false +} + +func (m *ReadRowsResponse_Chunk) GetCommitRow() bool { + if x, ok := m.GetChunk().(*ReadRowsResponse_Chunk_CommitRow); ok { + return x.CommitRow + } + return false +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ReadRowsResponse_Chunk) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _ReadRowsResponse_Chunk_OneofMarshaler, _ReadRowsResponse_Chunk_OneofUnmarshaler, []interface{}{ + (*ReadRowsResponse_Chunk_RowContents)(nil), + (*ReadRowsResponse_Chunk_ResetRow)(nil), + (*ReadRowsResponse_Chunk_CommitRow)(nil), + } +} + +func _ReadRowsResponse_Chunk_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ReadRowsResponse_Chunk) + // chunk + switch x := m.Chunk.(type) { + case *ReadRowsResponse_Chunk_RowContents: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.RowContents); err != nil { + return err + } + case *ReadRowsResponse_Chunk_ResetRow: + t := uint64(0) + if x.ResetRow { + t = 1 + } + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *ReadRowsResponse_Chunk_CommitRow: + t := uint64(0) + if x.CommitRow { + t = 1 + } + b.EncodeVarint(3<<3 | proto.WireVarint) + b.EncodeVarint(t) + case nil: + default: + return fmt.Errorf("ReadRowsResponse_Chunk.Chunk has unexpected type %T", x) + } + return nil +} + +func _ReadRowsResponse_Chunk_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ReadRowsResponse_Chunk) + switch tag { + case 1: // chunk.row_contents + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(google_bigtable_v11.Family) + err := b.DecodeMessage(msg) + m.Chunk = &ReadRowsResponse_Chunk_RowContents{msg} + return true, err + case 2: // chunk.reset_row + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Chunk = &ReadRowsResponse_Chunk_ResetRow{x != 0} + return true, err + case 3: // chunk.commit_row + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Chunk = &ReadRowsResponse_Chunk_CommitRow{x != 0} + return true, err + default: + return false, nil + } +} + +// Request message for BigtableService.SampleRowKeys. +type SampleRowKeysRequest struct { + // The unique name of the table from which to sample row keys. + TableName string `protobuf:"bytes,1,opt,name=table_name" json:"table_name,omitempty"` +} + +func (m *SampleRowKeysRequest) Reset() { *m = SampleRowKeysRequest{} } +func (m *SampleRowKeysRequest) String() string { return proto.CompactTextString(m) } +func (*SampleRowKeysRequest) ProtoMessage() {} + +// Response message for BigtableService.SampleRowKeys. +type SampleRowKeysResponse struct { + // Sorted streamed sequence of sample row keys in the table. The table might + // have contents before the first row key in the list and after the last one, + // but a key containing the empty string indicates "end of table" and will be + // the last response given, if present. + // Note that row keys in this list may not have ever been written to or read + // from, and users should therefore not make any assumptions about the row key + // structure that are specific to their use case. + RowKey []byte `protobuf:"bytes,1,opt,name=row_key,proto3" json:"row_key,omitempty"` + // Approximate total storage space used by all rows in the table which precede + // "row_key". Buffering the contents of all rows between two subsequent + // samples would require space roughly equal to the difference in their + // "offset_bytes" fields. + OffsetBytes int64 `protobuf:"varint,2,opt,name=offset_bytes" json:"offset_bytes,omitempty"` +} + +func (m *SampleRowKeysResponse) Reset() { *m = SampleRowKeysResponse{} } +func (m *SampleRowKeysResponse) String() string { return proto.CompactTextString(m) } +func (*SampleRowKeysResponse) ProtoMessage() {} + +// Request message for BigtableService.MutateRow. +type MutateRowRequest struct { + // The unique name of the table to which the mutation should be applied. + TableName string `protobuf:"bytes,1,opt,name=table_name" json:"table_name,omitempty"` + // The key of the row to which the mutation should be applied. + RowKey []byte `protobuf:"bytes,2,opt,name=row_key,proto3" json:"row_key,omitempty"` + // Changes to be atomically applied to the specified row. Entries are applied + // in order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry and at most 100000. + Mutations []*google_bigtable_v11.Mutation `protobuf:"bytes,3,rep,name=mutations" json:"mutations,omitempty"` +} + +func (m *MutateRowRequest) Reset() { *m = MutateRowRequest{} } +func (m *MutateRowRequest) String() string { return proto.CompactTextString(m) } +func (*MutateRowRequest) ProtoMessage() {} + +func (m *MutateRowRequest) GetMutations() []*google_bigtable_v11.Mutation { + if m != nil { + return m.Mutations + } + return nil +} + +// Request message for BigtableService.CheckAndMutateRowRequest +type CheckAndMutateRowRequest struct { + // The unique name of the table to which the conditional mutation should be + // applied. + TableName string `protobuf:"bytes,1,opt,name=table_name" json:"table_name,omitempty"` + // The key of the row to which the conditional mutation should be applied. + RowKey []byte `protobuf:"bytes,2,opt,name=row_key,proto3" json:"row_key,omitempty"` + // The filter to be applied to the contents of the specified row. Depending + // on whether or not any results are yielded, either "true_mutations" or + // "false_mutations" will be executed. If unset, checks that the row contains + // any values at all. + PredicateFilter *google_bigtable_v11.RowFilter `protobuf:"bytes,6,opt,name=predicate_filter" json:"predicate_filter,omitempty"` + // Changes to be atomically applied to the specified row if "predicate_filter" + // yields at least one cell when applied to "row_key". Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if "false_mutations" is empty, and at most + // 100000. + TrueMutations []*google_bigtable_v11.Mutation `protobuf:"bytes,4,rep,name=true_mutations" json:"true_mutations,omitempty"` + // Changes to be atomically applied to the specified row if "predicate_filter" + // does not yield any cells when applied to "row_key". Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if "true_mutations" is empty, and at most + // 100000. + FalseMutations []*google_bigtable_v11.Mutation `protobuf:"bytes,5,rep,name=false_mutations" json:"false_mutations,omitempty"` +} + +func (m *CheckAndMutateRowRequest) Reset() { *m = CheckAndMutateRowRequest{} } +func (m *CheckAndMutateRowRequest) String() string { return proto.CompactTextString(m) } +func (*CheckAndMutateRowRequest) ProtoMessage() {} + +func (m *CheckAndMutateRowRequest) GetPredicateFilter() *google_bigtable_v11.RowFilter { + if m != nil { + return m.PredicateFilter + } + return nil +} + +func (m *CheckAndMutateRowRequest) GetTrueMutations() []*google_bigtable_v11.Mutation { + if m != nil { + return m.TrueMutations + } + return nil +} + +func (m *CheckAndMutateRowRequest) GetFalseMutations() []*google_bigtable_v11.Mutation { + if m != nil { + return m.FalseMutations + } + return nil +} + +// Response message for BigtableService.CheckAndMutateRowRequest. +type CheckAndMutateRowResponse struct { + // Whether or not the request's "predicate_filter" yielded any results for + // the specified row. + PredicateMatched bool `protobuf:"varint,1,opt,name=predicate_matched" json:"predicate_matched,omitempty"` +} + +func (m *CheckAndMutateRowResponse) Reset() { *m = CheckAndMutateRowResponse{} } +func (m *CheckAndMutateRowResponse) String() string { return proto.CompactTextString(m) } +func (*CheckAndMutateRowResponse) ProtoMessage() {} + +// Request message for BigtableService.ReadModifyWriteRowRequest. +type ReadModifyWriteRowRequest struct { + // The unique name of the table to which the read/modify/write rules should be + // applied. + TableName string `protobuf:"bytes,1,opt,name=table_name" json:"table_name,omitempty"` + // The key of the row to which the read/modify/write rules should be applied. + RowKey []byte `protobuf:"bytes,2,opt,name=row_key,proto3" json:"row_key,omitempty"` + // Rules specifying how the specified row's contents are to be transformed + // into writes. Entries are applied in order, meaning that earlier rules will + // affect the results of later ones. + Rules []*google_bigtable_v11.ReadModifyWriteRule `protobuf:"bytes,3,rep,name=rules" json:"rules,omitempty"` +} + +func (m *ReadModifyWriteRowRequest) Reset() { *m = ReadModifyWriteRowRequest{} } +func (m *ReadModifyWriteRowRequest) String() string { return proto.CompactTextString(m) } +func (*ReadModifyWriteRowRequest) ProtoMessage() {} + +func (m *ReadModifyWriteRowRequest) GetRules() []*google_bigtable_v11.ReadModifyWriteRule { + if m != nil { + return m.Rules + } + return nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.proto new file mode 100644 index 0000000000..661310ad4a --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.proto @@ -0,0 +1,177 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.v1; + +import "google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableServiceMessagesProto"; +option java_package = "com.google.bigtable.v1"; + + +// Request message for BigtableServer.ReadRows. +message ReadRowsRequest { + // The unique name of the table from which to read. + string table_name = 1; + + // If neither row_key nor row_range is set, reads from all rows. + oneof target { + // The key of a single row from which to read. + bytes row_key = 2; + + // A range of rows from which to read. + RowRange row_range = 3; + } + + // The filter to apply to the contents of the specified row(s). If unset, + // reads the entire table. + RowFilter filter = 5; + + // By default, rows are read sequentially, producing results which are + // guaranteed to arrive in increasing row order. Setting + // "allow_row_interleaving" to true allows multiple rows to be interleaved in + // the response stream, which increases throughput but breaks this guarantee, + // and may force the client to use more memory to buffer partially-received + // rows. Cannot be set to true when specifying "num_rows_limit". + bool allow_row_interleaving = 6; + + // The read will terminate after committing to N rows' worth of results. The + // default (zero) is to return all results. + // Note that "allow_row_interleaving" cannot be set to true when this is set. + int64 num_rows_limit = 7; +} + +// Response message for BigtableService.ReadRows. +message ReadRowsResponse { + // Specifies a piece of a row's contents returned as part of the read + // response stream. + message Chunk { + oneof chunk { + // A subset of the data from a particular row. As long as no "reset_row" + // is received in between, multiple "row_contents" from the same row are + // from the same atomic view of that row, and will be received in the + // expected family/column/timestamp order. + Family row_contents = 1; + + // Indicates that the client should drop all previous chunks for + // "row_key", as it will be re-read from the beginning. + bool reset_row = 2; + + // Indicates that the client can safely process all previous chunks for + // "row_key", as its data has been fully read. + bool commit_row = 3; + } + } + + // The key of the row for which we're receiving data. + // Results will be received in increasing row key order, unless + // "allow_row_interleaving" was specified in the request. + bytes row_key = 1; + + // One or more chunks of the row specified by "row_key". + repeated Chunk chunks = 2; +} + +// Request message for BigtableService.SampleRowKeys. +message SampleRowKeysRequest { + // The unique name of the table from which to sample row keys. + string table_name = 1; +} + +// Response message for BigtableService.SampleRowKeys. +message SampleRowKeysResponse { + // Sorted streamed sequence of sample row keys in the table. The table might + // have contents before the first row key in the list and after the last one, + // but a key containing the empty string indicates "end of table" and will be + // the last response given, if present. + // Note that row keys in this list may not have ever been written to or read + // from, and users should therefore not make any assumptions about the row key + // structure that are specific to their use case. + bytes row_key = 1; + + // Approximate total storage space used by all rows in the table which precede + // "row_key". Buffering the contents of all rows between two subsequent + // samples would require space roughly equal to the difference in their + // "offset_bytes" fields. + int64 offset_bytes = 2; +} + +// Request message for BigtableService.MutateRow. +message MutateRowRequest { + // The unique name of the table to which the mutation should be applied. + string table_name = 1; + + // The key of the row to which the mutation should be applied. + bytes row_key = 2; + + // Changes to be atomically applied to the specified row. Entries are applied + // in order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry and at most 100000. + repeated Mutation mutations = 3; +} + +// Request message for BigtableService.CheckAndMutateRowRequest +message CheckAndMutateRowRequest { + // The unique name of the table to which the conditional mutation should be + // applied. + string table_name = 1; + + // The key of the row to which the conditional mutation should be applied. + bytes row_key = 2; + + // The filter to be applied to the contents of the specified row. Depending + // on whether or not any results are yielded, either "true_mutations" or + // "false_mutations" will be executed. If unset, checks that the row contains + // any values at all. + RowFilter predicate_filter = 6; + + // Changes to be atomically applied to the specified row if "predicate_filter" + // yields at least one cell when applied to "row_key". Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if "false_mutations" is empty, and at most + // 100000. + repeated Mutation true_mutations = 4; + + // Changes to be atomically applied to the specified row if "predicate_filter" + // does not yield any cells when applied to "row_key". Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if "true_mutations" is empty, and at most + // 100000. + repeated Mutation false_mutations = 5; +} + +// Response message for BigtableService.CheckAndMutateRowRequest. +message CheckAndMutateRowResponse { + // Whether or not the request's "predicate_filter" yielded any results for + // the specified row. + bool predicate_matched = 1; +} + +// Request message for BigtableService.ReadModifyWriteRowRequest. +message ReadModifyWriteRowRequest { + // The unique name of the table to which the read/modify/write rules should be + // applied. + string table_name = 1; + + // The key of the row to which the read/modify/write rules should be applied. + bytes row_key = 2; + + // Rules specifying how the specified row's contents are to be transformed + // into writes. Entries are applied in order, meaning that earlier rules will + // affect the results of later ones. + repeated ReadModifyWriteRule rules = 3; +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.pb.go new file mode 100644 index 0000000000..42d8b7bb31 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.pb.go @@ -0,0 +1,308 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto +// DO NOT EDIT! + +/* +Package google_bigtable_admin_table_v1 is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto + +It has these top-level messages: + Table + ColumnFamily + GcRule +*/ +package google_bigtable_admin_table_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_protobuf "google.golang.org/cloud/bigtable/internal/duration_proto" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type Table_TimestampGranularity int32 + +const ( + Table_MILLIS Table_TimestampGranularity = 0 +) + +var Table_TimestampGranularity_name = map[int32]string{ + 0: "MILLIS", +} +var Table_TimestampGranularity_value = map[string]int32{ + "MILLIS": 0, +} + +func (x Table_TimestampGranularity) String() string { + return proto.EnumName(Table_TimestampGranularity_name, int32(x)) +} + +// A collection of user data indexed by row, column, and timestamp. +// Each table is served using the resources of its parent cluster. +type Table struct { + // A unique identifier of the form + // /tables/[_a-zA-Z0-9][-_.a-zA-Z0-9]* + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The column families configured for this table, mapped by column family id. + ColumnFamilies map[string]*ColumnFamily `protobuf:"bytes,3,rep,name=column_families" json:"column_families,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // The granularity (e.g. MILLIS, MICROS) at which timestamps are stored in + // this table. Timestamps not matching the granularity will be rejected. + // Cannot be changed once the table is created. + Granularity Table_TimestampGranularity `protobuf:"varint,4,opt,name=granularity,enum=google.bigtable.admin.table.v1.Table_TimestampGranularity" json:"granularity,omitempty"` +} + +func (m *Table) Reset() { *m = Table{} } +func (m *Table) String() string { return proto.CompactTextString(m) } +func (*Table) ProtoMessage() {} + +func (m *Table) GetColumnFamilies() map[string]*ColumnFamily { + if m != nil { + return m.ColumnFamilies + } + return nil +} + +// A set of columns within a table which share a common configuration. +type ColumnFamily struct { + // A unique identifier of the form /columnFamilies/[-_.a-zA-Z0-9]+ + // The last segment is the same as the "name" field in + // google.bigtable.v1.Family. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // Garbage collection expression specified by the following grammar: + // GC = EXPR + // | "" ; + // EXPR = EXPR, "||", EXPR (* lowest precedence *) + // | EXPR, "&&", EXPR + // | "(", EXPR, ")" (* highest precedence *) + // | PROP ; + // PROP = "version() >", NUM32 + // | "age() >", NUM64, [ UNIT ] ; + // NUM32 = non-zero-digit { digit } ; (* # NUM32 <= 2^32 - 1 *) + // NUM64 = non-zero-digit { digit } ; (* # NUM64 <= 2^63 - 1 *) + // UNIT = "d" | "h" | "m" (* d=days, h=hours, m=minutes, else micros *) + // GC expressions can be up to 500 characters in length + // + // The different types of PROP are defined as follows: + // version() - cell index, counting from most recent and starting at 1 + // age() - age of the cell (current time minus cell timestamp) + // + // Example: "version() > 3 || (age() > 3d && version() > 1)" + // drop cells beyond the most recent three, and drop cells older than three + // days unless they're the most recent cell in the row/column + // + // Garbage collection executes opportunistically in the background, and so + // it's possible for reads to return a cell even if it matches the active GC + // expression for its family. + GcExpression string `protobuf:"bytes,2,opt,name=gc_expression" json:"gc_expression,omitempty"` + // Garbage collection rule specified as a protobuf. + // Supersedes `gc_expression`. + // Must serialize to at most 500 bytes. + // + // NOTE: Garbage collection executes opportunistically in the background, and + // so it's possible for reads to return a cell even if it matches the active + // GC expression for its family. + GcRule *GcRule `protobuf:"bytes,3,opt,name=gc_rule" json:"gc_rule,omitempty"` +} + +func (m *ColumnFamily) Reset() { *m = ColumnFamily{} } +func (m *ColumnFamily) String() string { return proto.CompactTextString(m) } +func (*ColumnFamily) ProtoMessage() {} + +func (m *ColumnFamily) GetGcRule() *GcRule { + if m != nil { + return m.GcRule + } + return nil +} + +// Rule for determining which cells to delete during garbage collection. +type GcRule struct { + // Types that are valid to be assigned to Rule: + // *GcRule_MaxNumVersions + // *GcRule_MaxAge + // *GcRule_Intersection_ + // *GcRule_Union_ + Rule isGcRule_Rule `protobuf_oneof:"rule"` +} + +func (m *GcRule) Reset() { *m = GcRule{} } +func (m *GcRule) String() string { return proto.CompactTextString(m) } +func (*GcRule) ProtoMessage() {} + +type isGcRule_Rule interface { + isGcRule_Rule() +} + +type GcRule_MaxNumVersions struct { + MaxNumVersions int32 `protobuf:"varint,1,opt,name=max_num_versions,oneof"` +} +type GcRule_MaxAge struct { + MaxAge *google_protobuf.Duration `protobuf:"bytes,2,opt,name=max_age,oneof"` +} +type GcRule_Intersection_ struct { + Intersection *GcRule_Intersection `protobuf:"bytes,3,opt,name=intersection,oneof"` +} +type GcRule_Union_ struct { + Union *GcRule_Union `protobuf:"bytes,4,opt,name=union,oneof"` +} + +func (*GcRule_MaxNumVersions) isGcRule_Rule() {} +func (*GcRule_MaxAge) isGcRule_Rule() {} +func (*GcRule_Intersection_) isGcRule_Rule() {} +func (*GcRule_Union_) isGcRule_Rule() {} + +func (m *GcRule) GetRule() isGcRule_Rule { + if m != nil { + return m.Rule + } + return nil +} + +func (m *GcRule) GetMaxNumVersions() int32 { + if x, ok := m.GetRule().(*GcRule_MaxNumVersions); ok { + return x.MaxNumVersions + } + return 0 +} + +func (m *GcRule) GetMaxAge() *google_protobuf.Duration { + if x, ok := m.GetRule().(*GcRule_MaxAge); ok { + return x.MaxAge + } + return nil +} + +func (m *GcRule) GetIntersection() *GcRule_Intersection { + if x, ok := m.GetRule().(*GcRule_Intersection_); ok { + return x.Intersection + } + return nil +} + +func (m *GcRule) GetUnion() *GcRule_Union { + if x, ok := m.GetRule().(*GcRule_Union_); ok { + return x.Union + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*GcRule) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), []interface{}) { + return _GcRule_OneofMarshaler, _GcRule_OneofUnmarshaler, []interface{}{ + (*GcRule_MaxNumVersions)(nil), + (*GcRule_MaxAge)(nil), + (*GcRule_Intersection_)(nil), + (*GcRule_Union_)(nil), + } +} + +func _GcRule_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*GcRule) + // rule + switch x := m.Rule.(type) { + case *GcRule_MaxNumVersions: + b.EncodeVarint(1<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.MaxNumVersions)) + case *GcRule_MaxAge: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.MaxAge); err != nil { + return err + } + case *GcRule_Intersection_: + b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Intersection); err != nil { + return err + } + case *GcRule_Union_: + b.EncodeVarint(4<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Union); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("GcRule.Rule has unexpected type %T", x) + } + return nil +} + +func _GcRule_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*GcRule) + switch tag { + case 1: // rule.max_num_versions + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.Rule = &GcRule_MaxNumVersions{int32(x)} + return true, err + case 2: // rule.max_age + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(google_protobuf.Duration) + err := b.DecodeMessage(msg) + m.Rule = &GcRule_MaxAge{msg} + return true, err + case 3: // rule.intersection + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(GcRule_Intersection) + err := b.DecodeMessage(msg) + m.Rule = &GcRule_Intersection_{msg} + return true, err + case 4: // rule.union + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(GcRule_Union) + err := b.DecodeMessage(msg) + m.Rule = &GcRule_Union_{msg} + return true, err + default: + return false, nil + } +} + +// A GcRule which deletes cells matching all of the given rules. +type GcRule_Intersection struct { + // Only delete cells which would be deleted by every element of `rules`. + Rules []*GcRule `protobuf:"bytes,1,rep,name=rules" json:"rules,omitempty"` +} + +func (m *GcRule_Intersection) Reset() { *m = GcRule_Intersection{} } +func (m *GcRule_Intersection) String() string { return proto.CompactTextString(m) } +func (*GcRule_Intersection) ProtoMessage() {} + +func (m *GcRule_Intersection) GetRules() []*GcRule { + if m != nil { + return m.Rules + } + return nil +} + +// A GcRule which deletes cells matching any of the given rules. +type GcRule_Union struct { + // Delete cells which would be deleted by any element of `rules`. + Rules []*GcRule `protobuf:"bytes,1,rep,name=rules" json:"rules,omitempty"` +} + +func (m *GcRule_Union) Reset() { *m = GcRule_Union{} } +func (m *GcRule_Union) String() string { return proto.CompactTextString(m) } +func (*GcRule_Union) ProtoMessage() {} + +func (m *GcRule_Union) GetRules() []*GcRule { + if m != nil { + return m.Rules + } + return nil +} + +func init() { + proto.RegisterEnum("google.bigtable.admin.table.v1.Table_TimestampGranularity", Table_TimestampGranularity_name, Table_TimestampGranularity_value) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto new file mode 100644 index 0000000000..a815152d16 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto @@ -0,0 +1,123 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.table.v1; + +import "google.golang.org/cloud/bigtable/internal/duration_proto/duration.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableTableDataProto"; +option java_package = "com.google.bigtable.admin.table.v1"; + + +// A collection of user data indexed by row, column, and timestamp. +// Each table is served using the resources of its parent cluster. +message Table { + enum TimestampGranularity { + MILLIS = 0; + } + + // A unique identifier of the form + // /tables/[_a-zA-Z0-9][-_.a-zA-Z0-9]* + string name = 1; + + // If this Table is in the process of being created, the Operation used to + // track its progress. As long as this operation is present, the Table will + // not accept any Table Admin or Read/Write requests. + + // The column families configured for this table, mapped by column family id. + map column_families = 3; + + // The granularity (e.g. MILLIS, MICROS) at which timestamps are stored in + // this table. Timestamps not matching the granularity will be rejected. + // Cannot be changed once the table is created. + TimestampGranularity granularity = 4; +} + +// A set of columns within a table which share a common configuration. +message ColumnFamily { + // A unique identifier of the form /columnFamilies/[-_.a-zA-Z0-9]+ + // The last segment is the same as the "name" field in + // google.bigtable.v1.Family. + string name = 1; + + // Garbage collection expression specified by the following grammar: + // GC = EXPR + // | "" ; + // EXPR = EXPR, "||", EXPR (* lowest precedence *) + // | EXPR, "&&", EXPR + // | "(", EXPR, ")" (* highest precedence *) + // | PROP ; + // PROP = "version() >", NUM32 + // | "age() >", NUM64, [ UNIT ] ; + // NUM32 = non-zero-digit { digit } ; (* # NUM32 <= 2^32 - 1 *) + // NUM64 = non-zero-digit { digit } ; (* # NUM64 <= 2^63 - 1 *) + // UNIT = "d" | "h" | "m" (* d=days, h=hours, m=minutes, else micros *) + // GC expressions can be up to 500 characters in length + // + // The different types of PROP are defined as follows: + // version() - cell index, counting from most recent and starting at 1 + // age() - age of the cell (current time minus cell timestamp) + // + // Example: "version() > 3 || (age() > 3d && version() > 1)" + // drop cells beyond the most recent three, and drop cells older than three + // days unless they're the most recent cell in the row/column + // + // Garbage collection executes opportunistically in the background, and so + // it's possible for reads to return a cell even if it matches the active GC + // expression for its family. + string gc_expression = 2; + + // Garbage collection rule specified as a protobuf. + // Supersedes `gc_expression`. + // Must serialize to at most 500 bytes. + // + // NOTE: Garbage collection executes opportunistically in the background, and + // so it's possible for reads to return a cell even if it matches the active + // GC expression for its family. + GcRule gc_rule = 3; +} + +// Rule for determining which cells to delete during garbage collection. +message GcRule { + // A GcRule which deletes cells matching all of the given rules. + message Intersection { + // Only delete cells which would be deleted by every element of `rules`. + repeated GcRule rules = 1; + } + + // A GcRule which deletes cells matching any of the given rules. + message Union { + // Delete cells which would be deleted by any element of `rules`. + repeated GcRule rules = 1; + } + + oneof rule { + // Delete all cells in a column except the most recent N. + int32 max_num_versions = 1; + + // Delete cells in a column older than the given age. + // Values must be at least one millisecond, and will be truncated to + // microsecond granularity. + google.protobuf.Duration max_age = 2; + + // Delete cells that would be deleted by every nested rule. + Intersection intersection = 3; + + // Delete cells that would be deleted by any nested rule. + Union union = 4; + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.pb.go new file mode 100644 index 0000000000..756f188892 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.pb.go @@ -0,0 +1,293 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.proto +// DO NOT EDIT! + +package google_bigtable_admin_table_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_bigtable_admin_table_v11 "google.golang.org/cloud/bigtable/internal/table_data_proto" +import google_protobuf1 "google.golang.org/cloud/bigtable/internal/empty" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// Client API for BigtableTableService service + +type BigtableTableServiceClient interface { + // Creates a new table, to be served from a specified cluster. + // The table can be created with a full set of initial column families, + // specified in the request. + CreateTable(ctx context.Context, in *CreateTableRequest, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.Table, error) + // Lists the names of all tables served from a specified cluster. + ListTables(ctx context.Context, in *ListTablesRequest, opts ...grpc.CallOption) (*ListTablesResponse, error) + // Gets the schema of the specified table, including its column families. + GetTable(ctx context.Context, in *GetTableRequest, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.Table, error) + // Permanently deletes a specified table and all of its data. + DeleteTable(ctx context.Context, in *DeleteTableRequest, opts ...grpc.CallOption) (*google_protobuf1.Empty, error) + // Changes the name of a specified table. + // Cannot be used to move tables between clusters, zones, or projects. + RenameTable(ctx context.Context, in *RenameTableRequest, opts ...grpc.CallOption) (*google_protobuf1.Empty, error) + // Creates a new column family within a specified table. + CreateColumnFamily(ctx context.Context, in *CreateColumnFamilyRequest, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.ColumnFamily, error) + // Changes the configuration of a specified column family. + UpdateColumnFamily(ctx context.Context, in *google_bigtable_admin_table_v11.ColumnFamily, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.ColumnFamily, error) + // Permanently deletes a specified column family and all of its data. + DeleteColumnFamily(ctx context.Context, in *DeleteColumnFamilyRequest, opts ...grpc.CallOption) (*google_protobuf1.Empty, error) +} + +type bigtableTableServiceClient struct { + cc *grpc.ClientConn +} + +func NewBigtableTableServiceClient(cc *grpc.ClientConn) BigtableTableServiceClient { + return &bigtableTableServiceClient{cc} +} + +func (c *bigtableTableServiceClient) CreateTable(ctx context.Context, in *CreateTableRequest, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.Table, error) { + out := new(google_bigtable_admin_table_v11.Table) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/CreateTable", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableTableServiceClient) ListTables(ctx context.Context, in *ListTablesRequest, opts ...grpc.CallOption) (*ListTablesResponse, error) { + out := new(ListTablesResponse) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/ListTables", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableTableServiceClient) GetTable(ctx context.Context, in *GetTableRequest, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.Table, error) { + out := new(google_bigtable_admin_table_v11.Table) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/GetTable", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableTableServiceClient) DeleteTable(ctx context.Context, in *DeleteTableRequest, opts ...grpc.CallOption) (*google_protobuf1.Empty, error) { + out := new(google_protobuf1.Empty) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/DeleteTable", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableTableServiceClient) RenameTable(ctx context.Context, in *RenameTableRequest, opts ...grpc.CallOption) (*google_protobuf1.Empty, error) { + out := new(google_protobuf1.Empty) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/RenameTable", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableTableServiceClient) CreateColumnFamily(ctx context.Context, in *CreateColumnFamilyRequest, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.ColumnFamily, error) { + out := new(google_bigtable_admin_table_v11.ColumnFamily) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/CreateColumnFamily", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableTableServiceClient) UpdateColumnFamily(ctx context.Context, in *google_bigtable_admin_table_v11.ColumnFamily, opts ...grpc.CallOption) (*google_bigtable_admin_table_v11.ColumnFamily, error) { + out := new(google_bigtable_admin_table_v11.ColumnFamily) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/UpdateColumnFamily", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *bigtableTableServiceClient) DeleteColumnFamily(ctx context.Context, in *DeleteColumnFamilyRequest, opts ...grpc.CallOption) (*google_protobuf1.Empty, error) { + out := new(google_protobuf1.Empty) + err := grpc.Invoke(ctx, "/google.bigtable.admin.table.v1.BigtableTableService/DeleteColumnFamily", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for BigtableTableService service + +type BigtableTableServiceServer interface { + // Creates a new table, to be served from a specified cluster. + // The table can be created with a full set of initial column families, + // specified in the request. + CreateTable(context.Context, *CreateTableRequest) (*google_bigtable_admin_table_v11.Table, error) + // Lists the names of all tables served from a specified cluster. + ListTables(context.Context, *ListTablesRequest) (*ListTablesResponse, error) + // Gets the schema of the specified table, including its column families. + GetTable(context.Context, *GetTableRequest) (*google_bigtable_admin_table_v11.Table, error) + // Permanently deletes a specified table and all of its data. + DeleteTable(context.Context, *DeleteTableRequest) (*google_protobuf1.Empty, error) + // Changes the name of a specified table. + // Cannot be used to move tables between clusters, zones, or projects. + RenameTable(context.Context, *RenameTableRequest) (*google_protobuf1.Empty, error) + // Creates a new column family within a specified table. + CreateColumnFamily(context.Context, *CreateColumnFamilyRequest) (*google_bigtable_admin_table_v11.ColumnFamily, error) + // Changes the configuration of a specified column family. + UpdateColumnFamily(context.Context, *google_bigtable_admin_table_v11.ColumnFamily) (*google_bigtable_admin_table_v11.ColumnFamily, error) + // Permanently deletes a specified column family and all of its data. + DeleteColumnFamily(context.Context, *DeleteColumnFamilyRequest) (*google_protobuf1.Empty, error) +} + +func RegisterBigtableTableServiceServer(s *grpc.Server, srv BigtableTableServiceServer) { + s.RegisterService(&_BigtableTableService_serviceDesc, srv) +} + +func _BigtableTableService_CreateTable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(CreateTableRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).CreateTable(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableTableService_ListTables_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(ListTablesRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).ListTables(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableTableService_GetTable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(GetTableRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).GetTable(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableTableService_DeleteTable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(DeleteTableRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).DeleteTable(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableTableService_RenameTable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(RenameTableRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).RenameTable(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableTableService_CreateColumnFamily_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(CreateColumnFamilyRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).CreateColumnFamily(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableTableService_UpdateColumnFamily_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(google_bigtable_admin_table_v11.ColumnFamily) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).UpdateColumnFamily(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +func _BigtableTableService_DeleteColumnFamily_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(DeleteColumnFamilyRequest) + if err := dec(in); err != nil { + return nil, err + } + out, err := srv.(BigtableTableServiceServer).DeleteColumnFamily(ctx, in) + if err != nil { + return nil, err + } + return out, nil +} + +var _BigtableTableService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "google.bigtable.admin.table.v1.BigtableTableService", + HandlerType: (*BigtableTableServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CreateTable", + Handler: _BigtableTableService_CreateTable_Handler, + }, + { + MethodName: "ListTables", + Handler: _BigtableTableService_ListTables_Handler, + }, + { + MethodName: "GetTable", + Handler: _BigtableTableService_GetTable_Handler, + }, + { + MethodName: "DeleteTable", + Handler: _BigtableTableService_DeleteTable_Handler, + }, + { + MethodName: "RenameTable", + Handler: _BigtableTableService_RenameTable_Handler, + }, + { + MethodName: "CreateColumnFamily", + Handler: _BigtableTableService_CreateColumnFamily_Handler, + }, + { + MethodName: "UpdateColumnFamily", + Handler: _BigtableTableService_UpdateColumnFamily_Handler, + }, + { + MethodName: "DeleteColumnFamily", + Handler: _BigtableTableService_DeleteColumnFamily_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.proto new file mode 100644 index 0000000000..1ccdfa24a9 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.proto @@ -0,0 +1,65 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.table.v1; + +import "google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto"; +import "google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto"; +import "google.golang.org/cloud/bigtable/internal/empty/empty.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableTableServicesProto"; +option java_package = "com.google.bigtable.admin.table.v1"; + + +// Service for creating, configuring, and deleting Cloud Bigtable tables. +// Provides access to the table schemas only, not the data stored within the tables. +service BigtableTableService { + // Creates a new table, to be served from a specified cluster. + // The table can be created with a full set of initial column families, + // specified in the request. + rpc CreateTable(CreateTableRequest) returns (Table) { + } + + // Lists the names of all tables served from a specified cluster. + rpc ListTables(ListTablesRequest) returns (ListTablesResponse) { + } + + // Gets the schema of the specified table, including its column families. + rpc GetTable(GetTableRequest) returns (Table) { + } + + // Permanently deletes a specified table and all of its data. + rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) { + } + + // Changes the name of a specified table. + // Cannot be used to move tables between clusters, zones, or projects. + rpc RenameTable(RenameTableRequest) returns (google.protobuf.Empty) { + } + + // Creates a new column family within a specified table. + rpc CreateColumnFamily(CreateColumnFamilyRequest) returns (ColumnFamily) { + } + + // Changes the configuration of a specified column family. + rpc UpdateColumnFamily(ColumnFamily) returns (ColumnFamily) { + } + + // Permanently deletes a specified column family and all of its data. + rpc DeleteColumnFamily(DeleteColumnFamilyRequest) returns (google.protobuf.Empty) { + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.pb.go new file mode 100644 index 0000000000..268f4ea0a5 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.pb.go @@ -0,0 +1,156 @@ +// Code generated by protoc-gen-go. +// source: google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto +// DO NOT EDIT! + +/* +Package google_bigtable_admin_table_v1 is a generated protocol buffer package. + +It is generated from these files: + google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto + google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service.proto + +It has these top-level messages: + CreateTableRequest + ListTablesRequest + ListTablesResponse + GetTableRequest + DeleteTableRequest + RenameTableRequest + CreateColumnFamilyRequest + DeleteColumnFamilyRequest +*/ +package google_bigtable_admin_table_v1 + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import google_bigtable_admin_table_v11 "google.golang.org/cloud/bigtable/internal/table_data_proto" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type CreateTableRequest struct { + // The unique name of the cluster in which to create the new table. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The name by which the new table should be referred to within the cluster, + // e.g. "foobar" rather than "/tables/foobar". + TableId string `protobuf:"bytes,2,opt,name=table_id" json:"table_id,omitempty"` + // The Table to create. The `name` field of the Table and all of its + // ColumnFamilies must be left blank, and will be populated in the response. + Table *google_bigtable_admin_table_v11.Table `protobuf:"bytes,3,opt,name=table" json:"table,omitempty"` + // The optional list of row keys that will be used to initially split the + // table into several tablets (Tablets are similar to HBase regions). + // Given two split keys, "s1" and "s2", three tablets will be created, + // spanning the key ranges: [, s1), [s1, s2), [s2, ). + // + // Example: + // * Row keys := ["a", "apple", "custom", "customer_1", "customer_2", + // "other", "zz"] + // * initial_split_keys := ["apple", "customer_1", "customer_2", "other"] + // * Key assignment: + // - Tablet 1 [, apple) => {"a"}. + // - Tablet 2 [apple, customer_1) => {"apple", "custom"}. + // - Tablet 3 [customer_1, customer_2) => {"customer_1"}. + // - Tablet 4 [customer_2, other) => {"customer_2"}. + // - Tablet 5 [other, ) => {"other", "zz"}. + InitialSplitKeys []string `protobuf:"bytes,4,rep,name=initial_split_keys" json:"initial_split_keys,omitempty"` +} + +func (m *CreateTableRequest) Reset() { *m = CreateTableRequest{} } +func (m *CreateTableRequest) String() string { return proto.CompactTextString(m) } +func (*CreateTableRequest) ProtoMessage() {} + +func (m *CreateTableRequest) GetTable() *google_bigtable_admin_table_v11.Table { + if m != nil { + return m.Table + } + return nil +} + +type ListTablesRequest struct { + // The unique name of the cluster for which tables should be listed. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *ListTablesRequest) Reset() { *m = ListTablesRequest{} } +func (m *ListTablesRequest) String() string { return proto.CompactTextString(m) } +func (*ListTablesRequest) ProtoMessage() {} + +type ListTablesResponse struct { + // The tables present in the requested cluster. + // At present, only the names of the tables are populated. + Tables []*google_bigtable_admin_table_v11.Table `protobuf:"bytes,1,rep,name=tables" json:"tables,omitempty"` +} + +func (m *ListTablesResponse) Reset() { *m = ListTablesResponse{} } +func (m *ListTablesResponse) String() string { return proto.CompactTextString(m) } +func (*ListTablesResponse) ProtoMessage() {} + +func (m *ListTablesResponse) GetTables() []*google_bigtable_admin_table_v11.Table { + if m != nil { + return m.Tables + } + return nil +} + +type GetTableRequest struct { + // The unique name of the requested table. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *GetTableRequest) Reset() { *m = GetTableRequest{} } +func (m *GetTableRequest) String() string { return proto.CompactTextString(m) } +func (*GetTableRequest) ProtoMessage() {} + +type DeleteTableRequest struct { + // The unique name of the table to be deleted. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *DeleteTableRequest) Reset() { *m = DeleteTableRequest{} } +func (m *DeleteTableRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteTableRequest) ProtoMessage() {} + +type RenameTableRequest struct { + // The current unique name of the table. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The new name by which the table should be referred to within its containing + // cluster, e.g. "foobar" rather than "/tables/foobar". + NewId string `protobuf:"bytes,2,opt,name=new_id" json:"new_id,omitempty"` +} + +func (m *RenameTableRequest) Reset() { *m = RenameTableRequest{} } +func (m *RenameTableRequest) String() string { return proto.CompactTextString(m) } +func (*RenameTableRequest) ProtoMessage() {} + +type CreateColumnFamilyRequest struct { + // The unique name of the table in which to create the new column family. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + // The name by which the new column family should be referred to within the + // table, e.g. "foobar" rather than "/columnFamilies/foobar". + ColumnFamilyId string `protobuf:"bytes,2,opt,name=column_family_id" json:"column_family_id,omitempty"` + // The column family to create. The `name` field must be left blank. + ColumnFamily *google_bigtable_admin_table_v11.ColumnFamily `protobuf:"bytes,3,opt,name=column_family" json:"column_family,omitempty"` +} + +func (m *CreateColumnFamilyRequest) Reset() { *m = CreateColumnFamilyRequest{} } +func (m *CreateColumnFamilyRequest) String() string { return proto.CompactTextString(m) } +func (*CreateColumnFamilyRequest) ProtoMessage() {} + +func (m *CreateColumnFamilyRequest) GetColumnFamily() *google_bigtable_admin_table_v11.ColumnFamily { + if m != nil { + return m.ColumnFamily + } + return nil +} + +type DeleteColumnFamilyRequest struct { + // The unique name of the column family to be deleted. + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` +} + +func (m *DeleteColumnFamilyRequest) Reset() { *m = DeleteColumnFamilyRequest{} } +func (m *DeleteColumnFamilyRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteColumnFamilyRequest) ProtoMessage() {} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto new file mode 100644 index 0000000000..9fa1b6a1c4 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto @@ -0,0 +1,101 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.table.v1; + +import "google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableTableServiceMessagesProto"; +option java_package = "com.google.bigtable.admin.table.v1"; + + +message CreateTableRequest { + // The unique name of the cluster in which to create the new table. + string name = 1; + + // The name by which the new table should be referred to within the cluster, + // e.g. "foobar" rather than "/tables/foobar". + string table_id = 2; + + // The Table to create. The `name` field of the Table and all of its + // ColumnFamilies must be left blank, and will be populated in the response. + Table table = 3; + + // The optional list of row keys that will be used to initially split the + // table into several tablets (Tablets are similar to HBase regions). + // Given two split keys, "s1" and "s2", three tablets will be created, + // spanning the key ranges: [, s1), [s1, s2), [s2, ). + // + // Example: + // * Row keys := ["a", "apple", "custom", "customer_1", "customer_2", + // "other", "zz"] + // * initial_split_keys := ["apple", "customer_1", "customer_2", "other"] + // * Key assignment: + // - Tablet 1 [, apple) => {"a"}. + // - Tablet 2 [apple, customer_1) => {"apple", "custom"}. + // - Tablet 3 [customer_1, customer_2) => {"customer_1"}. + // - Tablet 4 [customer_2, other) => {"customer_2"}. + // - Tablet 5 [other, ) => {"other", "zz"}. + repeated string initial_split_keys = 4; +} + +message ListTablesRequest { + // The unique name of the cluster for which tables should be listed. + string name = 1; +} + +message ListTablesResponse { + // The tables present in the requested cluster. + // At present, only the names of the tables are populated. + repeated Table tables = 1; +} + +message GetTableRequest { + // The unique name of the requested table. + string name = 1; +} + +message DeleteTableRequest { + // The unique name of the table to be deleted. + string name = 1; +} + +message RenameTableRequest { + // The current unique name of the table. + string name = 1; + + // The new name by which the table should be referred to within its containing + // cluster, e.g. "foobar" rather than "/tables/foobar". + string new_id = 2; +} + +message CreateColumnFamilyRequest { + // The unique name of the table in which to create the new column family. + string name = 1; + + // The name by which the new column family should be referred to within the + // table, e.g. "foobar" rather than "/columnFamilies/foobar". + string column_family_id = 2; + + // The column family to create. The `name` field must be left blank. + ColumnFamily column_family = 3; +} + +message DeleteColumnFamilyRequest { + // The unique name of the column family to be deleted. + string name = 1; +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/cloud.go b/Godeps/_workspace/src/google.golang.org/cloud/cloud.go new file mode 100644 index 0000000000..96d36baf2c --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/cloud.go @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cloud contains Google Cloud Platform APIs related types +// and common functions. +package cloud // import "google.golang.org/cloud" + +import ( + "net/http" + + "golang.org/x/net/context" + "google.golang.org/cloud/internal" +) + +// NewContext returns a new context that uses the provided http.Client. +// Provided http.Client is responsible to authorize and authenticate +// the requests made to the Google Cloud APIs. +// It mutates the client's original Transport to append the cloud +// package's user-agent to the outgoing requests. +// You can obtain the project ID from the Google Developers Console, +// https://console.developers.google.com. +func NewContext(projID string, c *http.Client) context.Context { + if c == nil { + panic("invalid nil *http.Client passed to NewContext") + } + return WithContext(context.Background(), projID, c) +} + +// WithContext returns a new context in a similar way NewContext does, +// but initiates the new context with the specified parent. +func WithContext(parent context.Context, projID string, c *http.Client) context.Context { + // TODO(bradfitz): delete internal.Transport. It's too wrappy for what it does. + // Do User-Agent some other way. + if _, ok := c.Transport.(*internal.Transport); !ok { + c.Transport = &internal.Transport{Base: c.Transport} + } + return internal.WithContext(parent, projID, c) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/cloud_test.go b/Godeps/_workspace/src/google.golang.org/cloud/cloud_test.go new file mode 100644 index 0000000000..2561fb030e --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/cloud_test.go @@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud + +import ( + "net/http" + "testing" + + "google.golang.org/cloud/internal" +) + +func TestClientTransportMutate(t *testing.T) { + c := &http.Client{Transport: http.DefaultTransport} + NewContext("project-id", c) + NewContext("project-id", c) + + tr, ok := c.Transport.(*internal.Transport) + if !ok { + t.Errorf("Transport is expected to be an internal.Transport, found to be a %T", c.Transport) + } + if _, ok := tr.Base.(*internal.Transport); ok { + t.Errorf("Transport's Base shouldn't have been an internal.Transport, found to be a %T", tr.Base) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/compute/metadata/metadata.go b/Godeps/_workspace/src/google.golang.org/cloud/compute/metadata/metadata.go new file mode 100644 index 0000000000..972972dd76 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/compute/metadata/metadata.go @@ -0,0 +1,327 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package metadata provides access to Google Compute Engine (GCE) +// metadata and API service accounts. +// +// This package is a wrapper around the GCE metadata service, +// as documented at https://developers.google.com/compute/docs/metadata. +package metadata // import "google.golang.org/cloud/compute/metadata" + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + "google.golang.org/cloud/internal" +) + +type cachedValue struct { + k string + trim bool + mu sync.Mutex + v string +} + +var ( + projID = &cachedValue{k: "project/project-id", trim: true} + projNum = &cachedValue{k: "project/numeric-project-id", trim: true} + instID = &cachedValue{k: "instance/id", trim: true} +) + +var metaClient = &http.Client{ + Transport: &internal.Transport{ + Base: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 750 * time.Millisecond, + KeepAlive: 30 * time.Second, + }).Dial, + ResponseHeaderTimeout: 750 * time.Millisecond, + }, + }, +} + +// NotDefinedError is returned when requested metadata is not defined. +// +// The underlying string is the suffix after "/computeMetadata/v1/". +// +// This error is not returned if the value is defined to be the empty +// string. +type NotDefinedError string + +func (suffix NotDefinedError) Error() string { + return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix)) +} + +// Get returns a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// +// If the GCE_METADATA_HOST environment variable is not defined, a default of +// 169.254.169.254 will be used instead. +// +// If the requested metadata is not defined, the returned error will +// be of type NotDefinedError. +func Get(suffix string) (string, error) { + val, _, err := getETag(suffix) + return val, err +} + +// getETag returns a value from the metadata service as well as the associated +// ETag. This func is otherwise equivalent to Get. +func getETag(suffix string) (value, etag string, err error) { + // Using a fixed IP makes it very difficult to spoof the metadata service in + // a container, which is an important use-case for local testing of cloud + // deployments. To enable spoofing of the metadata service, the environment + // variable GCE_METADATA_HOST is first inspected to decide where metadata + // requests shall go. + host := os.Getenv("GCE_METADATA_HOST") + if host == "" { + // Using 169.254.169.254 instead of "metadata" here because Go + // binaries built with the "netgo" tag and without cgo won't + // know the search suffix for "metadata" is + // ".google.internal", and this IP address is documented as + // being stable anyway. + host = "169.254.169.254" + } + url := "http://" + host + "/computeMetadata/v1/" + suffix + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Metadata-Flavor", "Google") + res, err := metaClient.Do(req) + if err != nil { + return "", "", err + } + defer res.Body.Close() + if res.StatusCode == http.StatusNotFound { + return "", "", NotDefinedError(suffix) + } + if res.StatusCode != 200 { + return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url) + } + all, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", err + } + return string(all), res.Header.Get("Etag"), nil +} + +func getTrimmed(suffix string) (s string, err error) { + s, err = Get(suffix) + s = strings.TrimSpace(s) + return +} + +func (c *cachedValue) get() (v string, err error) { + defer c.mu.Unlock() + c.mu.Lock() + if c.v != "" { + return c.v, nil + } + if c.trim { + v, err = getTrimmed(c.k) + } else { + v, err = Get(c.k) + } + if err == nil { + c.v = v + } + return +} + +var onGCE struct { + sync.Mutex + set bool + v bool +} + +// OnGCE reports whether this process is running on Google Compute Engine. +func OnGCE() bool { + defer onGCE.Unlock() + onGCE.Lock() + if onGCE.set { + return onGCE.v + } + onGCE.set = true + + // We use the DNS name of the metadata service here instead of the IP address + // because we expect that to fail faster in the not-on-GCE case. + res, err := metaClient.Get("http://metadata.google.internal") + if err != nil { + return false + } + onGCE.v = res.Header.Get("Metadata-Flavor") == "Google" + return onGCE.v +} + +// Subscribe subscribes to a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// +// Subscribe calls fn with the latest metadata value indicated by the provided +// suffix. If the metadata value is deleted, fn is called with the empty string +// and ok false. Subscribe blocks until fn returns a non-nil error or the value +// is deleted. Subscribe returns the error value returned from the last call to +// fn, which may be nil when ok == false. +func Subscribe(suffix string, fn func(v string, ok bool) error) error { + const failedSubscribeSleep = time.Second * 5 + + // First check to see if the metadata value exists at all. + val, lastETag, err := getETag(suffix) + if err != nil { + return err + } + + if err := fn(val, true); err != nil { + return err + } + + ok := true + suffix += "?wait_for_change=true&last_etag=" + for { + val, etag, err := getETag(suffix + url.QueryEscape(lastETag)) + if err != nil { + if _, deleted := err.(NotDefinedError); !deleted { + time.Sleep(failedSubscribeSleep) + continue // Retry on other errors. + } + ok = false + } + lastETag = etag + + if err := fn(val, ok); err != nil || !ok { + return err + } + } +} + +// ProjectID returns the current instance's project ID string. +func ProjectID() (string, error) { return projID.get() } + +// NumericProjectID returns the current instance's numeric project ID. +func NumericProjectID() (string, error) { return projNum.get() } + +// InternalIP returns the instance's primary internal IP address. +func InternalIP() (string, error) { + return getTrimmed("instance/network-interfaces/0/ip") +} + +// ExternalIP returns the instance's primary external (public) IP address. +func ExternalIP() (string, error) { + return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip") +} + +// Hostname returns the instance's hostname. This will be of the form +// ".c..internal". +func Hostname() (string, error) { + return getTrimmed("instance/hostname") +} + +// InstanceTags returns the list of user-defined instance tags, +// assigned when initially creating a GCE instance. +func InstanceTags() ([]string, error) { + var s []string + j, err := Get("instance/tags") + if err != nil { + return nil, err + } + if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil { + return nil, err + } + return s, nil +} + +// InstanceID returns the current VM's numeric instance ID. +func InstanceID() (string, error) { + return instID.get() +} + +// InstanceName returns the current VM's instance ID string. +func InstanceName() (string, error) { + host, err := Hostname() + if err != nil { + return "", err + } + return strings.Split(host, ".")[0], nil +} + +// Zone returns the current VM's zone, such as "us-central1-b". +func Zone() (string, error) { + zone, err := getTrimmed("instance/zone") + // zone is of the form "projects//zones/". + if err != nil { + return "", err + } + return zone[strings.LastIndex(zone, "/")+1:], nil +} + +// InstanceAttributes returns the list of user-defined attributes, +// assigned when initially creating a GCE VM instance. The value of an +// attribute can be obtained with InstanceAttributeValue. +func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") } + +// ProjectAttributes returns the list of user-defined attributes +// applying to the project as a whole, not just this VM. The value of +// an attribute can be obtained with ProjectAttributeValue. +func ProjectAttributes() ([]string, error) { return lines("project/attributes/") } + +func lines(suffix string) ([]string, error) { + j, err := Get(suffix) + if err != nil { + return nil, err + } + s := strings.Split(strings.TrimSpace(j), "\n") + for i := range s { + s[i] = strings.TrimSpace(s[i]) + } + return s, nil +} + +// InstanceAttributeValue returns the value of the provided VM +// instance attribute. +// +// If the requested attribute is not defined, the returned error will +// be of type NotDefinedError. +// +// InstanceAttributeValue may return ("", nil) if the attribute was +// defined to be the empty string. +func InstanceAttributeValue(attr string) (string, error) { + return Get("instance/attributes/" + attr) +} + +// ProjectAttributeValue returns the value of the provided +// project attribute. +// +// If the requested attribute is not defined, the returned error will +// be of type NotDefinedError. +// +// ProjectAttributeValue may return ("", nil) if the attribute was +// defined to be the empty string. +func ProjectAttributeValue(attr string) (string, error) { + return Get("project/attributes/" + attr) +} + +// Scopes returns the service account scopes for the given account. +// The account may be empty or the string "default" to use the instance's +// main account. +func Scopes(serviceAccount string) ([]string, error) { + if serviceAccount == "" { + serviceAccount = "default" + } + return lines("instance/service-accounts/" + serviceAccount + "/scopes") +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/container/container.go b/Godeps/_workspace/src/google.golang.org/cloud/container/container.go new file mode 100644 index 0000000000..b57bd95158 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/container/container.go @@ -0,0 +1,251 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package container contains a Google Container Engine client. +// +// For more information about the API, +// see https://cloud.google.com/container-engine/docs +package container // import "google.golang.org/cloud/container" + +import ( + "errors" + "net/http" + "time" + + "golang.org/x/net/context" + raw "google.golang.org/api/container/v1" + "google.golang.org/cloud/internal" +) + +type Type string + +var ( + TypeCreate Type = Type("createCluster") + TypeDelete Type = Type("deleteCluster") +) + +type Status string + +var ( + Done = Status("done") + Pending = Status("pending") + Running = Status("running") + Error = Status("error") + Provisioning = Status("provisioning") + Stopping = Status("stopping") +) + +// Resource is a Google Container Engine cluster resource. +type Resource struct { + // Name is the name of this cluster. The name must be unique + // within this project and zone, and can be up to 40 characters. + Name string + + // Description is the description of the cluster. Optional. + Description string + + // Zone is the Google Compute Engine zone in which the cluster resides. + Zone string + + // Status is the current status of the cluster. It could either be + // StatusError, StatusProvisioning, StatusRunning or StatusStopping. + Status Status + + // Num is the number of the nodes in this cluster resource. + Num int64 + + // APIVersion is the version of the Kubernetes master and kubelets running + // in this cluster. Allowed value is 0.4.2, or leave blank to + // pick up the latest stable release. + APIVersion string + + // Endpoint is the IP address of this cluster's Kubernetes master. + // The endpoint can be accessed at https://username:password@endpoint/. + // See Username and Password fields for the username and password information. + Endpoint string + + // Username is the username to use when accessing the Kubernetes master endpoint. + Username string + + // Password is the password to use when accessing the Kubernetes master endpoint. + Password string + + // ContainerIPv4CIDR is the IP addresses of the container pods in + // this cluster, in CIDR notation (e.g. 1.2.3.4/29). + ContainerIPv4CIDR string + + // ServicesIPv4CIDR is the IP addresses of the Kubernetes services in this + // cluster, in CIDR notation (e.g. 1.2.3.4/29). Service addresses are + // always in the 10.0.0.0/16 range. + ServicesIPv4CIDR string + + // MachineType is a Google Compute Engine machine type (e.g. n1-standard-1). + // If none set, the default type is used while creating a new cluster. + MachineType string + + // This field is ignored. It was removed from the underlying container API in v1. + SourceImage string + + // Created is the creation time of this cluster. + Created time.Time +} + +func resourceFromRaw(c *raw.Cluster) *Resource { + if c == nil { + return nil + } + r := &Resource{ + Name: c.Name, + Description: c.Description, + Zone: c.Zone, + Status: Status(c.Status), + Num: c.InitialNodeCount, + APIVersion: c.InitialClusterVersion, + Endpoint: c.Endpoint, + Username: c.MasterAuth.Username, + Password: c.MasterAuth.Password, + ContainerIPv4CIDR: c.ClusterIpv4Cidr, + ServicesIPv4CIDR: c.ServicesIpv4Cidr, + MachineType: c.NodeConfig.MachineType, + } + r.Created, _ = time.Parse(time.RFC3339, c.CreateTime) + return r +} + +func resourcesFromRaw(c []*raw.Cluster) []*Resource { + r := make([]*Resource, len(c)) + for i, val := range c { + r[i] = resourceFromRaw(val) + } + return r +} + +// Op represents a Google Container Engine API operation. +type Op struct { + // Name is the name of the operation. + Name string + + // Zone is the Google Compute Engine zone. + Zone string + + // This field is ignored. It was removed from the underlying container API in v1. + TargetURL string + + // Type is the operation type. It could be either be TypeCreate or TypeDelete. + Type Type + + // Status is the current status of this operation. It could be either + // OpDone or OpPending. + Status Status +} + +func opFromRaw(o *raw.Operation) *Op { + if o == nil { + return nil + } + return &Op{ + Name: o.Name, + Zone: o.Zone, + Type: Type(o.OperationType), + Status: Status(o.Status), + } +} + +func opsFromRaw(o []*raw.Operation) []*Op { + ops := make([]*Op, len(o)) + for i, val := range o { + ops[i] = opFromRaw(val) + } + return ops +} + +// Clusters returns a list of cluster resources from the specified zone. +// If no zone is specified, it returns all clusters under the user project. +func Clusters(ctx context.Context, zone string) ([]*Resource, error) { + s := rawService(ctx) + if zone == "" { + resp, err := s.Projects.Zones.Clusters.List(internal.ProjID(ctx), "-").Do() + if err != nil { + return nil, err + } + return resourcesFromRaw(resp.Clusters), nil + } + resp, err := s.Projects.Zones.Clusters.List(internal.ProjID(ctx), zone).Do() + if err != nil { + return nil, err + } + return resourcesFromRaw(resp.Clusters), nil +} + +// Cluster returns metadata about the specified cluster. +func Cluster(ctx context.Context, zone, name string) (*Resource, error) { + s := rawService(ctx) + resp, err := s.Projects.Zones.Clusters.Get(internal.ProjID(ctx), zone, name).Do() + if err != nil { + return nil, err + } + return resourceFromRaw(resp), nil +} + +// CreateCluster creates a new cluster with the provided metadata +// in the specified zone. +func CreateCluster(ctx context.Context, zone string, resource *Resource) (*Resource, error) { + panic("not implemented") +} + +// DeleteCluster deletes a cluster. +func DeleteCluster(ctx context.Context, zone, name string) error { + s := rawService(ctx) + _, err := s.Projects.Zones.Clusters.Delete(internal.ProjID(ctx), zone, name).Do() + return err +} + +// Operations returns a list of operations from the specified zone. +// If no zone is specified, it looks up for all of the operations +// that are running under the user's project. +func Operations(ctx context.Context, zone string) ([]*Op, error) { + s := rawService(ctx) + if zone == "" { + resp, err := s.Projects.Zones.Operations.List(internal.ProjID(ctx), "-").Do() + if err != nil { + return nil, err + } + return opsFromRaw(resp.Operations), nil + } + resp, err := s.Projects.Zones.Operations.List(internal.ProjID(ctx), zone).Do() + if err != nil { + return nil, err + } + return opsFromRaw(resp.Operations), nil +} + +// Operation returns an operation. +func Operation(ctx context.Context, zone, name string) (*Op, error) { + s := rawService(ctx) + resp, err := s.Projects.Zones.Operations.Get(internal.ProjID(ctx), zone, name).Do() + if err != nil { + return nil, err + } + if resp.StatusMessage != "" { + return nil, errors.New(resp.StatusMessage) + } + return opFromRaw(resp), nil +} + +func rawService(ctx context.Context) *raw.Service { + return internal.Service(ctx, "container", func(hc *http.Client) interface{} { + svc, _ := raw.New(hc) + return svc + }).(*raw.Service) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/datastore.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/datastore.go new file mode 100644 index 0000000000..35e704b61d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/datastore.go @@ -0,0 +1,496 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package datastore contains a Google Cloud Datastore client. +// +// This package is experimental and may make backwards-incompatible changes. +package datastore // import "google.golang.org/cloud/datastore" + +import ( + "errors" + "fmt" + "reflect" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + "google.golang.org/cloud" + pb "google.golang.org/cloud/internal/datastore" + "google.golang.org/cloud/internal/transport" +) + +const prodAddr = "https://www.googleapis.com/datastore/v1beta2/datasets/" + +const userAgent = "gcloud-golang-datastore/20150727" + +const ( + // ScopeDatastore grants permissions to view and/or manage datastore entities + ScopeDatastore = "https://www.googleapis.com/auth/datastore" + + // ScopeUserEmail grants permission to view the user's email address. + // It is required to access the datastore. + ScopeUserEmail = "https://www.googleapis.com/auth/userinfo.email" +) + +// protoClient is an interface for *transport.ProtoClient to support injecting +// fake clients in tests. +type protoClient interface { + Call(context.Context, string, proto.Message, proto.Message) error +} + +// Client is a client for reading and writing data in a datastore dataset. +type Client struct { + client protoClient + endpoint string + dataset string // Called dataset by the datastore API, synonym for project ID. +} + +// NewClient creates a new Client for a given dataset. +func NewClient(ctx context.Context, projectID string, opts ...cloud.ClientOption) (*Client, error) { + o := []cloud.ClientOption{ + cloud.WithEndpoint(prodAddr), + cloud.WithScopes(ScopeDatastore, ScopeUserEmail), + cloud.WithUserAgent(userAgent), + } + o = append(o, opts...) + client, err := transport.NewProtoClient(ctx, o...) + if err != nil { + return nil, fmt.Errorf("dialing: %v", err) + } + return &Client{ + client: client, + dataset: projectID, + }, nil + +} + +var ( + // ErrInvalidEntityType is returned when functions like Get or Next are + // passed a dst or src argument of invalid type. + ErrInvalidEntityType = errors.New("datastore: invalid entity type") + // ErrInvalidKey is returned when an invalid key is presented. + ErrInvalidKey = errors.New("datastore: invalid key") + // ErrNoSuchEntity is returned when no entity was found for a given key. + ErrNoSuchEntity = errors.New("datastore: no such entity") +) + +type multiArgType int + +const ( + multiArgTypeInvalid multiArgType = iota + multiArgTypePropertyLoadSaver + multiArgTypeStruct + multiArgTypeStructPtr + multiArgTypeInterface +) + +// nsKey is the type of the context.Context key to store the datastore +// namespace. +type nsKey struct{} + +// WithNamespace returns a new context that limits the scope its parent +// context with a Datastore namespace. +func WithNamespace(parent context.Context, namespace string) context.Context { + return context.WithValue(parent, nsKey{}, namespace) +} + +// ctxNamespace returns the active namespace for a context. +// It defaults to "" if no namespace was specified. +func ctxNamespace(ctx context.Context) string { + v, _ := ctx.Value(nsKey{}).(string) + return v +} + +// ErrFieldMismatch is returned when a field is to be loaded into a different +// type than the one it was stored from, or when a field is missing or +// unexported in the destination struct. +// StructType is the type of the struct pointed to by the destination argument +// passed to Get or to Iterator.Next. +type ErrFieldMismatch struct { + StructType reflect.Type + FieldName string + Reason string +} + +func (e *ErrFieldMismatch) Error() string { + return fmt.Sprintf("datastore: cannot load field %q into a %q: %s", + e.FieldName, e.StructType, e.Reason) +} + +func (c *Client) call(ctx context.Context, method string, req, resp proto.Message) error { + return c.client.Call(ctx, c.dataset+"/"+method, req, resp) +} + +func keyToProto(k *Key) *pb.Key { + if k == nil { + return nil + } + + // TODO(jbd): Eliminate unrequired allocations. + path := []*pb.Key_PathElement(nil) + for { + el := &pb.Key_PathElement{ + Kind: proto.String(k.kind), + } + if k.id != 0 { + el.Id = proto.Int64(k.id) + } + if k.name != "" { + el.Name = proto.String(k.name) + } + path = append([]*pb.Key_PathElement{el}, path...) + if k.parent == nil { + break + } + k = k.parent + } + key := &pb.Key{ + PathElement: path, + } + if k.namespace != "" { + key.PartitionId = &pb.PartitionId{ + Namespace: proto.String(k.namespace), + } + } + return key +} + +func protoToKey(p *pb.Key) *Key { + keys := make([]*Key, len(p.GetPathElement())) + for i, el := range p.GetPathElement() { + keys[i] = &Key{ + namespace: p.GetPartitionId().GetNamespace(), + kind: el.GetKind(), + id: el.GetId(), + name: el.GetName(), + } + } + for i := 0; i < len(keys)-1; i++ { + keys[i+1].parent = keys[i] + } + return keys[len(keys)-1] +} + +// multiKeyToProto is a batch version of keyToProto. +func multiKeyToProto(keys []*Key) []*pb.Key { + ret := make([]*pb.Key, len(keys)) + for i, k := range keys { + ret[i] = keyToProto(k) + } + return ret +} + +// multiKeyToProto is a batch version of keyToProto. +func multiProtoToKey(keys []*pb.Key) []*Key { + ret := make([]*Key, len(keys)) + for i, k := range keys { + ret[i] = protoToKey(k) + } + return ret +} + +// multiValid is a batch version of Key.valid. It returns an error, not a +// []bool. +func multiValid(key []*Key) error { + invalid := false + for _, k := range key { + if !k.valid() { + invalid = true + break + } + } + if !invalid { + return nil + } + err := make(MultiError, len(key)) + for i, k := range key { + if !k.valid() { + err[i] = ErrInvalidKey + } + } + return err +} + +// checkMultiArg checks that v has type []S, []*S, []I, or []P, for some struct +// type S, for some interface type I, or some non-interface non-pointer type P +// such that P or *P implements PropertyLoadSaver. +// +// It returns what category the slice's elements are, and the reflect.Type +// that represents S, I or P. +// +// As a special case, PropertyList is an invalid type for v. +func checkMultiArg(v reflect.Value) (m multiArgType, elemType reflect.Type) { + if v.Kind() != reflect.Slice { + return multiArgTypeInvalid, nil + } + if v.Type() == typeOfPropertyList { + return multiArgTypeInvalid, nil + } + elemType = v.Type().Elem() + if reflect.PtrTo(elemType).Implements(typeOfPropertyLoadSaver) { + return multiArgTypePropertyLoadSaver, elemType + } + switch elemType.Kind() { + case reflect.Struct: + return multiArgTypeStruct, elemType + case reflect.Interface: + return multiArgTypeInterface, elemType + case reflect.Ptr: + elemType = elemType.Elem() + if elemType.Kind() == reflect.Struct { + return multiArgTypeStructPtr, elemType + } + } + return multiArgTypeInvalid, nil +} + +// Get loads the entity stored for key into dst, which must be a struct pointer +// or implement PropertyLoadSaver. If there is no such entity for the key, Get +// returns ErrNoSuchEntity. +// +// The values of dst's unmatched struct fields are not modified, and matching +// slice-typed fields are not reset before appending to them. In particular, it +// is recommended to pass a pointer to a zero valued struct on each Get call. +// +// ErrFieldMismatch is returned when a field is to be loaded into a different +// type than the one it was stored from, or when a field is missing or +// unexported in the destination struct. ErrFieldMismatch is only returned if +// dst is a struct pointer. +func (c *Client) Get(ctx context.Context, key *Key, dst interface{}) error { + err := c.get(ctx, []*Key{key}, []interface{}{dst}, nil) + if me, ok := err.(MultiError); ok { + return me[0] + } + return err +} + +// GetMulti is a batch version of Get. +// +// dst must be a []S, []*S, []I or []P, for some struct type S, some interface +// type I, or some non-interface non-pointer type P such that P or *P +// implements PropertyLoadSaver. If an []I, each element must be a valid dst +// for Get: it must be a struct pointer or implement PropertyLoadSaver. +// +// As a special case, PropertyList is an invalid type for dst, even though a +// PropertyList is a slice of structs. It is treated as invalid to avoid being +// mistakenly passed when []PropertyList was intended. +func (c *Client) GetMulti(ctx context.Context, keys []*Key, dst interface{}) error { + return c.get(ctx, keys, dst, nil) +} + +func (c *Client) get(ctx context.Context, keys []*Key, dst interface{}, opts *pb.ReadOptions) error { + v := reflect.ValueOf(dst) + multiArgType, _ := checkMultiArg(v) + + // Sanity checks + if multiArgType == multiArgTypeInvalid { + return errors.New("datastore: dst has invalid type") + } + if len(keys) != v.Len() { + return errors.New("datastore: keys and dst slices have different length") + } + if len(keys) == 0 { + return nil + } + + // Go through keys, validate them, serialize then, and create a dict mapping them to their index + multiErr, any := make(MultiError, len(keys)), false + keyMap := make(map[string]int) + pbKeys := make([]*pb.Key, len(keys)) + for i, k := range keys { + if !k.valid() { + multiErr[i] = ErrInvalidKey + any = true + } else { + keyMap[k.String()] = i + pbKeys[i] = keyToProto(k) + } + } + if any { + return multiErr + } + req := &pb.LookupRequest{ + Key: pbKeys, + ReadOptions: opts, + } + resp := &pb.LookupResponse{} + if err := c.call(ctx, "lookup", req, resp); err != nil { + return err + } + if len(resp.Deferred) > 0 { + // TODO(jbd): Assess whether we should retry the deferred keys. + return errors.New("datastore: some entities temporarily unavailable") + } + if len(keys) != len(resp.Found)+len(resp.Missing) { + return errors.New("datastore: internal error: server returned the wrong number of entities") + } + for _, e := range resp.Found { + k := protoToKey(e.Entity.Key) + index := keyMap[k.String()] + elem := v.Index(index) + if multiArgType == multiArgTypePropertyLoadSaver || multiArgType == multiArgTypeStruct { + elem = elem.Addr() + } + err := loadEntity(elem.Interface(), e.Entity) + if err != nil { + multiErr[index] = err + any = true + } + } + for _, e := range resp.Missing { + k := protoToKey(e.Entity.Key) + multiErr[keyMap[k.String()]] = ErrNoSuchEntity + any = true + } + if any { + return multiErr + } + return nil +} + +// Put saves the entity src into the datastore with key k. src must be a struct +// pointer or implement PropertyLoadSaver; if a struct pointer then any +// unexported fields of that struct will be skipped. If k is an incomplete key, +// the returned key will be a unique key generated by the datastore. +func (c *Client) Put(ctx context.Context, key *Key, src interface{}) (*Key, error) { + k, err := c.PutMulti(ctx, []*Key{key}, []interface{}{src}) + if err != nil { + if me, ok := err.(MultiError); ok { + return nil, me[0] + } + return nil, err + } + return k[0], nil +} + +// PutMulti is a batch version of Put. +// +// src must satisfy the same conditions as the dst argument to GetMulti. +func (c *Client) PutMulti(ctx context.Context, keys []*Key, src interface{}) ([]*Key, error) { + mutation, err := putMutation(keys, src) + if err != nil { + return nil, err + } + + // Make the request. + req := &pb.CommitRequest{ + Mutation: mutation, + Mode: pb.CommitRequest_NON_TRANSACTIONAL.Enum(), + } + resp := &pb.CommitResponse{} + if err := c.call(ctx, "commit", req, resp); err != nil { + return nil, err + } + + // Copy any newly minted keys into the returned keys. + newKeys := make(map[int]int) // Map of index in returned slice to index in response. + ret := make([]*Key, len(keys)) + var idx int + for i, key := range keys { + if key.Incomplete() { + // This key will be in the mutation result. + newKeys[i] = idx + idx++ + } else { + ret[i] = key + } + } + if len(newKeys) != len(resp.MutationResult.InsertAutoIdKey) { + return nil, errors.New("datastore: internal error: server returned the wrong number of keys") + } + for retI, respI := range newKeys { + ret[retI] = protoToKey(resp.MutationResult.InsertAutoIdKey[respI]) + } + return ret, nil +} + +func putMutation(keys []*Key, src interface{}) (*pb.Mutation, error) { + v := reflect.ValueOf(src) + multiArgType, _ := checkMultiArg(v) + if multiArgType == multiArgTypeInvalid { + return nil, errors.New("datastore: src has invalid type") + } + if len(keys) != v.Len() { + return nil, errors.New("datastore: key and src slices have different length") + } + if len(keys) == 0 { + return nil, nil + } + if err := multiValid(keys); err != nil { + return nil, err + } + var upsert, insert []*pb.Entity + for i, k := range keys { + val := reflect.ValueOf(src).Index(i) + // If src is an interface slice []interface{}{ent1, ent2} + if val.Kind() == reflect.Interface && val.Elem().Kind() == reflect.Slice { + val = val.Elem() + } + // If src is a slice of ptrs []*T{ent1, ent2} + if val.Kind() == reflect.Ptr && val.Elem().Kind() == reflect.Slice { + val = val.Elem() + } + p, err := saveEntity(k, val.Interface()) + if err != nil { + return nil, fmt.Errorf("datastore: Error while saving %v: %v", k.String(), err) + } + if k.Incomplete() { + insert = append(insert, p) + } else { + upsert = append(upsert, p) + } + } + + return &pb.Mutation{ + InsertAutoId: insert, + Upsert: upsert, + }, nil +} + +// Delete deletes the entity for the given key. +func (c *Client) Delete(ctx context.Context, key *Key) error { + err := c.DeleteMulti(ctx, []*Key{key}) + if me, ok := err.(MultiError); ok { + return me[0] + } + return err +} + +// DeleteMulti is a batch version of Delete. +func (c *Client) DeleteMulti(ctx context.Context, keys []*Key) error { + mutation, err := deleteMutation(keys) + if err != nil { + return err + } + + req := &pb.CommitRequest{ + Mutation: mutation, + Mode: pb.CommitRequest_NON_TRANSACTIONAL.Enum(), + } + resp := &pb.CommitResponse{} + return c.call(ctx, "commit", req, resp) +} + +func deleteMutation(keys []*Key) (*pb.Mutation, error) { + protoKeys := make([]*pb.Key, len(keys)) + for i, k := range keys { + if k.Incomplete() { + return nil, fmt.Errorf("datastore: can't delete the incomplete key: %v", k) + } + protoKeys[i] = keyToProto(k) + } + + return &pb.Mutation{ + Delete: protoKeys, + }, nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/datastore_test.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/datastore_test.go new file mode 100644 index 0000000000..86896bf0fd --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/datastore_test.go @@ -0,0 +1,1359 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" +) + +type ( + myBlob []byte + myByte byte + myString string +) + +func makeMyByteSlice(n int) []myByte { + b := make([]myByte, n) + for i := range b { + b[i] = myByte(i) + } + return b +} + +func makeInt8Slice(n int) []int8 { + b := make([]int8, n) + for i := range b { + b[i] = int8(i) + } + return b +} + +func makeUint8Slice(n int) []uint8 { + b := make([]uint8, n) + for i := range b { + b[i] = uint8(i) + } + return b +} + +func newKey(stringID string, parent *Key) *Key { + return &Key{ + kind: "kind", + name: stringID, + id: 0, + parent: parent, + } +} + +var ( + testKey0 = newKey("name0", nil) + testKey1a = newKey("name1", nil) + testKey1b = newKey("name1", nil) + testKey2a = newKey("name2", testKey0) + testKey2b = newKey("name2", testKey0) +) + +type B0 struct { + B []byte `datastore:",noindex"` +} + +type B1 struct { + B []int8 +} + +type B2 struct { + B myBlob `datastore:",noindex"` +} + +type B3 struct { + B []myByte `datastore:",noindex"` +} + +type B4 struct { + B [][]byte +} + +type C0 struct { + I int + C chan int +} + +type C1 struct { + I int + C *chan int +} + +type C2 struct { + I int + C []chan int +} + +type C3 struct { + C string +} + +type E struct{} + +type K0 struct { + K *Key +} + +type K1 struct { + K []*Key +} + +type N0 struct { + X0 + Nonymous X0 + Ignore string `datastore:"-"` + Other string +} + +type N1 struct { + X0 + Nonymous []X0 + Ignore string `datastore:"-"` + Other string +} + +type N2 struct { + N1 `datastore:"red"` + Green N1 `datastore:"green"` + Blue N1 + White N1 `datastore:"-"` +} + +type O0 struct { + I int64 +} + +type O1 struct { + I int32 +} + +type U0 struct { + U uint +} + +type U1 struct { + U string +} + +type T struct { + T time.Time +} + +type X0 struct { + S string + I int + i int +} + +type X1 struct { + S myString + I int32 + J int64 +} + +type X2 struct { + Z string + i int +} + +type X3 struct { + S bool + I int +} + +type Y0 struct { + B bool + F []float64 + G []float64 +} + +type Y1 struct { + B bool + F float64 +} + +type Y2 struct { + B bool + F []int64 +} + +type Tagged struct { + A int `datastore:"a,noindex"` + B []int `datastore:"b"` + C int `datastore:",noindex"` + D int `datastore:""` + E int + I int `datastore:"-"` + J int `datastore:",noindex" json:"j"` + + Y0 `datastore:"-"` + Z chan int `datastore:"-,"` +} + +type InvalidTagged1 struct { + I int `datastore:"\t"` +} + +type InvalidTagged2 struct { + I int + J int `datastore:"I"` +} + +type Inner1 struct { + W int32 + X string +} + +type Inner2 struct { + Y float64 +} + +type Inner3 struct { + Z bool +} + +type Outer struct { + A int16 + I []Inner1 + J Inner2 + Inner3 +} + +type OuterEquivalent struct { + A int16 + IDotW []int32 `datastore:"I.W"` + IDotX []string `datastore:"I.X"` + JDotY float64 `datastore:"J.Y"` + Z bool +} + +type Dotted struct { + A DottedA `datastore:"A0.A1.A2"` +} + +type DottedA struct { + B DottedB `datastore:"B3"` +} + +type DottedB struct { + C int `datastore:"C4.C5"` +} + +type SliceOfSlices struct { + I int + S []struct { + J int + F []float64 + } +} + +type Recursive struct { + I int + R []Recursive +} + +type MutuallyRecursive0 struct { + I int + R []MutuallyRecursive1 +} + +type MutuallyRecursive1 struct { + I int + R []MutuallyRecursive0 +} + +type Doubler struct { + S string + I int64 + B bool +} + +func (d *Doubler) Load(props []Property) error { + return LoadStruct(d, props) +} + +func (d *Doubler) Save() ([]Property, error) { + // Save the default Property slice to an in-memory buffer (a PropertyList). + props, err := SaveStruct(d) + if err != nil { + return nil, err + } + var list PropertyList + if err := list.Load(props); err != nil { + return nil, err + } + + // Edit that PropertyList, and send it on. + for i := range list { + switch v := list[i].Value.(type) { + case string: + // + means string concatenation. + list[i].Value = v + v + case int64: + // + means integer addition. + list[i].Value = v + v + } + } + return list.Save() +} + +var _ PropertyLoadSaver = (*Doubler)(nil) + +type Deriver struct { + S, Derived, Ignored string +} + +func (e *Deriver) Load(props []Property) error { + for _, p := range props { + if p.Name != "S" { + continue + } + e.S = p.Value.(string) + e.Derived = "derived+" + e.S + } + return nil +} + +func (e *Deriver) Save() ([]Property, error) { + return []Property{ + { + Name: "S", + Value: e.S, + }, + }, nil +} + +var _ PropertyLoadSaver = (*Deriver)(nil) + +type BadMultiPropEntity struct{} + +func (e *BadMultiPropEntity) Load(props []Property) error { + return errors.New("unimplemented") +} + +func (e *BadMultiPropEntity) Save() ([]Property, error) { + // Write multiple properties with the same name "I", but Multiple is false. + var props []Property + for i := 0; i < 3; i++ { + props = append(props, Property{ + Name: "I", + Value: int64(i), + }) + } + return props, nil +} + +var _ PropertyLoadSaver = (*BadMultiPropEntity)(nil) + +type testCase struct { + desc string + src interface{} + want interface{} + putErr string + getErr string +} + +var testCases = []testCase{ + { + "chan save fails", + &C0{I: -1}, + &E{}, + "unsupported struct field", + "", + }, + { + "*chan save fails", + &C1{I: -1}, + &E{}, + "unsupported struct field", + "", + }, + { + "[]chan save fails", + &C2{I: -1, C: make([]chan int, 8)}, + &E{}, + "unsupported struct field", + "", + }, + { + "chan load fails", + &C3{C: "not a chan"}, + &C0{}, + "", + "type mismatch", + }, + { + "*chan load fails", + &C3{C: "not a *chan"}, + &C1{}, + "", + "type mismatch", + }, + { + "[]chan load fails", + &C3{C: "not a []chan"}, + &C2{}, + "", + "type mismatch", + }, + { + "empty struct", + &E{}, + &E{}, + "", + "", + }, + { + "key", + &K0{K: testKey1a}, + &K0{K: testKey1b}, + "", + "", + }, + { + "key with parent", + &K0{K: testKey2a}, + &K0{K: testKey2b}, + "", + "", + }, + { + "nil key", + &K0{}, + &K0{}, + "", + "", + }, + { + "all nil keys in slice", + &K1{[]*Key{nil, nil}}, + &K1{[]*Key{nil, nil}}, + "", + "", + }, + { + "some nil keys in slice", + &K1{[]*Key{testKey1a, nil, testKey2a}}, + &K1{[]*Key{testKey1b, nil, testKey2b}}, + "", + "", + }, + { + "overflow", + &O0{I: 1 << 48}, + &O1{}, + "", + "overflow", + }, + { + "time", + &T{T: time.Unix(1e9, 0)}, + &T{T: time.Unix(1e9, 0)}, + "", + "", + }, + { + "time as props", + &T{T: time.Unix(1e9, 0)}, + &PropertyList{ + Property{Name: "T", Value: time.Unix(1e9, 0), NoIndex: false}, + }, + "", + "", + }, + { + "uint save", + &U0{U: 1}, + &U0{}, + "unsupported struct field", + "", + }, + { + "uint load", + &U1{U: "not a uint"}, + &U0{}, + "", + "type mismatch", + }, + { + "zero", + &X0{}, + &X0{}, + "", + "", + }, + { + "basic", + &X0{S: "one", I: 2, i: 3}, + &X0{S: "one", I: 2}, + "", + "", + }, + { + "save string/int load myString/int32", + &X0{S: "one", I: 2, i: 3}, + &X1{S: "one", I: 2}, + "", + "", + }, + { + "missing fields", + &X0{S: "one", I: 2, i: 3}, + &X2{}, + "", + "no such struct field", + }, + { + "save string load bool", + &X0{S: "one", I: 2, i: 3}, + &X3{I: 2}, + "", + "type mismatch", + }, + { + "basic slice", + &Y0{B: true, F: []float64{7, 8, 9}}, + &Y0{B: true, F: []float64{7, 8, 9}}, + "", + "", + }, + { + "save []float64 load float64", + &Y0{B: true, F: []float64{7, 8, 9}}, + &Y1{B: true}, + "", + "requires a slice", + }, + { + "save []float64 load []int64", + &Y0{B: true, F: []float64{7, 8, 9}}, + &Y2{B: true}, + "", + "type mismatch", + }, + { + "single slice is too long", + &Y0{F: make([]float64, maxIndexedProperties+1)}, + &Y0{}, + "too many indexed properties", + "", + }, + { + "two slices are too long", + &Y0{F: make([]float64, maxIndexedProperties), G: make([]float64, maxIndexedProperties)}, + &Y0{}, + "too many indexed properties", + "", + }, + { + "one slice and one scalar are too long", + &Y0{F: make([]float64, maxIndexedProperties), B: true}, + &Y0{}, + "too many indexed properties", + "", + }, + { + "long blob", + &B0{B: makeUint8Slice(maxIndexedProperties + 1)}, + &B0{B: makeUint8Slice(maxIndexedProperties + 1)}, + "", + "", + }, + { + "long []int8 is too long", + &B1{B: makeInt8Slice(maxIndexedProperties + 1)}, + &B1{}, + "too many indexed properties", + "", + }, + { + "short []int8", + &B1{B: makeInt8Slice(3)}, + &B1{B: makeInt8Slice(3)}, + "", + "", + }, + { + "long myBlob", + &B2{B: makeUint8Slice(maxIndexedProperties + 1)}, + &B2{B: makeUint8Slice(maxIndexedProperties + 1)}, + "", + "", + }, + { + "short myBlob", + &B2{B: makeUint8Slice(3)}, + &B2{B: makeUint8Slice(3)}, + "", + "", + }, + { + "long []myByte", + &B3{B: makeMyByteSlice(maxIndexedProperties + 1)}, + &B3{B: makeMyByteSlice(maxIndexedProperties + 1)}, + "", + "", + }, + { + "short []myByte", + &B3{B: makeMyByteSlice(3)}, + &B3{B: makeMyByteSlice(3)}, + "", + "", + }, + { + "slice of blobs", + &B4{B: [][]byte{ + makeUint8Slice(3), + makeUint8Slice(4), + makeUint8Slice(5), + }}, + &B4{B: [][]byte{ + makeUint8Slice(3), + makeUint8Slice(4), + makeUint8Slice(5), + }}, + "", + "", + }, + { + "[]byte must be noindex", + &PropertyList{ + Property{Name: "B", Value: makeUint8Slice(1501), NoIndex: false}, + }, + nil, + "cannot index a Property", + "", + }, + { + "save tagged load props", + &Tagged{A: 1, B: []int{21, 22, 23}, C: 3, D: 4, E: 5, I: 6, J: 7}, + &PropertyList{ + // A and B are renamed to a and b; A and C are noindex, I is ignored. + // Order is alphabetical + Property{Name: "a", Value: int64(1), NoIndex: true}, + Property{Name: "b", Value: int64(21), NoIndex: false, Multiple: true}, + Property{Name: "b", Value: int64(22), NoIndex: false, Multiple: true}, + Property{Name: "b", Value: int64(23), NoIndex: false, Multiple: true}, + Property{Name: "C", Value: int64(3), NoIndex: true}, + Property{Name: "D", Value: int64(4), NoIndex: false}, + Property{Name: "E", Value: int64(5), NoIndex: false}, + Property{Name: "J", Value: int64(7), NoIndex: true}, + }, + "", + "", + }, + { + "save tagged load tagged", + &Tagged{A: 1, B: []int{21, 22, 23}, C: 3, D: 4, E: 5, I: 6, J: 7}, + &Tagged{A: 1, B: []int{21, 22, 23}, C: 3, D: 4, E: 5, J: 7}, + "", + "", + }, + { + "save props load tagged", + &PropertyList{ + Property{Name: "A", Value: int64(11), NoIndex: true}, + Property{Name: "a", Value: int64(12), NoIndex: true}, + }, + &Tagged{A: 12}, + "", + `cannot load field "A"`, + }, + { + "invalid tagged1", + &InvalidTagged1{I: 1}, + &InvalidTagged1{}, + "struct tag has invalid property name", + "", + }, + { + "invalid tagged2", + &InvalidTagged2{I: 1, J: 2}, + &InvalidTagged2{}, + "struct tag has repeated property name", + "", + }, + { + "doubler", + &Doubler{S: "s", I: 1, B: true}, + &Doubler{S: "ss", I: 2, B: true}, + "", + "", + }, + { + "save struct load props", + &X0{S: "s", I: 1}, + &PropertyList{ + Property{Name: "S", Value: "s", NoIndex: false}, + Property{Name: "I", Value: int64(1), NoIndex: false}, + }, + "", + "", + }, + { + "save props load struct", + &PropertyList{ + Property{Name: "S", Value: "s", NoIndex: false}, + Property{Name: "I", Value: int64(1), NoIndex: false}, + }, + &X0{S: "s", I: 1}, + "", + "", + }, + { + "nil-value props", + &PropertyList{ + Property{Name: "I", Value: nil, NoIndex: false}, + Property{Name: "B", Value: nil, NoIndex: false}, + Property{Name: "S", Value: nil, NoIndex: false}, + Property{Name: "F", Value: nil, NoIndex: false}, + Property{Name: "K", Value: nil, NoIndex: false}, + Property{Name: "T", Value: nil, NoIndex: false}, + Property{Name: "J", Value: nil, NoIndex: false}, + Property{Name: "J", Value: int64(7), NoIndex: false}, + Property{Name: "J", Value: nil, NoIndex: false}, + }, + &struct { + I int64 + B bool + S string + F float64 + K *Key + T time.Time + J []int64 + }{ + J: []int64{0, 7, 0}, + }, + "", + "", + }, + { + "save outer load props", + &Outer{ + A: 1, + I: []Inner1{ + {10, "ten"}, + {20, "twenty"}, + {30, "thirty"}, + }, + J: Inner2{ + Y: 3.14, + }, + Inner3: Inner3{ + Z: true, + }, + }, + &PropertyList{ + Property{Name: "A", Value: int64(1), NoIndex: false}, + Property{Name: "I.W", Value: int64(10), NoIndex: false, Multiple: true}, + Property{Name: "I.W", Value: int64(20), NoIndex: false, Multiple: true}, + Property{Name: "I.W", Value: int64(30), NoIndex: false, Multiple: true}, + Property{Name: "I.X", Value: "ten", NoIndex: false, Multiple: true}, + Property{Name: "I.X", Value: "twenty", NoIndex: false, Multiple: true}, + Property{Name: "I.X", Value: "thirty", NoIndex: false, Multiple: true}, + Property{Name: "J.Y", Value: float64(3.14), NoIndex: false}, + Property{Name: "Z", Value: true, NoIndex: false}, + }, + "", + "", + }, + { + "save props load outer-equivalent", + &PropertyList{ + Property{Name: "A", Value: int64(1), NoIndex: false}, + Property{Name: "I.W", Value: int64(10), NoIndex: false}, + Property{Name: "I.X", Value: "ten", NoIndex: false}, + Property{Name: "I.W", Value: int64(20), NoIndex: false}, + Property{Name: "I.X", Value: "twenty", NoIndex: false}, + Property{Name: "I.W", Value: int64(30), NoIndex: false}, + Property{Name: "I.X", Value: "thirty", NoIndex: false}, + Property{Name: "J.Y", Value: float64(3.14), NoIndex: false}, + Property{Name: "Z", Value: true, NoIndex: false}, + }, + &OuterEquivalent{ + A: 1, + IDotW: []int32{10, 20, 30}, + IDotX: []string{"ten", "twenty", "thirty"}, + JDotY: 3.14, + Z: true, + }, + "", + "", + }, + { + "save outer-equivalent load outer", + &OuterEquivalent{ + A: 1, + IDotW: []int32{10, 20, 30}, + IDotX: []string{"ten", "twenty", "thirty"}, + JDotY: 3.14, + Z: true, + }, + &Outer{ + A: 1, + I: []Inner1{ + {10, "ten"}, + {20, "twenty"}, + {30, "thirty"}, + }, + J: Inner2{ + Y: 3.14, + }, + Inner3: Inner3{ + Z: true, + }, + }, + "", + "", + }, + { + "dotted names save", + &Dotted{A: DottedA{B: DottedB{C: 88}}}, + &PropertyList{ + Property{Name: "A0.A1.A2.B3.C4.C5", Value: int64(88), NoIndex: false}, + }, + "", + "", + }, + { + "dotted names load", + &PropertyList{ + Property{Name: "A0.A1.A2.B3.C4.C5", Value: int64(99), NoIndex: false}, + }, + &Dotted{A: DottedA{B: DottedB{C: 99}}}, + "", + "", + }, + { + "save struct load deriver", + &X0{S: "s", I: 1}, + &Deriver{S: "s", Derived: "derived+s"}, + "", + "", + }, + { + "save deriver load struct", + &Deriver{S: "s", Derived: "derived+s", Ignored: "ignored"}, + &X0{S: "s"}, + "", + "", + }, + { + "zero time.Time", + &T{T: time.Time{}}, + &T{T: time.Time{}}, + "", + "", + }, + { + "time.Time near Unix zero time", + &T{T: time.Unix(0, 4e3)}, + &T{T: time.Unix(0, 4e3)}, + "", + "", + }, + { + "time.Time, far in the future", + &T{T: time.Date(99999, 1, 1, 0, 0, 0, 0, time.UTC)}, + &T{T: time.Date(99999, 1, 1, 0, 0, 0, 0, time.UTC)}, + "", + "", + }, + { + "time.Time, very far in the past", + &T{T: time.Date(-300000, 1, 1, 0, 0, 0, 0, time.UTC)}, + &T{}, + "time value out of range", + "", + }, + { + "time.Time, very far in the future", + &T{T: time.Date(294248, 1, 1, 0, 0, 0, 0, time.UTC)}, + &T{}, + "time value out of range", + "", + }, + { + "structs", + &N0{ + X0: X0{S: "one", I: 2, i: 3}, + Nonymous: X0{S: "four", I: 5, i: 6}, + Ignore: "ignore", + Other: "other", + }, + &N0{ + X0: X0{S: "one", I: 2}, + Nonymous: X0{S: "four", I: 5}, + Other: "other", + }, + "", + "", + }, + { + "slice of structs", + &N1{ + X0: X0{S: "one", I: 2, i: 3}, + Nonymous: []X0{ + {S: "four", I: 5, i: 6}, + {S: "seven", I: 8, i: 9}, + {S: "ten", I: 11, i: 12}, + {S: "thirteen", I: 14, i: 15}, + }, + Ignore: "ignore", + Other: "other", + }, + &N1{ + X0: X0{S: "one", I: 2}, + Nonymous: []X0{ + {S: "four", I: 5}, + {S: "seven", I: 8}, + {S: "ten", I: 11}, + {S: "thirteen", I: 14}, + }, + Other: "other", + }, + "", + "", + }, + { + "structs with slices of structs", + &N2{ + N1: N1{ + X0: X0{S: "rouge"}, + Nonymous: []X0{ + {S: "rosso0"}, + {S: "rosso1"}, + }, + }, + Green: N1{ + X0: X0{S: "vert"}, + Nonymous: []X0{ + {S: "verde0"}, + {S: "verde1"}, + {S: "verde2"}, + }, + }, + Blue: N1{ + X0: X0{S: "bleu"}, + Nonymous: []X0{ + {S: "blu0"}, + {S: "blu1"}, + {S: "blu2"}, + {S: "blu3"}, + }, + }, + }, + &N2{ + N1: N1{ + X0: X0{S: "rouge"}, + Nonymous: []X0{ + {S: "rosso0"}, + {S: "rosso1"}, + }, + }, + Green: N1{ + X0: X0{S: "vert"}, + Nonymous: []X0{ + {S: "verde0"}, + {S: "verde1"}, + {S: "verde2"}, + }, + }, + Blue: N1{ + X0: X0{S: "bleu"}, + Nonymous: []X0{ + {S: "blu0"}, + {S: "blu1"}, + {S: "blu2"}, + {S: "blu3"}, + }, + }, + }, + "", + "", + }, + { + "save structs load props", + &N2{ + N1: N1{ + X0: X0{S: "rouge"}, + Nonymous: []X0{ + {S: "rosso0"}, + {S: "rosso1"}, + }, + }, + Green: N1{ + X0: X0{S: "vert"}, + Nonymous: []X0{ + {S: "verde0"}, + {S: "verde1"}, + {S: "verde2"}, + }, + }, + Blue: N1{ + X0: X0{S: "bleu"}, + Nonymous: []X0{ + {S: "blu0"}, + {S: "blu1"}, + {S: "blu2"}, + {S: "blu3"}, + }, + }, + }, + &PropertyList{ + Property{Name: "red.S", Value: "rouge", NoIndex: false}, + Property{Name: "red.I", Value: int64(0), NoIndex: false}, + Property{Name: "red.Nonymous.S", Value: "rosso0", NoIndex: false, Multiple: true}, + Property{Name: "red.Nonymous.S", Value: "rosso1", NoIndex: false, Multiple: true}, + Property{Name: "red.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "red.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "red.Other", Value: "", NoIndex: false}, + Property{Name: "green.S", Value: "vert", NoIndex: false}, + Property{Name: "green.I", Value: int64(0), NoIndex: false}, + Property{Name: "green.Nonymous.S", Value: "verde0", NoIndex: false, Multiple: true}, + Property{Name: "green.Nonymous.S", Value: "verde1", NoIndex: false, Multiple: true}, + Property{Name: "green.Nonymous.S", Value: "verde2", NoIndex: false, Multiple: true}, + Property{Name: "green.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "green.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "green.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "green.Other", Value: "", NoIndex: false}, + Property{Name: "Blue.S", Value: "bleu", NoIndex: false}, + Property{Name: "Blue.I", Value: int64(0), NoIndex: false}, + Property{Name: "Blue.Nonymous.S", Value: "blu0", NoIndex: false, Multiple: true}, + Property{Name: "Blue.Nonymous.S", Value: "blu1", NoIndex: false, Multiple: true}, + Property{Name: "Blue.Nonymous.S", Value: "blu2", NoIndex: false, Multiple: true}, + Property{Name: "Blue.Nonymous.S", Value: "blu3", NoIndex: false, Multiple: true}, + Property{Name: "Blue.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "Blue.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "Blue.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "Blue.Nonymous.I", Value: int64(0), NoIndex: false, Multiple: true}, + Property{Name: "Blue.Other", Value: "", NoIndex: false}, + }, + "", + "", + }, + { + "save props load structs with ragged fields", + &PropertyList{ + Property{Name: "red.S", Value: "rot", NoIndex: false}, + Property{Name: "green.Nonymous.I", Value: int64(10), NoIndex: false}, + Property{Name: "green.Nonymous.I", Value: int64(11), NoIndex: false}, + Property{Name: "green.Nonymous.I", Value: int64(12), NoIndex: false}, + Property{Name: "green.Nonymous.I", Value: int64(13), NoIndex: false}, + Property{Name: "Blue.Nonymous.S", Value: "blau0", NoIndex: false}, + Property{Name: "Blue.Nonymous.I", Value: int64(20), NoIndex: false}, + Property{Name: "Blue.Nonymous.S", Value: "blau1", NoIndex: false}, + Property{Name: "Blue.Nonymous.I", Value: int64(21), NoIndex: false}, + Property{Name: "Blue.Nonymous.S", Value: "blau2", NoIndex: false}, + }, + &N2{ + N1: N1{ + X0: X0{S: "rot"}, + }, + Green: N1{ + Nonymous: []X0{ + {I: 10}, + {I: 11}, + {I: 12}, + {I: 13}, + }, + }, + Blue: N1{ + Nonymous: []X0{ + {S: "blau0", I: 20}, + {S: "blau1", I: 21}, + {S: "blau2"}, + }, + }, + }, + "", + "", + }, + { + "save structs with noindex tags", + &struct { + A struct { + X string `datastore:",noindex"` + Y string + } `datastore:",noindex"` + B struct { + X string `datastore:",noindex"` + Y string + } + }{}, + &PropertyList{ + Property{Name: "A.X", Value: "", NoIndex: true}, + Property{Name: "A.Y", Value: "", NoIndex: true}, + Property{Name: "B.X", Value: "", NoIndex: true}, + Property{Name: "B.Y", Value: "", NoIndex: false}, + }, + "", + "", + }, + { + "embedded struct with name override", + &struct { + Inner1 `datastore:"foo"` + }{}, + &PropertyList{ + Property{Name: "foo.W", Value: int64(0), NoIndex: false}, + Property{Name: "foo.X", Value: "", NoIndex: false}, + }, + "", + "", + }, + { + "slice of slices", + &SliceOfSlices{}, + nil, + "flattening nested structs leads to a slice of slices", + "", + }, + { + "recursive struct", + &Recursive{}, + nil, + "recursive struct", + "", + }, + { + "mutually recursive struct", + &MutuallyRecursive0{}, + nil, + "recursive struct", + "", + }, + { + "non-exported struct fields", + &struct { + i, J int64 + }{i: 1, J: 2}, + &PropertyList{ + Property{Name: "J", Value: int64(2), NoIndex: false}, + }, + "", + "", + }, + { + "json.RawMessage", + &struct { + J json.RawMessage + }{ + J: json.RawMessage("rawr"), + }, + &PropertyList{ + Property{Name: "J", Value: []byte("rawr"), NoIndex: false}, + }, + "", + "", + }, + { + "json.RawMessage to myBlob", + &struct { + B json.RawMessage + }{ + B: json.RawMessage("rawr"), + }, + &B2{B: myBlob("rawr")}, + "", + "", + }, +} + +// checkErr returns the empty string if either both want and err are zero, +// or if want is a non-empty substring of err's string representation. +func checkErr(want string, err error) string { + if err != nil { + got := err.Error() + if want == "" || strings.Index(got, want) == -1 { + return got + } + } else if want != "" { + return fmt.Sprintf("want error %q", want) + } + return "" +} + +func TestRoundTrip(t *testing.T) { + for _, tc := range testCases { + p, err := saveEntity(testKey0, tc.src) + if s := checkErr(tc.putErr, err); s != "" { + t.Errorf("%s: save: %s", tc.desc, s) + continue + } + if p == nil { + continue + } + var got interface{} + if _, ok := tc.want.(*PropertyList); ok { + got = new(PropertyList) + } else { + got = reflect.New(reflect.TypeOf(tc.want).Elem()).Interface() + } + err = loadEntity(got, p) + if s := checkErr(tc.getErr, err); s != "" { + t.Errorf("%s: load: %s", tc.desc, s) + continue + } + equal := false + if gotT, ok := got.(*T); ok { + // Round tripping a time.Time can result in a different time.Location: Local instead of UTC. + // We therefore test equality explicitly, instead of relying on reflect.DeepEqual. + equal = gotT.T.Equal(tc.want.(*T).T) + } else { + equal = reflect.DeepEqual(got, tc.want) + } + if !equal { + t.Errorf("%s: compare: got %v want %v", tc.desc, got, tc.want) + continue + } + } +} + +func TestQueryConstruction(t *testing.T) { + tests := []struct { + q, exp *Query + err string + }{ + { + q: NewQuery("Foo"), + exp: &Query{ + kind: "Foo", + limit: -1, + }, + }, + { + // Regular filtered query with standard spacing. + q: NewQuery("Foo").Filter("foo >", 7), + exp: &Query{ + kind: "Foo", + filter: []filter{ + { + FieldName: "foo", + Op: greaterThan, + Value: 7, + }, + }, + limit: -1, + }, + }, + { + // Filtered query with no spacing. + q: NewQuery("Foo").Filter("foo=", 6), + exp: &Query{ + kind: "Foo", + filter: []filter{ + { + FieldName: "foo", + Op: equal, + Value: 6, + }, + }, + limit: -1, + }, + }, + { + // Filtered query with funky spacing. + q: NewQuery("Foo").Filter(" foo< ", 8), + exp: &Query{ + kind: "Foo", + filter: []filter{ + { + FieldName: "foo", + Op: lessThan, + Value: 8, + }, + }, + limit: -1, + }, + }, + { + // Filtered query with multicharacter op. + q: NewQuery("Foo").Filter("foo >=", 9), + exp: &Query{ + kind: "Foo", + filter: []filter{ + { + FieldName: "foo", + Op: greaterEq, + Value: 9, + }, + }, + limit: -1, + }, + }, + { + // Query with ordering. + q: NewQuery("Foo").Order("bar"), + exp: &Query{ + kind: "Foo", + order: []order{ + { + FieldName: "bar", + Direction: ascending, + }, + }, + limit: -1, + }, + }, + { + // Query with reverse ordering, and funky spacing. + q: NewQuery("Foo").Order(" - bar"), + exp: &Query{ + kind: "Foo", + order: []order{ + { + FieldName: "bar", + Direction: descending, + }, + }, + limit: -1, + }, + }, + { + // Query with an empty ordering. + q: NewQuery("Foo").Order(""), + err: "empty order", + }, + { + // Query with a + ordering. + q: NewQuery("Foo").Order("+bar"), + err: "invalid order", + }, + } + for i, test := range tests { + if test.q.err != nil { + got := test.q.err.Error() + if !strings.Contains(got, test.err) { + t.Errorf("%d: error mismatch: got %q want something containing %q", i, got, test.err) + } + continue + } + if !reflect.DeepEqual(test.q, test.exp) { + t.Errorf("%d: mismatch: got %v want %v", i, test.q, test.exp) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/errors.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/errors.go new file mode 100644 index 0000000000..3077f80d3b --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/errors.go @@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file provides error functions for common API failure modes. + +package datastore + +import ( + "fmt" +) + +// MultiError is returned by batch operations when there are errors with +// particular elements. Errors will be in a one-to-one correspondence with +// the input elements; successful elements will have a nil entry. +type MultiError []error + +func (m MultiError) Error() string { + s, n := "", 0 + for _, e := range m { + if e != nil { + if n == 0 { + s = e.Error() + } + n++ + } + } + switch n { + case 0: + return "(0 errors)" + case 1: + return s + case 2: + return s + " (and 1 other error)" + } + return fmt.Sprintf("%s (and %d other errors)", s, n-1) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/example_test.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/example_test.go new file mode 100644 index 0000000000..391991c95a --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/example_test.go @@ -0,0 +1,251 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore_test + +import ( + "io/ioutil" + "log" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" + "google.golang.org/cloud/datastore" +) + +// TODO(djd): reevaluate this example given new Client config. +func Example_auth() *datastore.Client { + // Initialize an authorized context with Google Developers Console + // JSON key. Read the google package examples to learn more about + // different authorization flows you can use. + // http://godoc.org/golang.org/x/oauth2/google + jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json") + if err != nil { + log.Fatal(err) + } + conf, err := google.JWTConfigFromJSON( + jsonKey, + datastore.ScopeDatastore, + datastore.ScopeUserEmail, + ) + if err != nil { + log.Fatal(err) + } + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id", cloud.WithTokenSource(conf.TokenSource(ctx))) + if err != nil { + log.Fatal(err) + } + // Use the client (see other examples). + return client +} + +func ExampleGet() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + + type Article struct { + Title string + Description string + Body string `datastore:",noindex"` + Author *datastore.Key + PublishedAt time.Time + } + key := datastore.NewKey(ctx, "Article", "articled1", 0, nil) + article := &Article{} + if err := client.Get(ctx, key, article); err != nil { + log.Fatal(err) + } +} + +func ExamplePut() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + + type Article struct { + Title string + Description string + Body string `datastore:",noindex"` + Author *datastore.Key + PublishedAt time.Time + } + newKey := datastore.NewIncompleteKey(ctx, "Article", nil) + _, err = client.Put(ctx, newKey, &Article{ + Title: "The title of the article", + Description: "The description of the article...", + Body: "...", + Author: datastore.NewKey(ctx, "Author", "jbd", 0, nil), + PublishedAt: time.Now(), + }) + if err != nil { + log.Fatal(err) + } +} + +func ExampleDelete() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + + key := datastore.NewKey(ctx, "Article", "articled1", 0, nil) + if err := client.Delete(ctx, key); err != nil { + log.Fatal(err) + } +} + +type Post struct { + Title string + PublishedAt time.Time + Comments int +} + +func ExampleGetMulti() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + + keys := []*datastore.Key{ + datastore.NewKey(ctx, "Post", "post1", 0, nil), + datastore.NewKey(ctx, "Post", "post2", 0, nil), + datastore.NewKey(ctx, "Post", "post3", 0, nil), + } + posts := make([]Post, 3) + if err := client.GetMulti(ctx, keys, posts); err != nil { + log.Println(err) + } +} + +func ExamplePutMulti_slice() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + + keys := []*datastore.Key{ + datastore.NewKey(ctx, "Post", "post1", 0, nil), + datastore.NewKey(ctx, "Post", "post2", 0, nil), + } + + // PutMulti with a Post slice. + posts := []*Post{ + {Title: "Post 1", PublishedAt: time.Now()}, + {Title: "Post 2", PublishedAt: time.Now()}, + } + if _, err := client.PutMulti(ctx, keys, posts); err != nil { + log.Fatal(err) + } +} + +func ExamplePutMulti_interfaceSlice() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + + keys := []*datastore.Key{ + datastore.NewKey(ctx, "Post", "post1", 0, nil), + datastore.NewKey(ctx, "Post", "post2", 0, nil), + } + + // PutMulti with an empty interface slice. + posts := []interface{}{ + &Post{Title: "Post 1", PublishedAt: time.Now()}, + &Post{Title: "Post 2", PublishedAt: time.Now()}, + } + if _, err := client.PutMulti(ctx, keys, posts); err != nil { + log.Fatal(err) + } +} + +func ExampleQuery() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + + // Count the number of the post entities. + q := datastore.NewQuery("Post") + n, err := client.Count(ctx, q) + if err != nil { + log.Fatal(err) + } + log.Println("There are %d posts.", n) + + // List the posts published since yesterday. + yesterday := time.Now().Add(-24 * time.Hour) + q = datastore.NewQuery("Post").Filter("PublishedAt >", yesterday) + it := client.Run(ctx, q) + // Use the iterator. + _ = it + + // Order the posts by the number of comments they have recieved. + datastore.NewQuery("Post").Order("-Comments") + + // Start listing from an offset and limit the results. + datastore.NewQuery("Post").Offset(20).Limit(10) +} + +func ExampleTransaction() { + ctx := context.Background() + client, err := datastore.NewClient(ctx, "project-id") + if err != nil { + log.Fatal(err) + } + const retries = 3 + + // Increment a counter. + // See https://cloud.google.com/appengine/articles/sharding_counters for + // a more scalable solution. + type Counter struct { + Count int + } + + key := datastore.NewKey(ctx, "counter", "CounterA", 0, nil) + + for i := 0; i < retries; i++ { + tx, err := client.NewTransaction(ctx) + if err != nil { + break + } + + var c Counter + if err := tx.Get(key, &c); err != nil && err != datastore.ErrNoSuchEntity { + break + } + c.Count++ + if _, err := tx.Put(key, &c); err != nil { + break + } + + // Attempt to commit the transaction. If there's a conflict, try again. + if _, err := tx.Commit(); err != datastore.ErrConcurrentTransaction { + break + } + } + +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/integration_test.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/integration_test.go new file mode 100644 index 0000000000..01d52ab654 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/integration_test.go @@ -0,0 +1,569 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build integration + +package datastore + +import ( + "fmt" + "log" + + "reflect" + "sort" + "strings" + "testing" + "time" + + "golang.org/x/net/context" + "google.golang.org/cloud" + "google.golang.org/cloud/internal/testutil" +) + +func newClient(ctx context.Context) *Client { + ts := testutil.TokenSource(ctx, ScopeDatastore, ScopeUserEmail) + client, err := NewClient(ctx, testutil.ProjID(), cloud.WithTokenSource(ts)) + if err != nil { + log.Fatal(err) + } + return client +} + +func TestBasics(t *testing.T) { + type X struct { + I int + S string + T time.Time + } + ctx := context.Background() + client := newClient(ctx) + x0 := X{66, "99", time.Now().Truncate(time.Millisecond)} + k, err := client.Put(ctx, NewIncompleteKey(ctx, "BasicsX", nil), &x0) + if err != nil { + t.Fatalf("client.Put: %v", err) + } + x1 := X{} + err = client.Get(ctx, k, &x1) + if err != nil { + t.Errorf("client.Get: %v", err) + } + err = client.Delete(ctx, k) + if err != nil { + t.Errorf("client.Delete: %v", err) + } + if !reflect.DeepEqual(x0, x1) { + t.Errorf("compare: x0=%v, x1=%v", x0, x1) + } +} + +func TestListValues(t *testing.T) { + p0 := PropertyList{ + {Name: "L", Value: int64(12), Multiple: true}, + {Name: "L", Value: "string", Multiple: true}, + {Name: "L", Value: true, Multiple: true}, + } + ctx := context.Background() + client := newClient(ctx) + k, err := client.Put(ctx, NewIncompleteKey(ctx, "ListValue", nil), &p0) + if err != nil { + t.Fatalf("client.Put: %v", err) + } + var p1 PropertyList + if err := client.Get(ctx, k, &p1); err != nil { + t.Errorf("client.Get: %v", err) + } + if !reflect.DeepEqual(p0, p1) { + t.Errorf("compare:\np0=%v\np1=%#v", p0, p1) + } + if err = client.Delete(ctx, k); err != nil { + t.Errorf("client.Delete: %v", err) + } +} + +func TestGetMulti(t *testing.T) { + type X struct { + I int + } + ctx := context.Background() + client := newClient(ctx) + p := NewKey(ctx, "X", "", time.Now().Unix(), nil) + + cases := []struct { + key *Key + put bool + }{ + {key: NewKey(ctx, "X", "item1", 0, p), put: true}, + {key: NewKey(ctx, "X", "item2", 0, p), put: false}, + {key: NewKey(ctx, "X", "item3", 0, p), put: false}, + {key: NewKey(ctx, "X", "item4", 0, p), put: true}, + } + + var src, dst []*X + var srcKeys, dstKeys []*Key + for _, c := range cases { + dst = append(dst, &X{}) + dstKeys = append(dstKeys, c.key) + if c.put { + src = append(src, &X{}) + srcKeys = append(srcKeys, c.key) + } + } + if _, err := client.PutMulti(ctx, srcKeys, src); err != nil { + t.Error(err) + } + err := client.GetMulti(ctx, dstKeys, dst) + if err == nil { + t.Errorf("client.GetMulti got %v, expected error", err) + } + e, ok := err.(MultiError) + if !ok { + t.Errorf("client.GetMulti got %t, expected MultiError", err) + } + for i, err := range e { + got, want := err, (error)(nil) + if !cases[i].put { + got, want = err, ErrNoSuchEntity + } + if got != want { + t.Errorf("MultiError[%d] == %v, want %v", i, got, want) + } + } +} + +type Z struct { + S string + T string `datastore:",noindex"` + P []byte + K []byte `datastore:",noindex"` +} + +func (z Z) String() string { + var lens []string + v := reflect.ValueOf(z) + for i := 0; i < v.NumField(); i++ { + if l := v.Field(i).Len(); l > 0 { + lens = append(lens, fmt.Sprintf("len(%s)=%d", v.Type().Field(i).Name, l)) + } + } + return fmt.Sprintf("Z{ %s }", strings.Join(lens, ",")) +} + +func TestUnindexableValues(t *testing.T) { + x1500 := strings.Repeat("x", 1500) + x1501 := strings.Repeat("x", 1501) + testCases := []struct { + in Z + wantErr bool + }{ + {in: Z{S: x1500}, wantErr: false}, + {in: Z{S: x1501}, wantErr: true}, + {in: Z{T: x1500}, wantErr: false}, + {in: Z{T: x1501}, wantErr: false}, + {in: Z{P: []byte(x1500)}, wantErr: false}, + {in: Z{P: []byte(x1501)}, wantErr: true}, + {in: Z{K: []byte(x1500)}, wantErr: false}, + {in: Z{K: []byte(x1501)}, wantErr: false}, + } + ctx := context.Background() + client := newClient(ctx) + for _, tt := range testCases { + _, err := client.Put(ctx, NewIncompleteKey(ctx, "BasicsZ", nil), &tt.in) + if (err != nil) != tt.wantErr { + t.Errorf("client.Put %s got err %v, want err %t", tt.in, err, tt.wantErr) + } + } +} + +type SQChild struct { + I, J int + T, U int64 +} + +type SQTestCase struct { + desc string + q *Query + wantCount int + wantSum int +} + +func testSmallQueries(t *testing.T, ctx context.Context, client *Client, parent *Key, children []*SQChild, + testCases []SQTestCase, extraTests ...func()) { + keys := make([]*Key, len(children)) + for i := range keys { + keys[i] = NewIncompleteKey(ctx, "SQChild", parent) + } + keys, err := client.PutMulti(ctx, keys, children) + if err != nil { + t.Fatalf("client.PutMulti: %v", err) + } + defer func() { + err := client.DeleteMulti(ctx, keys) + if err != nil { + t.Errorf("client.DeleteMulti: %v", err) + } + }() + + for _, tc := range testCases { + count, err := client.Count(ctx, tc.q) + if err != nil { + t.Errorf("Count %q: %v", tc.desc, err) + continue + } + if count != tc.wantCount { + t.Errorf("Count %q: got %d want %d", tc.desc, count, tc.wantCount) + continue + } + } + + for _, tc := range testCases { + var got []SQChild + _, err := client.GetAll(ctx, tc.q, &got) + if err != nil { + t.Errorf("client.GetAll %q: %v", tc.desc, err) + continue + } + sum := 0 + for _, c := range got { + sum += c.I + c.J + } + if sum != tc.wantSum { + t.Errorf("sum %q: got %d want %d", tc.desc, sum, tc.wantSum) + continue + } + } + for _, x := range extraTests { + x() + } +} + +func TestFilters(t *testing.T) { + ctx := context.Background() + client := newClient(ctx) + parent := NewKey(ctx, "SQParent", "TestFilters", 0, nil) + now := time.Now().Truncate(time.Millisecond).Unix() + children := []*SQChild{ + {I: 0, T: now, U: now}, + {I: 1, T: now, U: now}, + {I: 2, T: now, U: now}, + {I: 3, T: now, U: now}, + {I: 4, T: now, U: now}, + {I: 5, T: now, U: now}, + {I: 6, T: now, U: now}, + {I: 7, T: now, U: now}, + } + baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now) + testSmallQueries(t, ctx, client, parent, children, []SQTestCase{ + { + "I>1", + baseQuery.Filter("I>", 1), + 6, + 2 + 3 + 4 + 5 + 6 + 7, + }, + { + "I>2 AND I<=5", + baseQuery.Filter("I>", 2).Filter("I<=", 5), + 3, + 3 + 4 + 5, + }, + { + "I>=3 AND I<3", + baseQuery.Filter("I>=", 3).Filter("I<", 3), + 0, + 0, + }, + { + "I=4", + baseQuery.Filter("I=", 4), + 1, + 4, + }, + }, func() { + got := []*SQChild{} + want := []*SQChild{ + {I: 0, T: now, U: now}, + {I: 1, T: now, U: now}, + {I: 2, T: now, U: now}, + {I: 3, T: now, U: now}, + {I: 4, T: now, U: now}, + {I: 5, T: now, U: now}, + {I: 6, T: now, U: now}, + {I: 7, T: now, U: now}, + } + _, err := client.GetAll(ctx, baseQuery.Order("I"), &got) + if err != nil { + t.Errorf("client.GetAll: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("compare: got=%v, want=%v", got, want) + } + }, func() { + got := []*SQChild{} + want := []*SQChild{ + {I: 7, T: now, U: now}, + {I: 6, T: now, U: now}, + {I: 5, T: now, U: now}, + {I: 4, T: now, U: now}, + {I: 3, T: now, U: now}, + {I: 2, T: now, U: now}, + {I: 1, T: now, U: now}, + {I: 0, T: now, U: now}, + } + _, err := client.GetAll(ctx, baseQuery.Order("-I"), &got) + if err != nil { + t.Errorf("client.GetAll: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("compare: got=%v, want=%v", got, want) + } + }) +} + +func TestEventualConsistency(t *testing.T) { + ctx := context.Background() + client := newClient(ctx) + parent := NewKey(ctx, "SQParent", "TestEventualConsistency", 0, nil) + now := time.Now().Truncate(time.Millisecond).Unix() + children := []*SQChild{ + {I: 0, T: now, U: now}, + {I: 1, T: now, U: now}, + {I: 2, T: now, U: now}, + } + query := NewQuery("SQChild").Ancestor(parent).Filter("T =", now).EventualConsistency() + testSmallQueries(t, ctx, client, parent, children, nil, func() { + got, err := client.Count(ctx, query) + if err != nil { + t.Fatalf("Count: %v", err) + } + if got < 0 || 3 < got { + t.Errorf("Count: got %d, want [0,3]", got) + } + }) +} + +func TestProjection(t *testing.T) { + ctx := context.Background() + client := newClient(ctx) + parent := NewKey(ctx, "SQParent", "TestProjection", 0, nil) + now := time.Now().Truncate(time.Millisecond).Unix() + children := []*SQChild{ + {I: 1 << 0, J: 100, T: now, U: now}, + {I: 1 << 1, J: 100, T: now, U: now}, + {I: 1 << 2, J: 200, T: now, U: now}, + {I: 1 << 3, J: 300, T: now, U: now}, + {I: 1 << 4, J: 300, T: now, U: now}, + } + baseQuery := NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("J>", 150) + testSmallQueries(t, ctx, client, parent, children, []SQTestCase{ + { + "project", + baseQuery.Project("J"), + 3, + 200 + 300 + 300, + }, + { + "distinct", + baseQuery.Project("J").Distinct(), + 2, + 200 + 300, + }, + { + "project on meaningful (GD_WHEN) field", + baseQuery.Project("U"), + 3, + 0, + }, + }) +} + +func TestAllocateIDs(t *testing.T) { + ctx := context.Background() + client := newClient(ctx) + keys := make([]*Key, 5) + for i := range keys { + keys[i] = NewIncompleteKey(ctx, "AllocID", nil) + } + keys, err := client.AllocateIDs(ctx, keys) + if err != nil { + t.Errorf("AllocID #0 failed: %v", err) + } + if want := len(keys); want != 5 { + t.Errorf("Expected to allocate 5 keys, %d keys are found", want) + } + for _, k := range keys { + if k.Incomplete() { + t.Errorf("Unexpeceted incomplete key found: %v", k) + } + } +} + +func TestGetAllWithFieldMismatch(t *testing.T) { + type Fat struct { + X, Y int + } + type Thin struct { + X int + } + + ctx := context.Background() + client := newClient(ctx) + // Ancestor queries (those within an entity group) are strongly consistent + // by default, which prevents a test from being flaky. + // See https://cloud.google.com/appengine/docs/go/datastore/queries#Go_Data_consistency + // for more information. + parent := NewKey(ctx, "SQParent", "TestGetAllWithFieldMismatch", 0, nil) + putKeys := make([]*Key, 3) + for i := range putKeys { + putKeys[i] = NewKey(ctx, "GetAllThing", "", int64(10+i), parent) + _, err := client.Put(ctx, putKeys[i], &Fat{X: 20 + i, Y: 30 + i}) + if err != nil { + t.Fatalf("client.Put: %v", err) + } + } + + var got []Thin + want := []Thin{ + {X: 20}, + {X: 21}, + {X: 22}, + } + getKeys, err := client.GetAll(ctx, NewQuery("GetAllThing").Ancestor(parent), &got) + if len(getKeys) != 3 && !reflect.DeepEqual(getKeys, putKeys) { + t.Errorf("client.GetAll: keys differ\ngetKeys=%v\nputKeys=%v", getKeys, putKeys) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("client.GetAll: entities differ\ngot =%v\nwant=%v", got, want) + } + if _, ok := err.(*ErrFieldMismatch); !ok { + t.Errorf("client.GetAll: got err=%v, want ErrFieldMismatch", err) + } +} + +func TestKindlessQueries(t *testing.T) { + type Dee struct { + I int + Why string + } + type Dum struct { + I int + Pling string + } + + ctx := context.Background() + client := newClient(ctx) + parent := NewKey(ctx, "Tweedle", "tweedle", 0, nil) + + keys := []*Key{ + NewKey(ctx, "Dee", "dee0", 0, parent), + NewKey(ctx, "Dum", "dum1", 0, parent), + NewKey(ctx, "Dum", "dum2", 0, parent), + NewKey(ctx, "Dum", "dum3", 0, parent), + } + src := []interface{}{ + &Dee{1, "binary0001"}, + &Dum{2, "binary0010"}, + &Dum{4, "binary0100"}, + &Dum{8, "binary1000"}, + } + keys, err := client.PutMulti(ctx, keys, src) + if err != nil { + t.Fatalf("put: %v", err) + } + + testCases := []struct { + desc string + query *Query + want []int + wantErr string + }{ + { + desc: "Dee", + query: NewQuery("Dee"), + want: []int{1}, + }, + { + desc: "Doh", + query: NewQuery("Doh"), + want: nil}, + { + desc: "Dum", + query: NewQuery("Dum"), + want: []int{2, 4, 8}, + }, + { + desc: "", + query: NewQuery(""), + want: []int{1, 2, 4, 8}, + }, + { + desc: "Kindless filter", + query: NewQuery("").Filter("__key__ =", keys[2]), + want: []int{4}, + }, + { + desc: "Kindless order", + query: NewQuery("").Order("__key__"), + want: []int{1, 2, 4, 8}, + }, + { + desc: "Kindless bad filter", + query: NewQuery("").Filter("I =", 4), + wantErr: "kind is required for filter: I", + }, + { + desc: "Kindless bad order", + query: NewQuery("").Order("-__key__"), + wantErr: "kind is required for all orders except __key__ ascending", + }, + } +loop: + for _, tc := range testCases { + q := tc.query.Ancestor(parent) + gotCount, err := client.Count(ctx, q) + if err != nil { + if tc.wantErr == "" || !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("count %q: err %v, want err %q", tc.desc, err, tc.wantErr) + } + continue + } + if tc.wantErr != "" { + t.Errorf("count %q: want err %q", tc.desc, tc.wantErr) + continue + } + if gotCount != len(tc.want) { + t.Errorf("count %q: got %d want %d", tc.desc, gotCount, len(tc.want)) + continue + } + var got []int + for iter := client.Run(ctx, q); ; { + var dst struct { + I int + Why, Pling string + } + _, err := iter.Next(&dst) + if err == Done { + break + } + if err != nil { + t.Errorf("iter.Next %q: %v", tc.desc, err) + continue loop + } + got = append(got, dst.I) + } + sort.Ints(got) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("elems %q: got %+v want %+v", tc.desc, got, tc.want) + continue + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/key.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/key.go new file mode 100644 index 0000000000..a1947d0c03 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/key.go @@ -0,0 +1,277 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "bytes" + "encoding/base64" + "encoding/gob" + "errors" + "strconv" + "strings" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + pb "google.golang.org/cloud/internal/datastore" +) + +// Key represents the datastore key for a stored entity, and is immutable. +type Key struct { + kind string + id int64 + name string + parent *Key + + namespace string +} + +func (k *Key) Kind() string { + return k.kind +} + +func (k *Key) ID() int64 { + return k.id +} + +func (k *Key) Name() string { + return k.name +} + +func (k *Key) Parent() *Key { + return k.parent +} + +func (k *Key) SetParent(v *Key) { + if v.Incomplete() { + panic("can't set an incomplete key as parent") + } + k.parent = v +} + +func (k *Key) Namespace() string { + return k.namespace +} + +// Complete returns whether the key does not refer to a stored entity. +func (k *Key) Incomplete() bool { + return k.name == "" && k.id == 0 +} + +// valid returns whether the key is valid. +func (k *Key) valid() bool { + if k == nil { + return false + } + for ; k != nil; k = k.parent { + if k.kind == "" { + return false + } + if k.name != "" && k.id != 0 { + return false + } + if k.parent != nil { + if k.parent.Incomplete() { + return false + } + if k.parent.namespace != k.namespace { + return false + } + } + } + return true +} + +func (k *Key) Equal(o *Key) bool { + for { + if k == nil || o == nil { + return k == o // if either is nil, both must be nil + } + if k.namespace != o.namespace || k.name != o.name || k.id != o.id || k.kind != o.kind { + return false + } + if k.parent == nil && o.parent == nil { + return true + } + k = k.parent + o = o.parent + } +} + +// marshal marshals the key's string representation to the buffer. +func (k *Key) marshal(b *bytes.Buffer) { + if k.parent != nil { + k.parent.marshal(b) + } + b.WriteByte('/') + b.WriteString(k.kind) + b.WriteByte(',') + if k.name != "" { + b.WriteString(k.name) + } else { + b.WriteString(strconv.FormatInt(k.id, 10)) + } +} + +// String returns a string representation of the key. +func (k *Key) String() string { + if k == nil { + return "" + } + b := bytes.NewBuffer(make([]byte, 0, 512)) + k.marshal(b) + return b.String() +} + +// Note: Fields not renamed compared to appengine gobKey struct +// This ensures gobs created by appengine can be read here, and vice/versa +type gobKey struct { + Kind string + StringID string + IntID int64 + Parent *gobKey + AppID string + Namespace string +} + +func keyToGobKey(k *Key) *gobKey { + if k == nil { + return nil + } + return &gobKey{ + Kind: k.kind, + StringID: k.name, + IntID: k.id, + Parent: keyToGobKey(k.parent), + Namespace: k.namespace, + } +} + +func gobKeyToKey(gk *gobKey) *Key { + if gk == nil { + return nil + } + return &Key{ + kind: gk.Kind, + name: gk.StringID, + id: gk.IntID, + parent: gobKeyToKey(gk.Parent), + namespace: gk.Namespace, + } +} + +func (k *Key) GobEncode() ([]byte, error) { + buf := new(bytes.Buffer) + if err := gob.NewEncoder(buf).Encode(keyToGobKey(k)); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (k *Key) GobDecode(buf []byte) error { + gk := new(gobKey) + if err := gob.NewDecoder(bytes.NewBuffer(buf)).Decode(gk); err != nil { + return err + } + *k = *gobKeyToKey(gk) + return nil +} + +func (k *Key) MarshalJSON() ([]byte, error) { + return []byte(`"` + k.Encode() + `"`), nil +} + +func (k *Key) UnmarshalJSON(buf []byte) error { + if len(buf) < 2 || buf[0] != '"' || buf[len(buf)-1] != '"' { + return errors.New("datastore: bad JSON key") + } + k2, err := DecodeKey(string(buf[1 : len(buf)-1])) + if err != nil { + return err + } + *k = *k2 + return nil +} + +// Encode returns an opaque representation of the key +// suitable for use in HTML and URLs. +// This is compatible with the Python and Java runtimes. +func (k *Key) Encode() string { + pKey := keyToProto(k) + + b, err := proto.Marshal(pKey) + if err != nil { + panic(err) + } + + // Trailing padding is stripped. + return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") +} + +// DecodeKey decodes a key from the opaque representation returned by Encode. +func DecodeKey(encoded string) (*Key, error) { + // Re-add padding. + if m := len(encoded) % 4; m != 0 { + encoded += strings.Repeat("=", 4-m) + } + + b, err := base64.URLEncoding.DecodeString(encoded) + if err != nil { + return nil, err + } + + pKey := new(pb.Key) + if err := proto.Unmarshal(b, pKey); err != nil { + return nil, err + } + + return protoToKey(pKey), nil +} + +// NewIncompleteKey creates a new incomplete key. +// kind cannot be empty. +func NewIncompleteKey(ctx context.Context, kind string, parent *Key) *Key { + return NewKey(ctx, kind, "", 0, parent) +} + +// NewKey creates a new key. +// kind cannot be empty. +// Either one or both of name and id must be zero. If both are zero, +// the key returned is incomplete. +// parent must either be a complete key or nil. +func NewKey(ctx context.Context, kind, name string, id int64, parent *Key) *Key { + return &Key{ + kind: kind, + name: name, + id: id, + parent: parent, + namespace: ctxNamespace(ctx), + } +} + +// AllocateIDs accepts a slice of incomplete keys and returns a +// slice of complete keys that are guaranteed to be valid in the datastore +func (c *Client) AllocateIDs(ctx context.Context, keys []*Key) ([]*Key, error) { + if keys == nil { + return nil, nil + } + + req := &pb.AllocateIdsRequest{Key: multiKeyToProto(keys)} + res := &pb.AllocateIdsResponse{} + if err := c.call(ctx, "allocateIds", req, res); err != nil { + return nil, err + } + + return multiProtoToKey(res.Key), nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/key_test.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/key_test.go new file mode 100644 index 0000000000..e134ffd586 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/key_test.go @@ -0,0 +1,223 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "bytes" + "encoding/gob" + "encoding/json" + "testing" + + "golang.org/x/net/context" +) + +func TestNamespace(t *testing.T) { + c := context.Background() + k := NewIncompleteKey(c, "foo", nil) + if got, want := k.Namespace(), ""; got != want { + t.Errorf("No namespace, k.Namespace() = %q, want %q", got, want) + } + + c = WithNamespace(c, "gopherspace") + k = NewIncompleteKey(c, "foo", nil) + if got, want := k.Namespace(), "gopherspace"; got != want { + t.Errorf("No namespace, k.Namespace() = %q, want %q", got, want) + } +} + +func TestParent(t *testing.T) { + c := context.Background() + k := NewIncompleteKey(c, "foo", nil) + par := NewKey(c, "foomum", "", 1248, nil) + k.SetParent(par) + if got := k.Parent(); got != par { + t.Errorf("k.Parent() = %v; want %v", got, par) + } +} + +func TestEqual(t *testing.T) { + c := context.Background() + cN := WithNamespace(c, "gopherspace") + + testCases := []struct { + x, y *Key + equal bool + }{ + { + x: nil, + y: nil, + equal: true, + }, + { + x: NewKey(c, "kindA", "", 0, nil), + y: NewIncompleteKey(c, "kindA", nil), + equal: true, + }, + { + x: NewKey(c, "kindA", "nameA", 0, nil), + y: NewKey(c, "kindA", "nameA", 0, nil), + equal: true, + }, + { + x: NewKey(cN, "kindA", "nameA", 0, nil), + y: NewKey(cN, "kindA", "nameA", 0, nil), + equal: true, + }, + { + x: NewKey(c, "kindA", "", 1337, NewKey(c, "kindX", "nameX", 0, nil)), + y: NewKey(c, "kindA", "", 1337, NewKey(c, "kindX", "nameX", 0, nil)), + equal: true, + }, + { + x: NewKey(c, "kindA", "nameA", 0, nil), + y: NewKey(c, "kindB", "nameA", 0, nil), + equal: false, + }, + { + x: NewKey(c, "kindA", "nameA", 0, nil), + y: NewKey(c, "kindA", "nameB", 0, nil), + equal: false, + }, + { + x: NewKey(c, "kindA", "nameA", 0, nil), + y: NewKey(c, "kindA", "", 1337, nil), + equal: false, + }, + { + x: NewKey(c, "kindA", "nameA", 0, nil), + y: NewKey(cN, "kindA", "nameA", 0, nil), + equal: false, + }, + { + x: NewKey(c, "kindA", "", 1337, NewKey(c, "kindX", "nameX", 0, nil)), + y: NewKey(c, "kindA", "", 1337, NewKey(c, "kindY", "nameX", 0, nil)), + equal: false, + }, + { + x: NewKey(c, "kindA", "", 1337, NewKey(c, "kindX", "nameX", 0, nil)), + y: NewKey(c, "kindA", "", 1337, nil), + equal: false, + }, + } + + for _, tt := range testCases { + if got := tt.x.Equal(tt.y); got != tt.equal { + t.Errorf("Equal(%v, %v) = %t; want %t", tt.x, tt.y, got, tt.equal) + } + if got := tt.y.Equal(tt.x); got != tt.equal { + t.Errorf("Equal(%v, %v) = %t; want %t", tt.y, tt.x, got, tt.equal) + } + } +} + +func TestEncoding(t *testing.T) { + c := context.Background() + cN := WithNamespace(c, "gopherspace") + + testCases := []struct { + k *Key + valid bool + }{ + { + k: nil, + valid: false, + }, + { + k: NewKey(c, "", "", 0, nil), + valid: false, + }, + { + k: NewKey(c, "kindA", "", 0, nil), + valid: true, + }, + { + k: NewKey(cN, "kindA", "", 0, nil), + valid: true, + }, + { + k: NewKey(c, "kindA", "nameA", 0, nil), + valid: true, + }, + { + k: NewKey(c, "kindA", "", 1337, nil), + valid: true, + }, + { + k: NewKey(c, "kindA", "nameA", 1337, nil), + valid: false, + }, + { + k: NewKey(c, "kindA", "", 0, NewKey(c, "kindB", "nameB", 0, nil)), + valid: true, + }, + { + k: NewKey(c, "kindA", "", 0, NewKey(c, "kindB", "", 0, nil)), + valid: false, + }, + { + k: NewKey(c, "kindA", "", 0, NewKey(cN, "kindB", "nameB", 0, nil)), + valid: false, + }, + } + + for _, tt := range testCases { + if got := tt.k.valid(); got != tt.valid { + t.Errorf("valid(%v) = %t; want %t", tt.k, got, tt.valid) + } + + // Check encoding/decoding for valid keys. + if !tt.valid { + continue + } + enc := tt.k.Encode() + dec, err := DecodeKey(enc) + if err != nil { + t.Errorf("DecodeKey(%q) from %v: %v", enc, tt.k, err) + continue + } + if !tt.k.Equal(dec) { + t.Errorf("Decoded key %v not equal to %v", dec, tt.k) + } + + b, err := json.Marshal(tt.k) + if err != nil { + t.Errorf("json.Marshal(%v): %v", tt.k, err) + continue + } + key := &Key{} + if err := json.Unmarshal(b, key); err != nil { + t.Errorf("json.Unmarshal(%s) for key %v: %v", b, tt.k, err) + continue + } + if !tt.k.Equal(key) { + t.Errorf("JSON decoded key %v not equal to %v", dec, tt.k) + } + + buf := &bytes.Buffer{} + gobEnc := gob.NewEncoder(buf) + if err := gobEnc.Encode(tt.k); err != nil { + t.Errorf("gobEnc.Encode(%v): %v", tt.k, err) + continue + } + gobDec := gob.NewDecoder(buf) + key = &Key{} + if err := gobDec.Decode(key); err != nil { + t.Errorf("gobDec.Decode() for key %v: %v", tt.k, err) + } + if !tt.k.Equal(key) { + t.Errorf("gob decoded key %v not equal to %v", dec, tt.k) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/load.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/load.go new file mode 100644 index 0000000000..ef2e211bf5 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/load.go @@ -0,0 +1,275 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "fmt" + "reflect" + "time" + + pb "google.golang.org/cloud/internal/datastore" +) + +var ( + typeOfByteSlice = reflect.TypeOf([]byte(nil)) + typeOfTime = reflect.TypeOf(time.Time{}) +) + +// typeMismatchReason returns a string explaining why the property p could not +// be stored in an entity field of type v.Type(). +func typeMismatchReason(p Property, v reflect.Value) string { + entityType := "empty" + switch p.Value.(type) { + case int64: + entityType = "int" + case bool: + entityType = "bool" + case string: + entityType = "string" + case float64: + entityType = "float" + case *Key: + entityType = "*datastore.Key" + case time.Time: + entityType = "time.Time" + case []byte: + entityType = "[]byte" + } + + return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type()) +} + +type propertyLoader struct { + // m holds the number of times a substruct field like "Foo.Bar.Baz" has + // been seen so far. The map is constructed lazily. + m map[string]int +} + +func (l *propertyLoader) load(codec *structCodec, structValue reflect.Value, p Property, prev map[string]struct{}) string { + var sliceOk bool + var v reflect.Value + // Traverse a struct's struct-typed fields. + for name := p.Name; ; { + decoder, ok := codec.byName[name] + if !ok { + return "no such struct field" + } + v = structValue.Field(decoder.index) + if !v.IsValid() { + return "no such struct field" + } + if !v.CanSet() { + return "cannot set struct field" + } + + if decoder.substructCodec == nil { + break + } + + if v.Kind() == reflect.Slice { + if l.m == nil { + l.m = make(map[string]int) + } + index := l.m[p.Name] + l.m[p.Name] = index + 1 + for v.Len() <= index { + v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem())) + } + structValue = v.Index(index) + sliceOk = true + } else { + structValue = v + } + // Strip the "I." from "I.X". + name = name[len(codec.byIndex[decoder.index].name):] + codec = decoder.substructCodec + } + + var slice reflect.Value + if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { + slice = v + v = reflect.New(v.Type().Elem()).Elem() + } else if _, ok := prev[p.Name]; ok && !sliceOk { + // Zero the field back out that was set previously, turns out its a slice and we don't know what to do with it + v.Set(reflect.Zero(v.Type())) + + return "multiple-valued property requires a slice field type" + } + + prev[p.Name] = struct{}{} + + pValue := p.Value + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + x, ok := pValue.(int64) + if !ok && pValue != nil { + return typeMismatchReason(p, v) + } + if v.OverflowInt(x) { + return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type()) + } + v.SetInt(x) + case reflect.Bool: + x, ok := pValue.(bool) + if !ok && pValue != nil { + return typeMismatchReason(p, v) + } + v.SetBool(x) + case reflect.String: + x, ok := pValue.(string) + if !ok && pValue != nil { + return typeMismatchReason(p, v) + } + v.SetString(x) + case reflect.Float32, reflect.Float64: + x, ok := pValue.(float64) + if !ok && pValue != nil { + return typeMismatchReason(p, v) + } + if v.OverflowFloat(x) { + return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type()) + } + v.SetFloat(x) + case reflect.Ptr: + x, ok := pValue.(*Key) + if !ok && pValue != nil { + return typeMismatchReason(p, v) + } + if _, ok := v.Interface().(*Key); !ok { + return typeMismatchReason(p, v) + } + v.Set(reflect.ValueOf(x)) + case reflect.Struct: + switch v.Type() { + case typeOfTime: + x, ok := pValue.(time.Time) + if !ok && pValue != nil { + return typeMismatchReason(p, v) + } + v.Set(reflect.ValueOf(x)) + default: + return typeMismatchReason(p, v) + } + case reflect.Slice: + x, ok := pValue.([]byte) + if !ok && pValue != nil { + return typeMismatchReason(p, v) + } + if v.Type().Elem().Kind() != reflect.Uint8 { + return typeMismatchReason(p, v) + } + v.SetBytes(x) + default: + return typeMismatchReason(p, v) + } + if slice.IsValid() { + slice.Set(reflect.Append(slice, v)) + } + return "" +} + +// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer. +func loadEntity(dst interface{}, src *pb.Entity) (err error) { + props := protoToProperties(src) + if e, ok := dst.(PropertyLoadSaver); ok { + return e.Load(props) + } + return LoadStruct(dst, props) +} + +func (s structPLS) Load(props []Property) error { + var fieldName, reason string + var l propertyLoader + + prev := make(map[string]struct{}) + for _, p := range props { + if errStr := l.load(s.codec, s.v, p, prev); errStr != "" { + // We don't return early, as we try to load as many properties as possible. + // It is valid to load an entity into a struct that cannot fully represent it. + // That case returns an error, but the caller is free to ignore it. + fieldName, reason = p.Name, errStr + } + } + if reason != "" { + return &ErrFieldMismatch{ + StructType: s.v.Type(), + FieldName: fieldName, + Reason: reason, + } + } + return nil +} + +func protoToProperties(src *pb.Entity) []Property { + props := src.Property + out := make([]Property, 0, len(props)) + for { + var ( + x *pb.Property + noIndex bool + ) + if len(props) > 0 { + x, props = props[0], props[1:] + noIndex = !x.GetValue().GetIndexed() + } else { + break + } + + if x.Value.ListValue == nil { + out = append(out, Property{ + Name: x.GetName(), + Value: propValue(x.Value), + NoIndex: noIndex, + Multiple: false, + }) + } else { + for _, v := range x.Value.ListValue { + out = append(out, Property{ + Name: x.GetName(), + Value: propValue(v), + NoIndex: noIndex, + Multiple: true, + }) + } + } + } + return out +} + +// propValue returns a Go value that combines the raw PropertyValue with a +// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time. +func propValue(v *pb.Value) interface{} { + //TODO(PSG-Luna): Support EntityValue + //TODO(PSG-Luna): GeoPoint seems gone from the v1 proto, reimplement it once it's readded + switch { + case v.IntegerValue != nil: + return *v.IntegerValue + case v.TimestampMicrosecondsValue != nil: + return fromUnixMicro(*v.TimestampMicrosecondsValue) + case v.BooleanValue != nil: + return *v.BooleanValue + case v.StringValue != nil: + return *v.StringValue + case v.BlobValue != nil: + return []byte(v.BlobValue) + case v.BlobKeyValue != nil: + return *v.BlobKeyValue + case v.DoubleValue != nil: + return *v.DoubleValue + case v.KeyValue != nil: + return protoToKey(v.KeyValue) + } + return nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/prop.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/prop.go new file mode 100644 index 0000000000..43b6c22937 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/prop.go @@ -0,0 +1,300 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "fmt" + "reflect" + "strings" + "sync" + "unicode" +) + +// Entities with more than this many indexed properties will not be saved. +const maxIndexedProperties = 5000 + +// []byte fields more than 1 megabyte long will not be loaded or saved. +const maxBlobLen = 1 << 20 + +// Property is a name/value pair plus some metadata. A datastore entity's +// contents are loaded and saved as a sequence of Properties. An entity can +// have multiple Properties with the same name, provided that p.Multiple is +// true on all of that entity's Properties with that name. +type Property struct { + // Name is the property name. + Name string + // Value is the property value. The valid types are: + // - int64 + // - bool + // - string + // - float64 + // - *Key + // - time.Time + // - []byte (up to 1 megabyte in length) + // This set is smaller than the set of valid struct field types that the + // datastore can load and save. A Property Value cannot be a slice (apart + // from []byte); use multiple Properties instead. Also, a Value's type + // must be explicitly on the list above; it is not sufficient for the + // underlying type to be on that list. For example, a Value of "type + // myInt64 int64" is invalid. Smaller-width integers and floats are also + // invalid. Again, this is more restrictive than the set of valid struct + // field types. + // + // A Value will have an opaque type when loading entities from an index, + // such as via a projection query. Load entities into a struct instead + // of a PropertyLoadSaver when using a projection query. + // + // A Value may also be the nil interface value; this is equivalent to + // Python's None but not directly representable by a Go struct. Loading + // a nil-valued property into a struct will set that field to the zero + // value. + Value interface{} + // NoIndex is whether the datastore cannot index this property. + // If NoIndex is set to false, []byte values are limited to 1500 bytes and + // string values are limited to 1500 bytes. + NoIndex bool + // Multiple is whether the entity can have multiple properties with + // the same name. Even if a particular instance only has one property with + // a certain name, Multiple should be true if a struct would best represent + // it as a field of type []T instead of type T. + Multiple bool +} + +// PropertyLoadSaver can be converted from and to a slice of Properties. +type PropertyLoadSaver interface { + Load([]Property) error + Save() ([]Property, error) +} + +// PropertyList converts a []Property to implement PropertyLoadSaver. +type PropertyList []Property + +var ( + typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem() + typeOfPropertyList = reflect.TypeOf(PropertyList(nil)) +) + +// Load loads all of the provided properties into l. +// It does not first reset *l to an empty slice. +func (l *PropertyList) Load(p []Property) error { + *l = append(*l, p...) + return nil +} + +// Save saves all of l's properties as a slice of Properties. +func (l *PropertyList) Save() ([]Property, error) { + return *l, nil +} + +// validPropertyName returns whether name consists of one or more valid Go +// identifiers joined by ".". +func validPropertyName(name string) bool { + if name == "" { + return false + } + for _, s := range strings.Split(name, ".") { + if s == "" { + return false + } + first := true + for _, c := range s { + if first { + first = false + if c != '_' && !unicode.IsLetter(c) { + return false + } + } else { + if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + } + } + return true +} + +// structTag is the parsed `datastore:"name,options"` tag of a struct field. +// If a field has no tag, or the tag has an empty name, then the structTag's +// name is just the field name. A "-" name means that the datastore ignores +// that field. +type structTag struct { + name string + noIndex bool +} + +// structCodec describes how to convert a struct to and from a sequence of +// properties. +type structCodec struct { + // byIndex gives the structTag for the i'th field. + byIndex []structTag + // byName gives the field codec for the structTag with the given name. + byName map[string]fieldCodec + // hasSlice is whether a struct or any of its nested or embedded structs + // has a slice-typed field (other than []byte). + hasSlice bool + // complete is whether the structCodec is complete. An incomplete + // structCodec may be encountered when walking a recursive struct. + complete bool +} + +// fieldCodec is a struct field's index and, if that struct field's type is +// itself a struct, that substruct's structCodec. +type fieldCodec struct { + index int + substructCodec *structCodec +} + +// structCodecs collects the structCodecs that have already been calculated. +var ( + structCodecsMutex sync.Mutex + structCodecs = make(map[reflect.Type]*structCodec) +) + +// getStructCodec returns the structCodec for the given struct type. +func getStructCodec(t reflect.Type) (*structCodec, error) { + structCodecsMutex.Lock() + defer structCodecsMutex.Unlock() + return getStructCodecLocked(t) +} + +// getStructCodecLocked implements getStructCodec. The structCodecsMutex must +// be held when calling this function. +func getStructCodecLocked(t reflect.Type) (ret *structCodec, retErr error) { + c, ok := structCodecs[t] + if ok { + return c, nil + } + c = &structCodec{ + byIndex: make([]structTag, t.NumField()), + byName: make(map[string]fieldCodec), + } + + // Add c to the structCodecs map before we are sure it is good. If t is + // a recursive type, it needs to find the incomplete entry for itself in + // the map. + structCodecs[t] = c + defer func() { + if retErr != nil { + delete(structCodecs, t) + } + }() + + for i := range c.byIndex { + f := t.Field(i) + name, opts := f.Tag.Get("datastore"), "" + if i := strings.Index(name, ","); i != -1 { + name, opts = name[:i], name[i+1:] + } + if name == "" { + if !f.Anonymous { + name = f.Name + } + } else if name == "-" { + c.byIndex[i] = structTag{name: name} + continue + } else if !validPropertyName(name) { + return nil, fmt.Errorf("datastore: struct tag has invalid property name: %q", name) + } + + substructType, fIsSlice := reflect.Type(nil), false + switch f.Type.Kind() { + case reflect.Struct: + substructType = f.Type + case reflect.Slice: + if f.Type.Elem().Kind() == reflect.Struct { + substructType = f.Type.Elem() + } + fIsSlice = f.Type != typeOfByteSlice + c.hasSlice = c.hasSlice || fIsSlice + } + + if substructType != nil && substructType != typeOfTime { + if name != "" { + name = name + "." + } + sub, err := getStructCodecLocked(substructType) + if err != nil { + return nil, err + } + if !sub.complete { + return nil, fmt.Errorf("datastore: recursive struct: field %q", f.Name) + } + if fIsSlice && sub.hasSlice { + return nil, fmt.Errorf( + "datastore: flattening nested structs leads to a slice of slices: field %q", f.Name) + } + c.hasSlice = c.hasSlice || sub.hasSlice + for relName := range sub.byName { + absName := name + relName + if _, ok := c.byName[absName]; ok { + return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", absName) + } + c.byName[absName] = fieldCodec{index: i, substructCodec: sub} + } + } else { + if _, ok := c.byName[name]; ok { + return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", name) + } + c.byName[name] = fieldCodec{index: i} + } + + c.byIndex[i] = structTag{ + name: name, + noIndex: opts == "noindex", + } + } + c.complete = true + return c, nil +} + +// structPLS adapts a struct to be a PropertyLoadSaver. +type structPLS struct { + v reflect.Value + codec *structCodec +} + +// newStructPLS returns a PropertyLoadSaver for the struct pointer p. +func newStructPLS(p interface{}) (PropertyLoadSaver, error) { + v := reflect.ValueOf(p) + if v.Kind() != reflect.Ptr || v.IsNil() || v.Elem().Kind() != reflect.Struct { + return nil, ErrInvalidEntityType + } + v = v.Elem() + codec, err := getStructCodec(v.Type()) + if err != nil { + return nil, err + } + return structPLS{v, codec}, nil +} + +// LoadStruct loads the properties from p to dst. +// dst must be a struct pointer. +func LoadStruct(dst interface{}, p []Property) error { + x, err := newStructPLS(dst) + if err != nil { + return err + } + return x.Load(p) +} + +// SaveStruct returns the properties from src as a slice of Properties. +// src must be a struct pointer. +func SaveStruct(src interface{}) ([]Property, error) { + x, err := newStructPLS(src) + if err != nil { + return nil, err + } + return x.Save() +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/query.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/query.go new file mode 100644 index 0000000000..3eb8a8500d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/query.go @@ -0,0 +1,755 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "encoding/base64" + "errors" + "fmt" + "math" + "reflect" + "strconv" + "strings" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + pb "google.golang.org/cloud/internal/datastore" +) + +type operator int + +const ( + lessThan operator = iota + lessEq + equal + greaterEq + greaterThan + + keyFieldName = "__key__" +) + +var operatorToProto = map[operator]*pb.PropertyFilter_Operator{ + lessThan: pb.PropertyFilter_LESS_THAN.Enum(), + lessEq: pb.PropertyFilter_LESS_THAN_OR_EQUAL.Enum(), + equal: pb.PropertyFilter_EQUAL.Enum(), + greaterEq: pb.PropertyFilter_GREATER_THAN_OR_EQUAL.Enum(), + greaterThan: pb.PropertyFilter_GREATER_THAN.Enum(), +} + +// filter is a conditional filter on query results. +type filter struct { + FieldName string + Op operator + Value interface{} +} + +type sortDirection int + +const ( + ascending sortDirection = iota + descending +) + +var sortDirectionToProto = map[sortDirection]*pb.PropertyOrder_Direction{ + ascending: pb.PropertyOrder_ASCENDING.Enum(), + descending: pb.PropertyOrder_DESCENDING.Enum(), +} + +// order is a sort order on query results. +type order struct { + FieldName string + Direction sortDirection +} + +// NewQuery creates a new Query for a specific entity kind. +// +// An empty kind means to return all entities, including entities created and +// managed by other App Engine features, and is called a kindless query. +// Kindless queries cannot include filters or sort orders on property values. +func NewQuery(kind string) *Query { + return &Query{ + kind: kind, + limit: -1, + } +} + +// Query represents a datastore query. +type Query struct { + kind string + ancestor *Key + filter []filter + order []order + projection []string + + distinct bool + keysOnly bool + eventual bool + limit int32 + offset int32 + start []byte + end []byte + + trans *Transaction + + err error +} + +func (q *Query) clone() *Query { + x := *q + // Copy the contents of the slice-typed fields to a new backing store. + if len(q.filter) > 0 { + x.filter = make([]filter, len(q.filter)) + copy(x.filter, q.filter) + } + if len(q.order) > 0 { + x.order = make([]order, len(q.order)) + copy(x.order, q.order) + } + return &x +} + +// Ancestor returns a derivative query with an ancestor filter. +// The ancestor should not be nil. +func (q *Query) Ancestor(ancestor *Key) *Query { + q = q.clone() + if ancestor == nil { + q.err = errors.New("datastore: nil query ancestor") + return q + } + q.ancestor = ancestor + return q +} + +// EventualConsistency returns a derivative query that returns eventually +// consistent results. +// It only has an effect on ancestor queries. +func (q *Query) EventualConsistency() *Query { + q = q.clone() + q.eventual = true + return q +} + +// Transaction returns a derivative query that is associated with the given +// transaction. +// +// All reads performed as part of the transaction will come from a single +// consistent snapshot. Furthermore, if the transaction is set to a +// serializable isolation level, another transaction cannot concurrently modify +// the data that is read or modified by this transaction. +func (q *Query) Transaction(t *Transaction) *Query { + q = q.clone() + q.trans = t + return q +} + +// Filter returns a derivative query with a field-based filter. +// The filterStr argument must be a field name followed by optional space, +// followed by an operator, one of ">", "<", ">=", "<=", or "=". +// Fields are compared against the provided value using the operator. +// Multiple filters are AND'ed together. +// Field names which contain spaces, quote marks, or operator characters +// should be passed as quoted Go string literals as returned by strconv.Quote +// or the fmt package's %q verb. +func (q *Query) Filter(filterStr string, value interface{}) *Query { + q = q.clone() + filterStr = strings.TrimSpace(filterStr) + if filterStr == "" { + q.err = fmt.Errorf("datastore: invalid filter %q", filterStr) + return q + } + f := filter{ + FieldName: strings.TrimRight(filterStr, " ><=!"), + Value: value, + } + switch op := strings.TrimSpace(filterStr[len(f.FieldName):]); op { + case "<=": + f.Op = lessEq + case ">=": + f.Op = greaterEq + case "<": + f.Op = lessThan + case ">": + f.Op = greaterThan + case "=": + f.Op = equal + default: + q.err = fmt.Errorf("datastore: invalid operator %q in filter %q", op, filterStr) + return q + } + var err error + f.FieldName, err = unquote(f.FieldName) + if err != nil { + q.err = fmt.Errorf("datastore: invalid syntax for quoted field name %q", f.FieldName) + return q + } + q.filter = append(q.filter, f) + return q +} + +// Order returns a derivative query with a field-based sort order. Orders are +// applied in the order they are added. The default order is ascending; to sort +// in descending order prefix the fieldName with a minus sign (-). +// Field names which contain spaces, quote marks, or the minus sign +// should be passed as quoted Go string literals as returned by strconv.Quote +// or the fmt package's %q verb. +func (q *Query) Order(fieldName string) *Query { + q = q.clone() + fieldName, dir := strings.TrimSpace(fieldName), ascending + if strings.HasPrefix(fieldName, "-") { + fieldName, dir = strings.TrimSpace(fieldName[1:]), descending + } else if strings.HasPrefix(fieldName, "+") { + q.err = fmt.Errorf("datastore: invalid order: %q", fieldName) + return q + } + fieldName, err := unquote(fieldName) + if err != nil { + q.err = fmt.Errorf("datastore: invalid syntax for quoted field name %q", fieldName) + return q + } + if fieldName == "" { + q.err = errors.New("datastore: empty order") + return q + } + q.order = append(q.order, order{ + Direction: dir, + FieldName: fieldName, + }) + return q +} + +// unquote optionally interprets s as a double-quoted or backquoted Go +// string literal if it begins with the relevant character. +func unquote(s string) (string, error) { + if s == "" || (s[0] != '`' && s[0] != '"') { + return s, nil + } + return strconv.Unquote(s) +} + +// Project returns a derivative query that yields only the given fields. It +// cannot be used with KeysOnly. +func (q *Query) Project(fieldNames ...string) *Query { + q = q.clone() + q.projection = append([]string(nil), fieldNames...) + return q +} + +// Distinct returns a derivative query that yields de-duplicated entities with +// respect to the set of projected fields. It is only used for projection +// queries. +func (q *Query) Distinct() *Query { + q = q.clone() + q.distinct = true + return q +} + +// KeysOnly returns a derivative query that yields only keys, not keys and +// entities. It cannot be used with projection queries. +func (q *Query) KeysOnly() *Query { + q = q.clone() + q.keysOnly = true + return q +} + +// Limit returns a derivative query that has a limit on the number of results +// returned. A negative value means unlimited. +func (q *Query) Limit(limit int) *Query { + q = q.clone() + if limit < math.MinInt32 || limit > math.MaxInt32 { + q.err = errors.New("datastore: query limit overflow") + return q + } + q.limit = int32(limit) + return q +} + +// Offset returns a derivative query that has an offset of how many keys to +// skip over before returning results. A negative value is invalid. +func (q *Query) Offset(offset int) *Query { + q = q.clone() + if offset < 0 { + q.err = errors.New("datastore: negative query offset") + return q + } + if offset > math.MaxInt32 { + q.err = errors.New("datastore: query offset overflow") + return q + } + q.offset = int32(offset) + return q +} + +// Start returns a derivative query with the given start point. +func (q *Query) Start(c Cursor) *Query { + q = q.clone() + if c.cc == nil { + q.err = errors.New("datastore: invalid cursor") + return q + } + q.start = c.cc + return q +} + +// End returns a derivative query with the given end point. +func (q *Query) End(c Cursor) *Query { + q = q.clone() + if c.cc == nil { + q.err = errors.New("datastore: invalid cursor") + return q + } + q.end = c.cc + return q +} + +// toProto converts the query to a protocol buffer. +func (q *Query) toProto(req *pb.RunQueryRequest) error { + dst := pb.Query{} + if len(q.projection) != 0 && q.keysOnly { + return errors.New("datastore: query cannot both project and be keys-only") + } + dst.Reset() + if q.kind != "" { + dst.Kind = []*pb.KindExpression{&pb.KindExpression{Name: proto.String(q.kind)}} + } + if q.projection != nil { + for _, propertyName := range q.projection { + dst.Projection = append(dst.Projection, &pb.PropertyExpression{Property: &pb.PropertyReference{Name: proto.String(propertyName)}}) + } + + if q.distinct { + for _, propertyName := range q.projection { + dst.GroupBy = append(dst.GroupBy, &pb.PropertyReference{Name: proto.String(propertyName)}) + } + } + } + if q.keysOnly { + dst.Projection = []*pb.PropertyExpression{&pb.PropertyExpression{Property: &pb.PropertyReference{Name: proto.String(keyFieldName)}}} + } + + var filters []*pb.Filter + for _, qf := range q.filter { + if qf.FieldName == "" { + return errors.New("datastore: empty query filter field name") + } + v, errStr := interfaceToProto(reflect.ValueOf(qf.Value).Interface()) + if errStr != "" { + return errors.New("datastore: bad query filter value type: " + errStr) + } + xf := &pb.PropertyFilter{ + Operator: operatorToProto[qf.Op], + Property: &pb.PropertyReference{Name: proto.String(qf.FieldName)}, + Value: v, + } + if xf.Operator == nil { + return errors.New("datastore: unknown query filter operator") + } + filters = append(filters, &pb.Filter{PropertyFilter: xf}) + } + + if q.ancestor != nil { + filters = append(filters, &pb.Filter{ + PropertyFilter: &pb.PropertyFilter{ + Property: &pb.PropertyReference{Name: proto.String("__key__")}, + Operator: pb.PropertyFilter_HAS_ANCESTOR.Enum(), + Value: &pb.Value{KeyValue: keyToProto(q.ancestor)}, + }}) + } + + if len(filters) == 1 { + dst.Filter = filters[0] + } else if len(filters) > 1 { + dst.Filter = &pb.Filter{CompositeFilter: &pb.CompositeFilter{ + Operator: pb.CompositeFilter_AND.Enum(), + Filter: filters, + }} + } + + for _, qo := range q.order { + if qo.FieldName == "" { + return errors.New("datastore: empty query order field name") + } + xo := &pb.PropertyOrder{ + Property: &pb.PropertyReference{Name: proto.String(qo.FieldName)}, + Direction: sortDirectionToProto[qo.Direction], + } + if xo.Direction == nil { + return errors.New("datastore: unknown query order direction") + } + dst.Order = append(dst.Order, xo) + } + if q.limit >= 0 { + dst.Limit = proto.Int32(q.limit) + } + if q.offset != 0 { + dst.Offset = proto.Int32(q.offset) + } + dst.StartCursor = q.start + dst.EndCursor = q.end + + if t := q.trans; t != nil { + if t.id == nil { + return errExpiredTransaction + } + req.ReadOptions = &pb.ReadOptions{Transaction: t.id} + } + + req.Query = &dst + return nil +} + +// Count returns the number of results for the given query. +func (c *Client) Count(ctx context.Context, q *Query) (int, error) { + // Check that the query is well-formed. + if q.err != nil { + return 0, q.err + } + + // Run a copy of the query, with keysOnly true (if we're not a projection, + // since the two are incompatible). + newQ := q.clone() + newQ.keysOnly = len(newQ.projection) == 0 + req := &pb.RunQueryRequest{} + + if ns := ctxNamespace(ctx); ns != "" { + req.PartitionId = &pb.PartitionId{ + Namespace: proto.String(ns), + } + } + if err := newQ.toProto(req); err != nil { + return 0, err + } + res := &pb.RunQueryResponse{} + if err := c.call(ctx, "runQuery", req, res); err != nil { + return 0, err + } + var n int + b := res.Batch + for { + n += len(b.GetEntityResult()) + if b.GetMoreResults() != pb.QueryResultBatch_NOT_FINISHED { + break + } + var err error + // TODO(jbd): Support count queries that have a limit and an offset. + if err = callNext(ctx, c, req, res, 0, 0); err != nil { + return 0, err + } + } + return int(n), nil +} + +func callNext(ctx context.Context, client *Client, req *pb.RunQueryRequest, res *pb.RunQueryResponse, offset, limit int32) error { + if res.GetBatch().EndCursor == nil { + return errors.New("datastore: internal error: server did not return a cursor") + } + req.Query.StartCursor = res.GetBatch().GetEndCursor() + if limit >= 0 { + req.Query.Limit = proto.Int32(limit) + } + if offset != 0 { + req.Query.Offset = proto.Int32(offset) + } + res.Reset() + return client.call(ctx, "runQuery", req, res) +} + +// GetAll runs the provided query in the given context and returns all keys +// that match that query, as well as appending the values to dst. +// +// dst must have type *[]S or *[]*S or *[]P, for some struct type S or some non- +// interface, non-pointer type P such that P or *P implements PropertyLoadSaver. +// +// As a special case, *PropertyList is an invalid type for dst, even though a +// PropertyList is a slice of structs. It is treated as invalid to avoid being +// mistakenly passed when *[]PropertyList was intended. +// +// The keys returned by GetAll will be in a 1-1 correspondence with the entities +// added to dst. +// +// If q is a ``keys-only'' query, GetAll ignores dst and only returns the keys. +func (c *Client) GetAll(ctx context.Context, q *Query, dst interface{}) ([]*Key, error) { + var ( + dv reflect.Value + mat multiArgType + elemType reflect.Type + errFieldMismatch error + ) + if !q.keysOnly { + dv = reflect.ValueOf(dst) + if dv.Kind() != reflect.Ptr || dv.IsNil() { + return nil, ErrInvalidEntityType + } + dv = dv.Elem() + mat, elemType = checkMultiArg(dv) + if mat == multiArgTypeInvalid || mat == multiArgTypeInterface { + return nil, ErrInvalidEntityType + } + } + + var keys []*Key + for t := c.Run(ctx, q); ; { + k, e, err := t.next() + if err == Done { + break + } + if err != nil { + return keys, err + } + if !q.keysOnly { + ev := reflect.New(elemType) + if elemType.Kind() == reflect.Map { + // This is a special case. The zero values of a map type are + // not immediately useful; they have to be make'd. + // + // Funcs and channels are similar, in that a zero value is not useful, + // but even a freshly make'd channel isn't useful: there's no fixed + // channel buffer size that is always going to be large enough, and + // there's no goroutine to drain the other end. Theoretically, these + // types could be supported, for example by sniffing for a constructor + // method or requiring prior registration, but for now it's not a + // frequent enough concern to be worth it. Programmers can work around + // it by explicitly using Iterator.Next instead of the Query.GetAll + // convenience method. + x := reflect.MakeMap(elemType) + ev.Elem().Set(x) + } + if err = loadEntity(ev.Interface(), e); err != nil { + if _, ok := err.(*ErrFieldMismatch); ok { + // We continue loading entities even in the face of field mismatch errors. + // If we encounter any other error, that other error is returned. Otherwise, + // an ErrFieldMismatch is returned. + errFieldMismatch = err + } else { + return keys, err + } + } + if mat != multiArgTypeStructPtr { + ev = ev.Elem() + } + dv.Set(reflect.Append(dv, ev)) + } + keys = append(keys, k) + } + return keys, errFieldMismatch +} + +// Run runs the given query in the given context. +func (c *Client) Run(ctx context.Context, q *Query) *Iterator { + if q.err != nil { + return &Iterator{err: q.err} + } + t := &Iterator{ + ctx: ctx, + client: c, + limit: q.limit, + q: q, + prevCC: q.start, + } + t.req.Reset() + if ns := ctxNamespace(ctx); ns != "" { + t.req.PartitionId = &pb.PartitionId{ + Namespace: proto.String(ns), + } + } + if err := q.toProto(&t.req); err != nil { + t.err = err + return t + } + if err := c.call(ctx, "runQuery", &t.req, &t.res); err != nil { + t.err = err + return t + } + b := t.res.GetBatch() + offset := q.offset - b.GetSkippedResults() + for offset > 0 && b.GetMoreResults() == pb.QueryResultBatch_NOT_FINISHED { + t.prevCC = b.GetEndCursor() + var err error + if err = callNext(t.ctx, c, &t.req, &t.res, offset, t.limit); err != nil { + t.err = err + break + } + skip := b.GetSkippedResults() + if skip < 0 { + t.err = errors.New("datastore: internal error: negative number of skipped_results") + break + } + offset -= skip + } + if offset < 0 { + t.err = errors.New("datastore: internal error: query offset was overshot") + } + return t +} + +// Iterator is the result of running a query. +type Iterator struct { + ctx context.Context + client *Client + err error + // req is the request we sent previously, we need to keep track of it to resend it + req pb.RunQueryRequest + // res is the result of the most recent RunQuery or Next API call. + res pb.RunQueryResponse + // i is how many elements of res.Result we have iterated over. + i int + // limit is the limit on the number of results this iterator should return. + // A negative value means unlimited. + limit int32 + // q is the original query which yielded this iterator. + q *Query + // prevCC is the compiled cursor that marks the end of the previous batch + // of results. + prevCC []byte +} + +// Done is returned when a query iteration has completed. +var Done = errors.New("datastore: query has no more results") + +// Next returns the key of the next result. When there are no more results, +// Done is returned as the error. +// +// If the query is not keys only and dst is non-nil, it also loads the entity +// stored for that key into the struct pointer or PropertyLoadSaver dst, with +// the same semantics and possible errors as for the Get function. +func (t *Iterator) Next(dst interface{}) (*Key, error) { + k, e, err := t.next() + if err != nil { + return nil, err + } + if dst != nil && !t.q.keysOnly { + err = loadEntity(dst, e) + } + return k, err +} + +func (t *Iterator) next() (*Key, *pb.Entity, error) { + if t.err != nil { + return nil, nil, t.err + } + + // Issue datastore_v3/Next RPCs as necessary. + b := t.res.GetBatch() + for t.i == len(b.EntityResult) { + if b.GetMoreResults() != pb.QueryResultBatch_NOT_FINISHED { + t.err = Done + return nil, nil, t.err + } + t.prevCC = b.GetEndCursor() + if err := callNext(t.ctx, t.client, &t.req, &t.res, 0, t.limit); err != nil { + t.err = err + return nil, nil, t.err + } + if b.GetSkippedResults() != 0 { + t.err = errors.New("datastore: internal error: iterator has skipped results") + return nil, nil, t.err + } + t.i = 0 + if t.limit >= 0 { + t.limit -= int32(len(b.EntityResult)) + if t.limit < 0 { + t.err = errors.New("datastore: internal error: query returned more results than the limit") + return nil, nil, t.err + } + } + } + + // Extract the key from the t.i'th element of t.res.Result. + e := b.EntityResult[t.i] + t.i++ + if e.Entity.Key == nil { + return nil, nil, errors.New("datastore: internal error: server did not return a key") + } + k := protoToKey(e.Entity.Key) + if k.Incomplete() { + return nil, nil, errors.New("datastore: internal error: server returned an invalid key") + } + return k, e.Entity, nil +} + +// Cursor returns a cursor for the iterator's current location. +func (t *Iterator) Cursor() (Cursor, error) { + if t.err != nil && t.err != Done { + return Cursor{}, t.err + } + // If we are at either end of the current batch of results, + // return the compiled cursor at that end. + b := t.res.Batch + skipped := b.GetSkippedResults() + if t.i == 0 && skipped == 0 { + if t.prevCC == nil { + // A nil pointer (of type *pb.CompiledCursor) means no constraint: + // passing it as the end cursor of a new query means unlimited results + // (glossing over the integer limit parameter for now). + // A non-nil pointer to an empty pb.CompiledCursor means the start: + // passing it as the end cursor of a new query means 0 results. + // If prevCC was nil, then the original query had no start cursor, but + // Iterator.Cursor should return "the start" instead of unlimited. + return Cursor{}, nil + } + return Cursor{t.prevCC}, nil + } + if t.i == len(b.EntityResult) { + return Cursor{b.EndCursor}, nil + } + // Otherwise, re-run the query offset to this iterator's position, starting from + // the most recent compiled cursor. This is done on a best-effort basis, as it + // is racy; if a concurrent process has added or removed entities, then the + // cursor returned may be inconsistent. + q := t.q.clone() + q.start = t.prevCC + q.offset = skipped + int32(t.i) + q.limit = 0 + q.keysOnly = len(q.projection) == 0 + t1 := t.client.Run(t.ctx, t.q) + _, _, err := t1.next() + if err != Done { + if err == nil { + err = fmt.Errorf("datastore: internal error: zero-limit query did not have zero results") + } + return Cursor{}, err + } + return Cursor{t1.res.Batch.EndCursor}, nil +} + +// Cursor is an iterator's position. It can be converted to and from an opaque +// string. A cursor can be used from different HTTP requests, but only with a +// query with the same kind, ancestor, filter and order constraints. +type Cursor struct { + cc []byte +} + +// String returns a base-64 string representation of a cursor. +func (c Cursor) String() string { + if c.cc == nil { + return "" + } + + return strings.TrimRight(base64.URLEncoding.EncodeToString(c.cc), "=") +} + +// Decode decodes a cursor from its base-64 string representation. +func DecodeCursor(s string) (Cursor, error) { + if s == "" { + return Cursor{}, nil + } + if n := len(s) % 4; n != 0 { + s += strings.Repeat("=", 4-n) + } + b, err := base64.URLEncoding.DecodeString(s) + if err != nil { + return Cursor{}, err + } + return Cursor{b}, nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/query_test.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/query_test.go new file mode 100644 index 0000000000..5d0caaff5d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/query_test.go @@ -0,0 +1,482 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "errors" + "fmt" + "reflect" + "testing" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + pb "google.golang.org/cloud/internal/datastore" +) + +var ( + key1 = &pb.Key{ + PathElement: []*pb.Key_PathElement{ + { + Kind: proto.String("Gopher"), + Id: proto.Int64(6), + }, + }, + } + key2 = &pb.Key{ + PathElement: []*pb.Key_PathElement{ + { + Kind: proto.String("Gopher"), + Id: proto.Int64(6), + }, + { + Kind: proto.String("Gopher"), + Id: proto.Int64(8), + }, + }, + } +) + +type fakeClient func(req, resp proto.Message) (err error) + +func (c fakeClient) Call(ctx context.Context, method string, req, resp proto.Message) error { + return c(req, resp) +} + +func fakeRunQuery(in *pb.RunQueryRequest, out *pb.RunQueryResponse) error { + expectedIn := &pb.RunQueryRequest{ + Query: &pb.Query{ + Kind: []*pb.KindExpression{&pb.KindExpression{Name: proto.String("Gopher")}}, + }, + } + if !proto.Equal(in, expectedIn) { + return fmt.Errorf("unsupported argument: got %v want %v", in, expectedIn) + } + *out = pb.RunQueryResponse{ + Batch: &pb.QueryResultBatch{ + MoreResults: pb.QueryResultBatch_NO_MORE_RESULTS.Enum(), + EntityResultType: pb.EntityResult_FULL.Enum(), + EntityResult: []*pb.EntityResult{ + &pb.EntityResult{ + Entity: &pb.Entity{ + Key: key1, + Property: []*pb.Property{ + { + Name: proto.String("Name"), + Value: &pb.Value{StringValue: proto.String("George")}, + }, + { + Name: proto.String("Height"), + Value: &pb.Value{ + IntegerValue: proto.Int64(32), + }, + }, + }, + }, + }, + &pb.EntityResult{ + Entity: &pb.Entity{ + Key: key2, + Property: []*pb.Property{ + { + Name: proto.String("Name"), + Value: &pb.Value{StringValue: proto.String("Rufus")}, + }, + // No height for Rufus. + }, + }, + }, + }, + }, + } + return nil +} + +type StructThatImplementsPLS struct{} + +func (StructThatImplementsPLS) Load(p []Property) error { return nil } +func (StructThatImplementsPLS) Save() ([]Property, error) { return nil, nil } + +var _ PropertyLoadSaver = StructThatImplementsPLS{} + +type StructPtrThatImplementsPLS struct{} + +func (*StructPtrThatImplementsPLS) Load(p []Property) error { return nil } +func (*StructPtrThatImplementsPLS) Save() ([]Property, error) { return nil, nil } + +var _ PropertyLoadSaver = &StructPtrThatImplementsPLS{} + +type PropertyMap map[string]Property + +func (m PropertyMap) Load(props []Property) error { + for _, p := range props { + m[p.Name] = p + } + return nil +} + +func (m PropertyMap) Save() ([]Property, error) { + props := make([]Property, 0, len(m)) + for _, p := range m { + props = append(props, p) + } + return props, nil +} + +var _ PropertyLoadSaver = PropertyMap{} + +type Gopher struct { + Name string + Height int +} + +// typeOfEmptyInterface is the type of interface{}, but we can't use +// reflect.TypeOf((interface{})(nil)) directly because TypeOf takes an +// interface{}. +var typeOfEmptyInterface = reflect.TypeOf((*interface{})(nil)).Elem() + +func TestCheckMultiArg(t *testing.T) { + testCases := []struct { + v interface{} + mat multiArgType + elemType reflect.Type + }{ + // Invalid cases. + {nil, multiArgTypeInvalid, nil}, + {Gopher{}, multiArgTypeInvalid, nil}, + {&Gopher{}, multiArgTypeInvalid, nil}, + {PropertyList{}, multiArgTypeInvalid, nil}, // This is a special case. + {PropertyMap{}, multiArgTypeInvalid, nil}, + {[]*PropertyList(nil), multiArgTypeInvalid, nil}, + {[]*PropertyMap(nil), multiArgTypeInvalid, nil}, + {[]**Gopher(nil), multiArgTypeInvalid, nil}, + {[]*interface{}(nil), multiArgTypeInvalid, nil}, + // Valid cases. + { + []PropertyList(nil), + multiArgTypePropertyLoadSaver, + reflect.TypeOf(PropertyList{}), + }, + { + []PropertyMap(nil), + multiArgTypePropertyLoadSaver, + reflect.TypeOf(PropertyMap{}), + }, + { + []StructThatImplementsPLS(nil), + multiArgTypePropertyLoadSaver, + reflect.TypeOf(StructThatImplementsPLS{}), + }, + { + []StructPtrThatImplementsPLS(nil), + multiArgTypePropertyLoadSaver, + reflect.TypeOf(StructPtrThatImplementsPLS{}), + }, + { + []Gopher(nil), + multiArgTypeStruct, + reflect.TypeOf(Gopher{}), + }, + { + []*Gopher(nil), + multiArgTypeStructPtr, + reflect.TypeOf(Gopher{}), + }, + { + []interface{}(nil), + multiArgTypeInterface, + typeOfEmptyInterface, + }, + } + for _, tc := range testCases { + mat, elemType := checkMultiArg(reflect.ValueOf(tc.v)) + if mat != tc.mat || elemType != tc.elemType { + t.Errorf("checkMultiArg(%T): got %v, %v want %v, %v", + tc.v, mat, elemType, tc.mat, tc.elemType) + } + } +} + +func TestSimpleQuery(t *testing.T) { + struct1 := Gopher{Name: "George", Height: 32} + struct2 := Gopher{Name: "Rufus"} + pList1 := PropertyList{ + { + Name: "Name", + Value: "George", + }, + { + Name: "Height", + Value: int64(32), + }, + } + pList2 := PropertyList{ + { + Name: "Name", + Value: "Rufus", + }, + } + pMap1 := PropertyMap{ + "Name": Property{ + Name: "Name", + Value: "George", + }, + "Height": Property{ + Name: "Height", + Value: int64(32), + }, + } + pMap2 := PropertyMap{ + "Name": Property{ + Name: "Name", + Value: "Rufus", + }, + } + + testCases := []struct { + dst interface{} + want interface{} + }{ + // The destination must have type *[]P, *[]S or *[]*S, for some non-interface + // type P such that *P implements PropertyLoadSaver, or for some struct type S. + {new([]Gopher), &[]Gopher{struct1, struct2}}, + {new([]*Gopher), &[]*Gopher{&struct1, &struct2}}, + {new([]PropertyList), &[]PropertyList{pList1, pList2}}, + {new([]PropertyMap), &[]PropertyMap{pMap1, pMap2}}, + + // Any other destination type is invalid. + {0, nil}, + {Gopher{}, nil}, + {PropertyList{}, nil}, + {PropertyMap{}, nil}, + {[]int{}, nil}, + {[]Gopher{}, nil}, + {[]PropertyList{}, nil}, + {new(int), nil}, + {new(Gopher), nil}, + {new(PropertyList), nil}, // This is a special case. + {new(PropertyMap), nil}, + {new([]int), nil}, + {new([]map[int]int), nil}, + {new([]map[string]Property), nil}, + {new([]map[string]interface{}), nil}, + {new([]*int), nil}, + {new([]*map[int]int), nil}, + {new([]*map[string]Property), nil}, + {new([]*map[string]interface{}), nil}, + {new([]**Gopher), nil}, + {new([]*PropertyList), nil}, + {new([]*PropertyMap), nil}, + } + for _, tc := range testCases { + nCall := 0 + client := &Client{ + client: fakeClient(func(in, out proto.Message) error { + nCall++ + return fakeRunQuery(in.(*pb.RunQueryRequest), out.(*pb.RunQueryResponse)) + }), + } + ctx := context.Background() + + var ( + expectedErr error + expectedNCall int + ) + if tc.want == nil { + expectedErr = ErrInvalidEntityType + } else { + expectedNCall = 1 + } + keys, err := client.GetAll(ctx, NewQuery("Gopher"), tc.dst) + if err != expectedErr { + t.Errorf("dst type %T: got error %v, want %v", tc.dst, err, expectedErr) + continue + } + if nCall != expectedNCall { + t.Errorf("dst type %T: Context.Call was called an incorrect number of times: got %d want %d", tc.dst, nCall, expectedNCall) + continue + } + if err != nil { + continue + } + + key1 := NewKey(ctx, "Gopher", "", 6, nil) + expectedKeys := []*Key{ + key1, + NewKey(ctx, "Gopher", "", 8, key1), + } + if l1, l2 := len(keys), len(expectedKeys); l1 != l2 { + t.Errorf("dst type %T: got %d keys, want %d keys", tc.dst, l1, l2) + continue + } + for i, key := range keys { + if !keysEqual(key, expectedKeys[i]) { + t.Errorf("dst type %T: got key #%d %v, want %v", tc.dst, i, key, expectedKeys[i]) + continue + } + } + + if !reflect.DeepEqual(tc.dst, tc.want) { + t.Errorf("dst type %T: Entities got %+v, want %+v", tc.dst, tc.dst, tc.want) + continue + } + } +} + +// keysEqual is like (*Key).Equal, but ignores the App ID. +func keysEqual(a, b *Key) bool { + for a != nil && b != nil { + if a.Kind() != b.Kind() || a.Name() != b.Name() || a.ID() != b.ID() { + return false + } + a, b = a.Parent(), b.Parent() + } + return a == b +} + +func TestQueriesAreImmutable(t *testing.T) { + // Test that deriving q2 from q1 does not modify q1. + q0 := NewQuery("foo") + q1 := NewQuery("foo") + q2 := q1.Offset(2) + if !reflect.DeepEqual(q0, q1) { + t.Errorf("q0 and q1 were not equal") + } + if reflect.DeepEqual(q1, q2) { + t.Errorf("q1 and q2 were equal") + } + + // Test that deriving from q4 twice does not conflict, even though + // q4 has a long list of order clauses. This tests that the arrays + // backed by a query's slice of orders are not shared. + f := func() *Query { + q := NewQuery("bar") + // 47 is an ugly number that is unlikely to be near a re-allocation + // point in repeated append calls. For example, it's not near a power + // of 2 or a multiple of 10. + for i := 0; i < 47; i++ { + q = q.Order(fmt.Sprintf("x%d", i)) + } + return q + } + q3 := f().Order("y") + q4 := f() + q5 := q4.Order("y") + q6 := q4.Order("z") + if !reflect.DeepEqual(q3, q5) { + t.Errorf("q3 and q5 were not equal") + } + if reflect.DeepEqual(q5, q6) { + t.Errorf("q5 and q6 were equal") + } +} + +func TestFilterParser(t *testing.T) { + testCases := []struct { + filterStr string + wantOK bool + wantFieldName string + wantOp operator + }{ + // Supported ops. + {"x<", true, "x", lessThan}, + {"x <", true, "x", lessThan}, + {"x <", true, "x", lessThan}, + {" x < ", true, "x", lessThan}, + {"x <=", true, "x", lessEq}, + {"x =", true, "x", equal}, + {"x >=", true, "x", greaterEq}, + {"x >", true, "x", greaterThan}, + {"in >", true, "in", greaterThan}, + {"in>", true, "in", greaterThan}, + // Valid but (currently) unsupported ops. + {"x!=", false, "", 0}, + {"x !=", false, "", 0}, + {" x != ", false, "", 0}, + {"x IN", false, "", 0}, + {"x in", false, "", 0}, + // Invalid ops. + {"x EQ", false, "", 0}, + {"x lt", false, "", 0}, + {"x <>", false, "", 0}, + {"x >>", false, "", 0}, + {"x ==", false, "", 0}, + {"x =<", false, "", 0}, + {"x =>", false, "", 0}, + {"x !", false, "", 0}, + {"x ", false, "", 0}, + {"x", false, "", 0}, + // Quoted and interesting field names. + {"x > y =", true, "x > y", equal}, + {"` x ` =", true, " x ", equal}, + {`" x " =`, true, " x ", equal}, + {`" \"x " =`, true, ` "x `, equal}, + {`" x =`, false, "", 0}, + {`" x ="`, false, "", 0}, + {"` x \" =", false, "", 0}, + } + for _, tc := range testCases { + q := NewQuery("foo").Filter(tc.filterStr, 42) + if ok := q.err == nil; ok != tc.wantOK { + t.Errorf("%q: ok=%t, want %t", tc.filterStr, ok, tc.wantOK) + continue + } + if !tc.wantOK { + continue + } + if len(q.filter) != 1 { + t.Errorf("%q: len=%d, want %d", tc.filterStr, len(q.filter), 1) + continue + } + got, want := q.filter[0], filter{tc.wantFieldName, tc.wantOp, 42} + if got != want { + t.Errorf("%q: got %v, want %v", tc.filterStr, got, want) + continue + } + } +} + +func TestNamespaceQuery(t *testing.T) { + gotNamespace := make(chan string, 1) + ctx := context.Background() + client := &Client{ + client: fakeClient(func(req, resp proto.Message) error { + gotNamespace <- req.(*pb.RunQueryRequest).GetPartitionId().GetNamespace() + return errors.New("not implemented") + }), + } + + var gs []Gopher + + client.GetAll(ctx, NewQuery("gopher"), &gs) + if got, want := <-gotNamespace, ""; got != want { + t.Errorf("GetAll: got namespace %q, want %q", got, want) + } + client.Count(ctx, NewQuery("gopher")) + if got, want := <-gotNamespace, ""; got != want { + t.Errorf("Count: got namespace %q, want %q", got, want) + } + + const ns = "not_default" + ctx = WithNamespace(ctx, ns) + + client.GetAll(ctx, NewQuery("gopher"), &gs) + if got, want := <-gotNamespace, ns; got != want { + t.Errorf("GetAll: got namespace %q, want %q", got, want) + } + client.Count(ctx, NewQuery("gopher")) + if got, want := <-gotNamespace, ns; got != want { + t.Errorf("Count: got namespace %q, want %q", got, want) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/save.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/save.go new file mode 100644 index 0000000000..a208000aa2 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/save.go @@ -0,0 +1,210 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "errors" + "fmt" + "reflect" + "time" + + "github.com/golang/protobuf/proto" + pb "google.golang.org/cloud/internal/datastore" +) + +// saveEntity saves an EntityProto into a PropertyLoadSaver or struct pointer. +func saveEntity(key *Key, src interface{}) (*pb.Entity, error) { + var err error + var props []Property + if e, ok := src.(PropertyLoadSaver); ok { + props, err = e.Save() + } else { + props, err = SaveStruct(src) + } + if err != nil { + return nil, err + } + return propertiesToProto(key, props) +} + +func saveStructProperty(props *[]Property, name string, noIndex, multiple bool, v reflect.Value) error { + p := Property{ + Name: name, + NoIndex: noIndex, + Multiple: multiple, + } + + switch x := v.Interface().(type) { + case *Key, time.Time: + p.Value = x + default: + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + p.Value = v.Int() + case reflect.Bool: + p.Value = v.Bool() + case reflect.String: + p.Value = v.String() + case reflect.Float32, reflect.Float64: + p.Value = v.Float() + case reflect.Slice: + if v.Type().Elem().Kind() == reflect.Uint8 { + p.Value = v.Bytes() + } + case reflect.Struct: + if !v.CanAddr() { + return fmt.Errorf("datastore: unsupported struct field: value is unaddressable") + } + sub, err := newStructPLS(v.Addr().Interface()) + if err != nil { + return fmt.Errorf("datastore: unsupported struct field: %v", err) + } + return sub.(structPLS).save(props, name, noIndex, multiple) + } + } + if p.Value == nil { + return fmt.Errorf("datastore: unsupported struct field type: %v", v.Type()) + } + *props = append(*props, p) + return nil +} + +func (s structPLS) Save() ([]Property, error) { + var props []Property + if err := s.save(&props, "", false, false); err != nil { + return nil, err + } + return props, nil +} + +func (s structPLS) save(props *[]Property, prefix string, noIndex, multiple bool) error { + for i, t := range s.codec.byIndex { + if t.name == "-" { + continue + } + name := t.name + if prefix != "" { + name = prefix + name + } + v := s.v.Field(i) + if !v.IsValid() || !v.CanSet() { + continue + } + noIndex1 := noIndex || t.noIndex + // For slice fields that aren't []byte, save each element. + if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 { + for j := 0; j < v.Len(); j++ { + if err := saveStructProperty(props, name, noIndex1, true, v.Index(j)); err != nil { + return err + } + } + continue + } + // Otherwise, save the field itself. + if err := saveStructProperty(props, name, noIndex1, multiple, v); err != nil { + return err + } + } + return nil +} + +func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) { + e := &pb.Entity{ + Key: keyToProto(key), + } + indexedProps := 0 + prevMultiple := make(map[string]*pb.Property) + for _, p := range props { + val, err := interfaceToProto(p.Value) + if err != "" { + return nil, fmt.Errorf("datastore: %s for a Property with Name %q", err, p.Name) + } + if !p.NoIndex { + rVal := reflect.ValueOf(p.Value) + if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 { + indexedProps += rVal.Len() + } else { + indexedProps++ + } + } + if indexedProps > maxIndexedProperties { + return nil, errors.New("datastore: too many indexed properties") + } + switch v := p.Value.(type) { + case string: + case []byte: + if len(v) > 1500 && !p.NoIndex { + return nil, fmt.Errorf("datastore: cannot index a Property with Name %q", p.Name) + } + } + val.Indexed = proto.Bool(!p.NoIndex) + if p.Multiple { + x, ok := prevMultiple[p.Name] + if !ok { + x = &pb.Property{ + Name: proto.String(p.Name), + Value: &pb.Value{}, + } + prevMultiple[p.Name] = x + e.Property = append(e.Property, x) + } + x.Value.ListValue = append(x.Value.ListValue, val) + } else { + e.Property = append(e.Property, &pb.Property{ + Name: proto.String(p.Name), + Value: val, + }) + } + } + return e, nil +} + +func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) { + val := new(pb.Value) + switch v := iv.(type) { + case int: + val.IntegerValue = proto.Int64(int64(v)) + case int32: + val.IntegerValue = proto.Int64(int64(v)) + case int64: + val.IntegerValue = proto.Int64(v) + case bool: + val.BooleanValue = proto.Bool(v) + case string: + val.StringValue = proto.String(v) + case float32: + val.DoubleValue = proto.Float64(float64(v)) + case float64: + val.DoubleValue = proto.Float64(v) + case *Key: + if v != nil { + val.KeyValue = keyToProto(v) + } + case time.Time: + if v.Before(minTime) || v.After(maxTime) { + return nil, fmt.Sprintf("time value out of range") + } + val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v)) + case []byte: + val.BlobValue = v + default: + if iv != nil { + return nil, fmt.Sprintf("invalid Value type %t", iv) + } + } + // TODO(jbd): Support ListValue and EntityValue. + // TODO(jbd): Support types whose underlying type is one of the types above. + return val, "" +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/testdata/index.yaml b/Godeps/_workspace/src/google.golang.org/cloud/datastore/testdata/index.yaml new file mode 100644 index 0000000000..47bc9de867 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/testdata/index.yaml @@ -0,0 +1,41 @@ +indexes: + +- kind: SQChild + ancestor: yes + properties: + - name: T + - name: I + +- kind: SQChild + ancestor: yes + properties: + - name: T + - name: I + direction: desc + +- kind: SQChild + ancestor: yes + properties: + - name: I + - name: T + - name: U + +- kind: SQChild + ancestor: yes + properties: + - name: I + - name: T + - name: U + +- kind: SQChild + ancestor: yes + properties: + - name: T + - name: J + +- kind: SQChild + ancestor: yes + properties: + - name: T + - name: J + - name: U \ No newline at end of file diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/time.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/time.go new file mode 100644 index 0000000000..e7f6a19314 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/time.go @@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "math" + "time" +) + +var ( + minTime = time.Unix(int64(math.MinInt64)/1e6, (int64(math.MinInt64)%1e6)*1e3) + maxTime = time.Unix(int64(math.MaxInt64)/1e6, (int64(math.MaxInt64)%1e6)*1e3) +) + +func toUnixMicro(t time.Time) int64 { + // We cannot use t.UnixNano() / 1e3 because we want to handle times more than + // 2^63 nanoseconds (which is about 292 years) away from 1970, and those cannot + // be represented in the numerator of a single int64 divide. + return t.Unix()*1e6 + int64(t.Nanosecond()/1e3) +} + +func fromUnixMicro(t int64) time.Time { + return time.Unix(t/1e6, (t%1e6)*1e3) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/time_test.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/time_test.go new file mode 100644 index 0000000000..5cc846c4cc --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/time_test.go @@ -0,0 +1,75 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "testing" + "time" +) + +func TestUnixMicro(t *testing.T) { + // Test that all these time.Time values survive a round trip to unix micros. + testCases := []time.Time{ + {}, + time.Date(2, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(23, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(234, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(1600, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(1700, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(1800, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC), + time.Unix(-1e6, -1000), + time.Unix(-1e6, 0), + time.Unix(-1e6, +1000), + time.Unix(-60, -1000), + time.Unix(-60, 0), + time.Unix(-60, +1000), + time.Unix(-1, -1000), + time.Unix(-1, 0), + time.Unix(-1, +1000), + time.Unix(0, -3000), + time.Unix(0, -2000), + time.Unix(0, -1000), + time.Unix(0, 0), + time.Unix(0, +1000), + time.Unix(0, +2000), + time.Unix(+60, -1000), + time.Unix(+60, 0), + time.Unix(+60, +1000), + time.Unix(+1e6, -1000), + time.Unix(+1e6, 0), + time.Unix(+1e6, +1000), + time.Date(1999, 12, 31, 23, 59, 59, 999000, time.UTC), + time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(2006, 1, 2, 15, 4, 5, 678000, time.UTC), + time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC), + time.Date(3456, 1, 1, 0, 0, 0, 0, time.UTC), + } + for _, tc := range testCases { + got := fromUnixMicro(toUnixMicro(tc)) + if !got.Equal(tc) { + t.Errorf("got %q, want %q", got, tc) + } + } + + // Test that a time.Time that isn't an integral number of microseconds + // is not perfectly reconstructed after a round trip. + t0 := time.Unix(0, 123) + t1 := fromUnixMicro(toUnixMicro(t0)) + if t1.Nanosecond()%1000 != 0 || t0.Nanosecond()%1000 == 0 { + t.Errorf("quantization to Β΅s: got %q with %d ns, started with %d ns", t1, t1.Nanosecond(), t0.Nanosecond()) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/datastore/transaction.go b/Godeps/_workspace/src/google.golang.org/cloud/datastore/transaction.go new file mode 100644 index 0000000000..344c313e90 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/datastore/transaction.go @@ -0,0 +1,241 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "errors" + "net/http" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + + pb "google.golang.org/cloud/internal/datastore" + "google.golang.org/cloud/internal/transport" +) + +// ErrConcurrentTransaction is returned when a transaction is rolled back due +// to a conflict with a concurrent transaction. +var ErrConcurrentTransaction = errors.New("datastore: concurrent transaction") + +var errExpiredTransaction = errors.New("datastore: transaction expired") + +// A TransactionOption configures the Transaction returned by NewTransaction. +type TransactionOption interface { + apply(*pb.BeginTransactionRequest) +} + +type isolation struct { + level pb.BeginTransactionRequest_IsolationLevel +} + +func (i isolation) apply(req *pb.BeginTransactionRequest) { + req.IsolationLevel = i.level.Enum() +} + +var ( + // Snapshot causes the transaction to enforce a snapshot isolation level. + Snapshot TransactionOption = isolation{pb.BeginTransactionRequest_SNAPSHOT} + // Serializable causes the transaction to enforce a serializable isolation level. + Serializable TransactionOption = isolation{pb.BeginTransactionRequest_SERIALIZABLE} +) + +// Transaction represents a set of datastore operations to be committed atomically. +// +// Operations are enqueued by calling the Put and Delete methods on Transaction +// (or their Multi-equivalents). These operations are only committed when the +// Commit method is invoked. To ensure consistency, reads must be performed by +// using Transaction's Get method or by using the Transaction method when +// building a query. +// +// A Transaction must be committed or rolled back exactly once. +type Transaction struct { + id []byte + client *Client + ctx context.Context + mutation *pb.Mutation // The mutations to apply. + pending []*PendingKey // Incomplete keys pending transaction completion. +} + +// NewTransaction starts a new transaction. +func (c *Client) NewTransaction(ctx context.Context, opts ...TransactionOption) (*Transaction, error) { + req, resp := &pb.BeginTransactionRequest{}, &pb.BeginTransactionResponse{} + for _, o := range opts { + o.apply(req) + } + if err := c.call(ctx, "beginTransaction", req, resp); err != nil { + return nil, err + } + + return &Transaction{ + id: resp.Transaction, + ctx: ctx, + client: c, + mutation: &pb.Mutation{}, + }, nil +} + +// Commit applies the enqueued operations atomically. +func (t *Transaction) Commit() (*Commit, error) { + if t.id == nil { + return nil, errExpiredTransaction + } + req := &pb.CommitRequest{ + Transaction: t.id, + Mutation: t.mutation, + Mode: pb.CommitRequest_TRANSACTIONAL.Enum(), + } + t.id = nil + resp := &pb.CommitResponse{} + if err := t.client.call(t.ctx, "commit", req, resp); err != nil { + if e, ok := err.(*transport.ErrHTTP); ok && e.StatusCode == http.StatusConflict { + // TODO(jbd): Make sure that we explicitly handle the case where response + // has an HTTP 409 and the error message indicates that it's an concurrent + // transaction error. + return nil, ErrConcurrentTransaction + } + return nil, err + } + + // Copy any newly minted keys into the returned keys. + if len(t.pending) != len(resp.MutationResult.InsertAutoIdKey) { + return nil, errors.New("datastore: internal error: server returned the wrong number of keys") + } + commit := &Commit{} + for i, p := range t.pending { + p.key = protoToKey(resp.MutationResult.InsertAutoIdKey[i]) + p.commit = commit + } + + return commit, nil +} + +// Rollback abandons a pending transaction. +func (t *Transaction) Rollback() error { + if t.id == nil { + return errExpiredTransaction + } + id := t.id + t.id = nil + return t.client.call(t.ctx, "rollback", &pb.RollbackRequest{Transaction: id}, &pb.RollbackResponse{}) +} + +// Get is the transaction-specific version of the package function Get. +// All reads performed during the transaction will come from a single consistent +// snapshot. Furthermore, if the transaction is set to a serializable isolation +// level, another transaction cannot concurrently modify the data that is read +// or modified by this transaction. +func (t *Transaction) Get(key *Key, dst interface{}) error { + err := t.client.get(t.ctx, []*Key{key}, []interface{}{dst}, &pb.ReadOptions{Transaction: t.id}) + if me, ok := err.(MultiError); ok { + return me[0] + } + return err +} + +// GetMulti is a batch version of Get. +func (t *Transaction) GetMulti(keys []*Key, dst interface{}) error { + if t.id == nil { + return errExpiredTransaction + } + return t.client.get(t.ctx, keys, dst, &pb.ReadOptions{Transaction: t.id}) +} + +// Put is the transaction-specific version of the package function Put. +// +// Put returns a PendingKey which can be resolved into a Key using the +// return value from a successful Commit. If key is an incomplete key, the +// returned pending key will resolve to a unique key generated by the +// datastore. +func (t *Transaction) Put(key *Key, src interface{}) (*PendingKey, error) { + h, err := t.PutMulti([]*Key{key}, []interface{}{src}) + if err != nil { + if me, ok := err.(MultiError); ok { + return nil, me[0] + } + return nil, err + } + return h[0], nil +} + +// PutMulti is a batch version of Put. One PendingKey is returned for each +// element of src in the same order. +func (t *Transaction) PutMulti(keys []*Key, src interface{}) ([]*PendingKey, error) { + if t.id == nil { + return nil, errExpiredTransaction + } + mutation, err := putMutation(keys, src) + if err != nil { + return nil, err + } + proto.Merge(t.mutation, mutation) + + // Prepare the returned handles, pre-populating where possible. + ret := make([]*PendingKey, len(keys)) + for i, key := range keys { + h := &PendingKey{} + if key.Incomplete() { + // This key will be in the final commit result. + t.pending = append(t.pending, h) + } else { + h.key = key + } + + ret[i] = h + } + + return ret, nil +} + +// Delete is the transaction-specific version of the package function Delete. +// Delete enqueues the deletion of the entity for the given key, to be +// committed atomically upon calling Commit. +func (t *Transaction) Delete(key *Key) error { + err := t.DeleteMulti([]*Key{key}) + if me, ok := err.(MultiError); ok { + return me[0] + } + return err +} + +// DeleteMulti is a batch version of Delete. +func (t *Transaction) DeleteMulti(keys []*Key) error { + if t.id == nil { + return errExpiredTransaction + } + mutation, err := deleteMutation(keys) + if err != nil { + return err + } + proto.Merge(t.mutation, mutation) + return nil +} + +// Commit represents the result of a committed transaction. +type Commit struct{} + +// Key resolves a pending key handle into a final key. +func (c *Commit) Key(p *PendingKey) *Key { + if c != p.commit { + panic("PendingKey was not created by corresponding transaction") + } + return p.key +} + +// PendingKey represents the key for newly-inserted entity. It can be +// resolved into a Key by calling the Key method of Commit. +type PendingKey struct { + key *Key + commit *Commit +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/concat_table/main.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/concat_table/main.go new file mode 100644 index 0000000000..978801b043 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/concat_table/main.go @@ -0,0 +1,111 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// concat_table is an example client of the bigquery client library. +// It concatenates two BigQuery tables and writes the result to another table. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/cloud/bigquery" +) + +var ( + project = flag.String("project", "", "The ID of a Google Cloud Platform project") + dataset = flag.String("dataset", "", "The ID of a BigQuery dataset") + src1 = flag.String("src1", "", "The ID of the first BigQuery table to concatenate") + src2 = flag.String("src2", "", "The ID of the second BigQuery table to concatenate") + dest = flag.String("dest", "", "The ID of the BigQuery table to write the result to") + pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status") +) + +func main() { + flag.Parse() + + flagsOk := true + for _, f := range []string{"project", "dataset", "src1", "src2", "dest"} { + if flag.Lookup(f).Value.String() == "" { + fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f) + flagsOk = false + } + } + if !flagsOk { + os.Exit(1) + } + if *src1 == *src2 || *src1 == *dest || *src2 == *dest { + log.Fatalf("Different values must be supplied for each of --src1, --src2 and --dest") + } + + httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope) + if err != nil { + log.Fatalf("Creating http client: %v", err) + } + + client, err := bigquery.NewClient(httpClient, *project) + if err != nil { + log.Fatalf("Creating bigquery client: %v", err) + } + + s1 := &bigquery.Table{ + ProjectID: *project, + DatasetID: *dataset, + TableID: *src1, + } + + s2 := &bigquery.Table{ + ProjectID: *project, + DatasetID: *dataset, + TableID: *src2, + } + + d := &bigquery.Table{ + ProjectID: *project, + DatasetID: *dataset, + TableID: *dest, + } + + // Concatenate data. + job, err := client.Copy(context.Background(), d, bigquery.Tables{s1, s2}, bigquery.WriteTruncate) + + if err != nil { + log.Fatalf("Concatenating: %v", err) + } + + fmt.Printf("Job for concatenation operation: %+v\n", job) + fmt.Printf("Waiting for job to complete.\n") + + for range time.Tick(*pollint) { + status, err := job.Status(context.Background()) + if err != nil { + fmt.Printf("Failure determining status: %v", err) + break + } + if !status.Done() { + continue + } + if err := status.Err(); err == nil { + fmt.Printf("Success\n") + } else { + fmt.Printf("Failure: %+v\n", err) + } + break + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/load/main.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/load/main.go new file mode 100644 index 0000000000..30ed9dbba0 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/load/main.go @@ -0,0 +1,104 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// load is an example client of the bigquery client library. +// It loads a file from Google Cloud Storage into a BigQuery table. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/cloud/bigquery" +) + +var ( + project = flag.String("project", "", "The ID of a Google Cloud Platform project") + dataset = flag.String("dataset", "", "The ID of a BigQuery dataset") + table = flag.String("table", "", "The ID of a BigQuery table to load data into") + bucket = flag.String("bucket", "", "The name of a Google Cloud Storage bucket to load data from") + object = flag.String("object", "", "The name of a Google Cloud Storage object to load data from. Must exist within the bucket specified by --bucket") + skiprows = flag.Int64("skiprows", 0, "The number of rows of the source data to skip when loading") + pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status") +) + +func main() { + flag.Parse() + + flagsOk := true + for _, f := range []string{"project", "dataset", "table", "bucket", "object"} { + if flag.Lookup(f).Value.String() == "" { + fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f) + flagsOk = false + } + } + if !flagsOk { + os.Exit(1) + } + + httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope) + if err != nil { + log.Fatalf("Creating http client: %v", err) + } + + client, err := bigquery.NewClient(httpClient, *project) + if err != nil { + log.Fatalf("Creating bigquery client: %v", err) + } + + table := &bigquery.Table{ + ProjectID: *project, + DatasetID: *dataset, + TableID: *table, + } + + gcs := client.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object)) + gcs.SkipLeadingRows = *skiprows + + // Load data from Google Cloud Storage into a BigQuery table. + job, err := client.Copy( + context.Background(), table, gcs, + bigquery.MaxBadRecords(1), + bigquery.AllowQuotedNewlines(), + bigquery.WriteTruncate) + + if err != nil { + log.Fatalf("Loading data: %v", err) + } + + fmt.Printf("Job for data load operation: %+v\n", job) + fmt.Printf("Waiting for job to complete.\n") + + for range time.Tick(*pollint) { + status, err := job.Status(context.Background()) + if err != nil { + fmt.Printf("Failure determining status: %v", err) + break + } + if !status.Done() { + continue + } + if err := status.Err(); err == nil { + fmt.Printf("Success\n") + } else { + fmt.Printf("Failure: %+v\n", err) + } + break + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/query/main.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/query/main.go new file mode 100644 index 0000000000..d6dc0b6a1b --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/query/main.go @@ -0,0 +1,108 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// query is an example client of the bigquery client library. +// It submits a query and writes the result to a table. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/cloud/bigquery" +) + +var ( + project = flag.String("project", "", "The ID of a Google Cloud Platform project") + dataset = flag.String("dataset", "", "The ID of a BigQuery dataset") + q = flag.String("q", "", "The query string") + dest = flag.String("dest", "", "The ID of the BigQuery table to write the result to. If unset, an ephemeral table ID will be generated.") + pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status") + wait = flag.Bool("wait", false, "Whether to wait for the query job to complete.") +) + +func main() { + flag.Parse() + + flagsOk := true + for _, f := range []string{"project", "dataset", "q"} { + if flag.Lookup(f).Value.String() == "" { + fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f) + flagsOk = false + } + } + if !flagsOk { + os.Exit(1) + } + + httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope) + if err != nil { + log.Fatalf("Creating http client: %v", err) + } + + client, err := bigquery.NewClient(httpClient, *project) + if err != nil { + log.Fatalf("Creating bigquery client: %v", err) + } + + d := &bigquery.Table{} + + if *dest != "" { + d.ProjectID = *project + d.DatasetID = *dataset + d.TableID = *dest + } + + query := &bigquery.Query{ + Q: *q, + DefaultProjectID: *project, + DefaultDatasetID: *dataset, + } + + // Query data. + job, err := client.Copy(context.Background(), d, query, bigquery.WriteTruncate) + + if err != nil { + log.Fatalf("Querying: %v", err) + } + + fmt.Printf("Submitted query. Job ID: %s\n", job.ID()) + if !*wait { + return + } + + fmt.Printf("Waiting for job to complete.\n") + + for range time.Tick(*pollint) { + status, err := job.Status(context.Background()) + if err != nil { + fmt.Printf("Failure determining status: %v", err) + break + } + if !status.Done() { + continue + } + if err := status.Err(); err == nil { + fmt.Printf("Success\n") + } else { + fmt.Printf("Failure: %+v\n", err) + } + break + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/read/main.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/read/main.go new file mode 100644 index 0000000000..181bd7c780 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigquery/read/main.go @@ -0,0 +1,148 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// read is an example client of the bigquery client library. +// It reads from a table, returning the data via an Iterator. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "regexp" + "strings" + "text/tabwriter" + + "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/cloud/bigquery" +) + +var ( + project = flag.String("project", "", "The ID of a Google Cloud Platform project") + dataset = flag.String("dataset", "", "The ID of a BigQuery dataset") + table = flag.String("table", ".*", "A regular expression to match the IDs of tables to read.") + jobID = flag.String("jobid", "", "The ID of a query job that has already been submitted."+ + " If set, --dataset, --table will be ignored, and results will be read from the specified job.") +) + +func printValues(it *bigquery.Iterator) { + // one-space padding. + tw := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + + for it.Next(context.Background()) { + var vals bigquery.ValueList + if err := it.Get(&vals); err != nil { + fmt.Printf("err calling get: %v\n", err) + } else { + sep := "" + for _, v := range vals { + fmt.Fprintf(tw, "%s%v", sep, v) + sep = "\t" + } + fmt.Fprintf(tw, "\n") + } + } + tw.Flush() + + fmt.Printf("\n") + if err := it.Err(); err != nil { + fmt.Printf("err reading: %v\n", err) + } +} + +func printTable(client *bigquery.Client, t *bigquery.Table) { + it, err := client.Read(context.Background(), t) + if err != nil { + log.Fatalf("Reading: %v", err) + } + + id := t.FullyQualifiedName() + fmt.Printf("%s\n%s\n", id, strings.Repeat("-", len(id))) + printValues(it) +} + +func printQueryResults(client *bigquery.Client, queryJobID string) { + job, err := client.JobFromID(context.Background(), queryJobID) + if err != nil { + log.Fatalf("Loading job: %v", err) + } + + it, err := client.Read(context.Background(), job) + if err != nil { + log.Fatalf("Reading: %v", err) + } + + // TODO: print schema. + printValues(it) +} + +func main() { + flag.Parse() + + flagsOk := true + if flag.Lookup("project").Value.String() == "" { + fmt.Fprintf(os.Stderr, "Flag --project is required\n") + flagsOk = false + } + + var sourceFlagCount int + if flag.Lookup("dataset").Value.String() != "" { + sourceFlagCount++ + } + if flag.Lookup("jobid").Value.String() != "" { + sourceFlagCount++ + } + if sourceFlagCount != 1 { + fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n") + flagsOk = false + } + + if !flagsOk { + os.Exit(1) + } + + tableRE, err := regexp.Compile(*table) + if err != nil { + fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table) + os.Exit(1) + } + + httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope) + if err != nil { + log.Fatalf("Creating http client: %v", err) + } + + client, err := bigquery.NewClient(httpClient, *project) + if err != nil { + log.Fatalf("Creating bigquery client: %v", err) + } + + if *jobID != "" { + printQueryResults(client, *jobID) + return + } + ds := client.Dataset(*dataset) + var tables []*bigquery.Table + tables, err = ds.ListTables(context.Background()) + if err != nil { + log.Fatalf("Listing tables: %v", err) + } + for _, t := range tables { + if tableRE.MatchString(t.TableID) { + printTable(client, t) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/README.md b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/README.md new file mode 100644 index 0000000000..3f3e92155e --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/README.md @@ -0,0 +1,30 @@ +# Cloud Bigtable on Managed VMs using Go +# (Hello World for Cloud Bigtable) + +This app counts how often each user visits. + +## Prerequisites + +1. Set up Cloud Console. + 1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project. + You will need the project ID later. + 1. Go to **Settings > Project Billing Settings** and enable billing. + 1. Select **APIs & Auth > APIs**. + 1. Enable the **Cloud Bigtable API** and the **Cloud Bigtable Admin API**. + (You may need to search for the API). +1. Set up gcloud. + 1. `gcloud components update` + 1. `gcloud auth login` + 1. `gcloud config set project PROJECT_ID` +1. Download App Engine SDK for Go. + 1. `go get -u google.golang.org/appengine/...` +1. In helloworld.go, change the constants `project`, `zone` and `cluster` + +## Running locally + +1. From the sample project folder, `gcloud preview app run app.yaml` + +## Deploying on Google App Engine Managed VM + +1. Install and start [Docker](https://cloud.google.com/appengine/docs/managed-vms/getting-started#install_docker). +1. From the sample project folder, `aedeploy gcloud preview app deploy app.yaml` diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/app.yaml b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/app.yaml new file mode 100644 index 0000000000..4f5fef0e5a --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/app.yaml @@ -0,0 +1,11 @@ +runtime: go +api_version: go1 +vm: true + +manual_scaling: + instances: 1 + +handlers: +# Serve only the web root. +- url: / + script: _go_app diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/helloworld.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/helloworld.go new file mode 100644 index 0000000000..8f1bd702e9 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/bigtable-hello/helloworld.go @@ -0,0 +1,169 @@ +// Copyright 2015 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +/* +helloworld tracks how often a user has visited the index page. + +This program demonstrates usage of the Cloud Bigtable API for Managed VMs and Go. +Instructions for running this program are in the README.md. +*/ +package main + +import ( + "bytes" + "encoding/binary" + "html/template" + "log" + "net/http" + + "golang.org/x/net/context" + "google.golang.org/appengine" + aelog "google.golang.org/appengine/log" + "google.golang.org/appengine/user" + "google.golang.org/cloud/bigtable" +) + +// User-provided constants. +const ( + project = "PROJECT_ID" + zone = "CLUSTER_ZONE" + cluster = "CLUSTER_NAME" +) + +var ( + tableName = "bigtable-hello" + familyName = "emails" + + // Client is initialized by main. + client *bigtable.Client +) + +func main() { + ctx := context.Background() + + // Set up admin client, tables, and column families. + // NewAdminClient uses Application Default Credentials to authenticate. + adminClient, err := bigtable.NewAdminClient(ctx, project, zone, cluster) + if err != nil { + log.Fatalf("Unable to create a table admin client. %v", err) + } + tables, err := adminClient.Tables(ctx) + if err != nil { + log.Fatalf("Unable to fetch table list. %v", err) + } + if !sliceContains(tables, tableName) { + if err := adminClient.CreateTable(ctx, tableName); err != nil { + log.Fatalf("Unable to create table: %v. %v", tableName, err) + } + } + tblInfo, err := adminClient.TableInfo(ctx, tableName) + if err != nil { + log.Fatalf("Unable to read info for table: %v. %v", tableName, err) + } + if !sliceContains(tblInfo.Families, familyName) { + if err := adminClient.CreateColumnFamily(ctx, tableName, familyName); err != nil { + log.Fatalf("Unable to create column family: %v. %v", familyName, err) + } + } + adminClient.Close() + + // Set up Bigtable data operations client. + // NewClient uses Application Default Credentials to authenticate. + client, err = bigtable.NewClient(ctx, project, zone, cluster) + if err != nil { + log.Fatalf("Unable to create data operations client. %v", err) + } + + http.Handle("/", appHandler(mainHandler)) + appengine.Main() // Never returns. +} + +// mainHandler tracks how many times each user has visited this page. +func mainHandler(w http.ResponseWriter, r *http.Request) *appError { + if r.URL.Path != "/" { + http.NotFound(w, r) + return nil + } + + ctx := appengine.NewContext(r) + u := user.Current(ctx) + if u == nil { + login, err := user.LoginURL(ctx, r.URL.String()) + if err != nil { + return &appError{err, "Error finding login URL", http.StatusInternalServerError} + } + http.Redirect(w, r, login, http.StatusFound) + return nil + } + logoutURL, err := user.LogoutURL(ctx, "/") + if err != nil { + return &appError{err, "Error finding logout URL", http.StatusInternalServerError} + } + + // Display hello page. + tbl := client.Open(tableName) + rmw := bigtable.NewReadModifyWrite() + rmw.Increment(familyName, u.Email, 1) + row, err := tbl.ApplyReadModifyWrite(ctx, u.Email, rmw) + if err != nil { + return &appError{err, "Error applying ReadModifyWrite to row: " + u.Email, http.StatusInternalServerError} + } + data := struct { + Username, Logout string + Visits uint64 + }{ + Username: u.Email, + // Retrieve the most recently edited column. + Visits: binary.BigEndian.Uint64(row[familyName][0].Value), + Logout: logoutURL, + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return &appError{err, "Error writing template", http.StatusInternalServerError} + } + buf.WriteTo(w) + return nil +} + +var tmpl = template.Must(template.New("").Parse(` + + +

+{{with .Username}} Hello {{.}}{{end}} +{{with .Logout}}Sign out{{end}} + +

+ +

+You have visited {{.Visits}} +

+ +`)) + +// sliceContains reports whether the provided string is present in the given slice of strings. +func sliceContains(list []string, target string) bool { + for _, s := range list { + if s == target { + return true + } + } + return false +} + +// More info about this method of error handling can be found at: http://blog.golang.org/error-handling-and-go +type appHandler func(http.ResponseWriter, *http.Request) *appError + +type appError struct { + Error error + Message string + Code int +} + +func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if e := fn(w, r); e != nil { + ctx := appengine.NewContext(r) + aelog.Errorf(ctx, "%v", e.Error) + http.Error(w, e.Message, e.Code) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/search/search.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/search/search.go new file mode 100644 index 0000000000..c9b1339d58 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/bigtable/search/search.go @@ -0,0 +1,439 @@ +/* +Copyright 2015 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This is a sample web server that uses Cloud Bigtable as the storage layer +// for a simple document-storage and full-text-search service. +// It has three functions: +// - Add a document. This adds the content of a user-supplied document to the +// Bigtable, and adds references to the document to an index in the Bigtable. +// The document is indexed under each unique word in the document. +// - Search the index. This returns documents containing each word in a user +// query, with snippets and links to view the whole document. +// - Clear the table. This deletes and recreates the Bigtable, +package main + +import ( + "bytes" + "flag" + "fmt" + "html/template" + "io" + "log" + "net/http" + "os" + "strings" + "sync" + "time" + "unicode" + + "golang.org/x/net/context" + "google.golang.org/cloud/bigtable" +) + +var ( + project = flag.String("project", "", "The name of the project.") + zone = flag.String("zone", "", "The zone of the project.") + cluster = flag.String("cluster", "", "The name of the Cloud Bigtable cluster.") + tableName = flag.String("table", "docindex", "The name of the table containing the documents and index.") + credFile = flag.String("creds", "", "File containing credentials") + rebuild = flag.Bool("rebuild", false, "Rebuild the table from scratch on startup.") + + client *bigtable.Client + adminClient *bigtable.AdminClient + table *bigtable.Table + + addTemplate = template.Must(template.New("").Parse(` +Added {{.Title}} +`)) + + contentTemplate = template.Must(template.New("").Parse(` +{{.Title}}

+{{.Content}} +`)) + + searchTemplate = template.Must(template.New("").Parse(` +Results for {{.Query}}:

+{{range .Results}} +{{.Title}}
+{{.Snippet}}

+{{end}} +`)) +) + +const ( + // prototypeTableName is an existing table containing some documents. + // Rebuilding a table will populate it with the data from this table. + prototypeTableName = "shakespearetemplate" + indexColumnFamily = "i" + contentColumnFamily = "c" + mainPage = ` + + + Document Search + + + Search for documents: +
+
+
+ + + Add a document: +
+ Document name: +
+ Document text: +
+
+ + + Rebuild table: +
+
+ + + + ` +) + +func main() { + flag.Parse() + + if *tableName == prototypeTableName { + log.Fatal("Can't use " + prototypeTableName + " as your table.") + } + + // Let the library get credentials from file. + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", *credFile) + + // Make an admin client. + var err error + if adminClient, err = bigtable.NewAdminClient(context.Background(), *project, *zone, *cluster); err != nil { + log.Fatal("Bigtable NewAdminClient:", err) + } + + // Make a regular client. + client, err = bigtable.NewClient(context.Background(), *project, *zone, *cluster) + if err != nil { + log.Fatal("Bigtable NewClient:", err) + } + + // Open the table. + table = client.Open(*tableName) + + // Rebuild the table if the command-line flag is set. + if *rebuild { + if err := rebuildTable(); err != nil { + log.Fatal(err) + } + } + + // Set up HTML handlers, and start the web server. + http.HandleFunc("/search", handleSearch) + http.HandleFunc("/content", handleContent) + http.HandleFunc("/add", handleAddDoc) + http.HandleFunc("/clearindex", handleClear) + http.HandleFunc("/", handleMain) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +// handleMain outputs the home page, containing a search box, an "add document" box, and "clear table" button. +func handleMain(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, mainPage) +} + +// tokenize splits a string into tokens. +// This is very simple, it's not a good tokenization function. +func tokenize(s string) []string { + wordMap := make(map[string]bool) + f := strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) }) + for _, word := range f { + word = strings.ToLower(word) + wordMap[word] = true + } + words := make([]string, 0, len(wordMap)) + for word := range wordMap { + words = append(words, word) + } + return words +} + +// handleContent fetches the content of a document from the Bigtable and returns it. +func handleContent(w http.ResponseWriter, r *http.Request) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + name := r.FormValue("name") + if len(name) == 0 { + http.Error(w, "No document name supplied.", http.StatusBadRequest) + return + } + + row, err := table.ReadRow(ctx, name) + if err != nil { + http.Error(w, "Error reading content: "+err.Error(), http.StatusInternalServerError) + return + } + content := row[contentColumnFamily] + if len(content) == 0 { + http.Error(w, "Document not found.", http.StatusNotFound) + return + } + var buf bytes.Buffer + if err := contentTemplate.ExecuteTemplate(&buf, "", struct{ Title, Content string }{name, string(content[0].Value)}); err != nil { + http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError) + return + } + io.Copy(w, &buf) +} + +// handleSearch responds to search queries, returning links and snippets for matching documents. +func handleSearch(w http.ResponseWriter, r *http.Request) { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + query := r.FormValue("q") + // Split the query into words. + words := tokenize(query) + if len(words) == 0 { + http.Error(w, "Empty query.", http.StatusBadRequest) + return + } + + // readRows reads from many rows concurrently. + readRows := func(rows []string) ([]bigtable.Row, error) { + results := make([]bigtable.Row, len(rows)) + errors := make([]error, len(rows)) + var wg sync.WaitGroup + for i, row := range rows { + wg.Add(1) + go func(i int, row string) { + defer wg.Done() + results[i], errors[i] = table.ReadRow(ctx, row) + }(i, row) + } + wg.Wait() + for _, err := range errors { + if err != nil { + return nil, err + } + } + return results, nil + } + + // For each query word, get the list of documents containing it. + results, err := readRows(words) + if err != nil { + http.Error(w, "Error reading index: "+err.Error(), http.StatusInternalServerError) + return + } + + // Count how many of the query words each result contained. + hits := make(map[string]int) + for _, r := range results { + for _, r := range r[indexColumnFamily] { + hits[r.Column]++ + } + } + + // Build a slice of all the documents that matched every query word. + var matches []string + for doc, count := range hits { + if count == len(words) { + matches = append(matches, doc[len(indexColumnFamily+":"):]) + } + } + + // Fetch the content of those documents from the Bigtable. + content, err := readRows(matches) + if err != nil { + http.Error(w, "Error reading results: "+err.Error(), http.StatusInternalServerError) + return + } + + type result struct{ Title, Snippet string } + data := struct { + Query string + Results []result + }{query, nil} + + // Output links and snippets. + for i, doc := range matches { + var text string + c := content[i][contentColumnFamily] + if len(c) > 0 { + text = string(c[0].Value) + } + if len(text) > 100 { + text = text[:100] + "..." + } + data.Results = append(data.Results, result{doc, text}) + } + var buf bytes.Buffer + if err := searchTemplate.ExecuteTemplate(&buf, "", data); err != nil { + http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError) + return + } + io.Copy(w, &buf) +} + +// handleAddDoc adds a document to the index. +func handleAddDoc(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "POST requests only", http.StatusMethodNotAllowed) + return + } + + ctx, _ := context.WithTimeout(context.Background(), time.Minute) + + name := r.FormValue("name") + if len(name) == 0 { + http.Error(w, "Empty document name!", http.StatusBadRequest) + return + } + + content := r.FormValue("content") + if len(content) == 0 { + http.Error(w, "Empty document content!", http.StatusBadRequest) + return + } + + var ( + writeErr error // Set if any write fails. + mu sync.Mutex // Protects writeErr + wg sync.WaitGroup // Used to wait for all writes to finish. + ) + + // writeOneColumn writes one column in one row, updates err if there is an error, + // and signals wg that one operation has finished. + writeOneColumn := func(row, family, column, value string, ts bigtable.Timestamp) { + mut := bigtable.NewMutation() + mut.Set(family, column, ts, []byte(value)) + err := table.Apply(ctx, row, mut) + if err != nil { + mu.Lock() + writeErr = err + mu.Unlock() + } + } + + // Start a write to store the document content. + wg.Add(1) + go func() { + writeOneColumn(name, contentColumnFamily, "", content, bigtable.Now()) + wg.Done() + }() + + // Start writes to store the document name in the index for each word in the document. + words := tokenize(content) + for _, word := range words { + var ( + row = word + family = indexColumnFamily + column = name + value = "" + ts = bigtable.Now() + ) + wg.Add(1) + go func() { + // TODO: should use a semaphore to limit the number of concurrent writes. + writeOneColumn(row, family, column, value, ts) + wg.Done() + }() + } + wg.Wait() + if writeErr != nil { + http.Error(w, "Error writing to Bigtable: "+writeErr.Error(), http.StatusInternalServerError) + return + } + var buf bytes.Buffer + if err := addTemplate.ExecuteTemplate(&buf, "", struct{ Title string }{name}); err != nil { + http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError) + return + } + io.Copy(w, &buf) +} + +// rebuildTable deletes the table if it exists, then creates the table, with the index column family. +func rebuildTable() error { + ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute) + adminClient.DeleteTable(ctx, *tableName) + if err := adminClient.CreateTable(ctx, *tableName); err != nil { + return fmt.Errorf("CreateTable: %v", err) + } + time.Sleep(20 * time.Second) + if err := adminClient.CreateColumnFamily(ctx, *tableName, indexColumnFamily); err != nil { + return fmt.Errorf("CreateColumnFamily: %v", err) + } + if err := adminClient.CreateColumnFamily(ctx, *tableName, contentColumnFamily); err != nil { + return fmt.Errorf("CreateColumnFamily: %v", err) + } + + // Open the prototype table. It contains a number of documents to get started with. + prototypeTable := client.Open(prototypeTableName) + + var ( + writeErr error // Set if any write fails. + mu sync.Mutex // Protects writeErr + wg sync.WaitGroup // Used to wait for all writes to finish. + ) + copyRowToTable := func(row bigtable.Row) bool { + mu.Lock() + failed := writeErr != nil + mu.Unlock() + if failed { + return false + } + mut := bigtable.NewMutation() + for family, items := range row { + for _, item := range items { + // Get the column name, excluding the column family name and ':' character. + columnWithoutFamily := item.Column[len(family)+1:] + mut.Set(family, columnWithoutFamily, bigtable.Now(), item.Value) + } + } + wg.Add(1) + go func() { + // TODO: should use a semaphore to limit the number of concurrent writes. + if err := table.Apply(ctx, row.Key(), mut); err != nil { + mu.Lock() + writeErr = err + mu.Unlock() + } + wg.Done() + }() + return true + } + + // Create a filter that only accepts the column families we're interested in. + filter := bigtable.FamilyFilter(indexColumnFamily + "|" + contentColumnFamily) + // Read every row from prototypeTable, and call copyRowToTable to copy it to our table. + err := prototypeTable.ReadRows(ctx, bigtable.InfiniteRange(""), copyRowToTable, bigtable.RowFilter(filter)) + wg.Wait() + if err != nil { + return err + } + return writeErr +} + +// handleClear calls rebuildTable +func handleClear(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "POST requests only", http.StatusMethodNotAllowed) + return + } + if err := rebuildTable(); err != nil { + http.Error(w, "Failed to rebuild index: "+err.Error(), http.StatusInternalServerError) + return + } + fmt.Fprint(w, "Rebuilt index.\n") +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/pubsub/cmdline/main.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/pubsub/cmdline/main.go new file mode 100644 index 0000000000..c51614d1ce --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/pubsub/cmdline/main.go @@ -0,0 +1,354 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main contains a simple command line tool for Cloud Pub/Sub +// Cloud Pub/Sub docs: https://cloud.google.com/pubsub/docs +package main + +import ( + "errors" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + "time" + + "golang.org/x/net/context" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" + "google.golang.org/cloud/compute/metadata" + "google.golang.org/cloud/pubsub" +) + +var ( + jsonFile = flag.String("j", "", "A path to your JSON key file for your service account downloaded from Google Developer Console, not needed if you run it on Compute Engine instances.") + projID = flag.String("p", "", "The ID of your Google Cloud project.") + reportMPS = flag.Bool("report", false, "Reports the incoming/outgoing message rate in msg/sec if set.") + size = flag.Int("size", 10, "Batch size for pull_messages and publish_messages subcommands.") +) + +const ( + usage = `Available arguments are: + create_topic + delete_topic + create_subscription + delete_subscription + publish + pull_messages + publish_messages +` + tick = 1 * time.Second +) + +func usageAndExit(msg string) { + fmt.Fprintln(os.Stderr, msg) + fmt.Println("Flags:") + flag.PrintDefaults() + fmt.Fprint(os.Stderr, usage) + os.Exit(2) +} + +// Check the length of the arguments. +func checkArgs(argv []string, min int) { + if len(argv) < min { + usageAndExit("Missing arguments") + } +} + +// newClient creates http.Client with a jwt service account when +// jsonFile flag is specified, otherwise by obtaining the GCE service +// account's access token. +func newClient(jsonFile string) (*http.Client, error) { + if jsonFile != "" { + jsonKey, err := ioutil.ReadFile(jsonFile) + if err != nil { + return nil, err + } + conf, err := google.JWTConfigFromJSON(jsonKey, pubsub.ScopePubSub) + if err != nil { + return nil, err + } + return conf.Client(oauth2.NoContext), nil + } + if metadata.OnGCE() { + c := &http.Client{ + Transport: &oauth2.Transport{ + Source: google.ComputeTokenSource(""), + }, + } + if *projID == "" { + projectID, err := metadata.ProjectID() + if err != nil { + return nil, fmt.Errorf("ProjectID failed, %v", err) + } + *projID = projectID + } + return c, nil + } + return nil, errors.New("Could not create an authenticated client.") +} + +func listTopics(ctx context.Context, argv []string) { + panic("listTopics not implemented yet") +} + +func createTopic(ctx context.Context, argv []string) { + checkArgs(argv, 2) + topic := argv[1] + err := pubsub.CreateTopic(ctx, topic) + if err != nil { + log.Fatalf("CreateTopic failed, %v", err) + } + fmt.Printf("Topic %s was created.\n", topic) +} + +func deleteTopic(ctx context.Context, argv []string) { + checkArgs(argv, 2) + topic := argv[1] + err := pubsub.DeleteTopic(ctx, topic) + if err != nil { + log.Fatalf("DeleteTopic failed, %v", err) + } + fmt.Printf("Topic %s was deleted.\n", topic) +} + +func listSubscriptions(ctx context.Context, argv []string) { + panic("listSubscriptions not implemented yet") +} + +func createSubscription(ctx context.Context, argv []string) { + checkArgs(argv, 3) + sub := argv[1] + topic := argv[2] + err := pubsub.CreateSub(ctx, sub, topic, 60*time.Second, "") + if err != nil { + log.Fatalf("CreateSub failed, %v", err) + } + fmt.Printf("Subscription %s was created.\n", sub) +} + +func deleteSubscription(ctx context.Context, argv []string) { + checkArgs(argv, 2) + sub := argv[1] + err := pubsub.DeleteSub(ctx, sub) + if err != nil { + log.Fatalf("DeleteSub failed, %v", err) + } + fmt.Printf("Subscription %s was deleted.\n", sub) +} + +func publish(ctx context.Context, argv []string) { + checkArgs(argv, 3) + topic := argv[1] + message := argv[2] + msgIDs, err := pubsub.Publish(ctx, topic, &pubsub.Message{ + Data: []byte(message), + }) + if err != nil { + log.Fatalf("Publish failed, %v", err) + } + fmt.Printf("Message '%s' published to a topic %s and the message id is %s\n", message, topic, msgIDs[0]) +} + +type reporter struct { + reportTitle string + lastC uint64 + c uint64 + result <-chan int +} + +func (r *reporter) report() { + ticker := time.NewTicker(tick) + defer func() { + ticker.Stop() + }() + for { + select { + case <-ticker.C: + n := r.c - r.lastC + r.lastC = r.c + mps := n / uint64(tick/time.Second) + log.Printf("%s ~%d msgs/s, total: %d", r.reportTitle, mps, r.c) + case n := <-r.result: + r.c += uint64(n) + } + } +} + +func ack(ctx context.Context, sub string, ackID ...string) { + err := pubsub.Ack(ctx, sub, ackID...) + if err != nil { + log.Printf("Ack failed, %v\n", err) + } +} + +func pullLoop(ctx context.Context, sub string, result chan<- int) { + for { + msgs, err := pubsub.PullWait(ctx, sub, *size) + if err != nil { + log.Printf("PullWait failed, %v\n", err) + time.Sleep(5 * time.Second) + continue + } + if len(msgs) == 0 { + log.Println("Received no messages") + continue + } + if *reportMPS { + result <- len(msgs) + } + ackIDs := make([]string, len(msgs)) + for i, msg := range msgs { + if !*reportMPS { + fmt.Printf("Got a message: %s\n", msg.Data) + } + ackIDs[i] = msg.AckID + } + go ack(ctx, sub, ackIDs...) + } +} + +func pullMessages(ctx context.Context, argv []string) { + checkArgs(argv, 3) + sub := argv[1] + workers, err := strconv.Atoi(argv[2]) + if err != nil { + log.Fatalf("Atoi failed, %v", err) + } + result := make(chan int, 1024) + for i := 0; i < int(workers); i++ { + go pullLoop(ctx, sub, result) + } + if *reportMPS { + r := reporter{reportTitle: "Received", result: result} + r.report() + } else { + select {} + } +} + +func publishLoop(ctx context.Context, topic string, workerid int, result chan<- int) { + var r uint64 + for { + msgs := make([]*pubsub.Message, *size) + for i := 0; i < *size; i++ { + msgs[i] = &pubsub.Message{ + Data: []byte(fmt.Sprintf("Worker: %d, Round: %d, Message: %d", workerid, r, i)), + } + } + _, err := pubsub.Publish(ctx, topic, msgs...) + if err != nil { + log.Printf("Publish failed, %v\n", err) + return + } + r++ + if *reportMPS { + result <- *size + } + } +} + +func publishMessages(ctx context.Context, argv []string) { + checkArgs(argv, 3) + topic := argv[1] + workers, err := strconv.Atoi(argv[2]) + if err != nil { + log.Fatalf("Atoi failed, %v", err) + } + result := make(chan int, 1024) + for i := 0; i < int(workers); i++ { + go publishLoop(ctx, topic, i, result) + } + if *reportMPS { + r := reporter{reportTitle: "Sent", result: result} + r.report() + } else { + select {} + } +} + +// This example demonstrates calling the Cloud Pub/Sub API. As of 22 +// Oct 2014, the Cloud Pub/Sub API is only available if you're +// whitelisted. If you're interested in using it, please apply for the +// Limited Preview program at the following form: +// http://goo.gl/Wql9HL +// +// Also, before running this example, be sure to enable Cloud Pub/Sub +// service on your project in Developer Console at: +// https://console.developers.google.com/ +// +// Unless you run this sample on Compute Engine instance, please +// create a new service account and download a JSON key file for it at +// the developer console: https://console.developers.google.com/ +// +// It has the following subcommands: +// +// create_topic +// delete_topic +// create_subscription +// delete_subscription +// publish +// pull_messages +// publish_messages +// +// You can choose any names for topic and subscription as long as they +// follow the naming rule described at: +// https://cloud.google.com/pubsub/overview#names +// +// You can create/delete topics/subscriptions by self-explanatory +// subcommands. +// +// The "publish" subcommand is for publishing a single message to a +// specified Cloud Pub/Sub topic. +// +// The "pull_messages" subcommand is for continuously pulling messages +// from a specified Cloud Pub/Sub subscription with specified number +// of workers. +// +// The "publish_messages" subcommand is for continuously publishing +// messages to a specified Cloud Pub/Sub topic with specified number +// of workers. +func main() { + flag.Parse() + argv := flag.Args() + checkArgs(argv, 1) + client, err := newClient(*jsonFile) + if err != nil { + log.Fatalf("clientAndId failed, %v", err) + } + if *projID == "" { + usageAndExit("Please specify Project ID.") + } + ctx := cloud.NewContext(*projID, client) + m := map[string]func(ctx context.Context, argv []string){ + "create_topic": createTopic, + "delete_topic": deleteTopic, + "create_subscription": createSubscription, + "delete_subscription": deleteSubscription, + "publish": publish, + "pull_messages": pullMessages, + "publish_messages": publishMessages, + } + subcommand := argv[0] + f, ok := m[subcommand] + if !ok { + usageAndExit(fmt.Sprintf("Function not found for %s", subcommand)) + } + f(ctx, argv) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appengine/app.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appengine/app.go new file mode 100644 index 0000000000..0889f12562 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appengine/app.go @@ -0,0 +1,401 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gcsdemo is an example App Engine app using the Google Cloud Storage API. +package gcsdemo + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/appengine" + "google.golang.org/appengine/file" + "google.golang.org/appengine/log" + "google.golang.org/appengine/urlfetch" + "google.golang.org/cloud" + "google.golang.org/cloud/storage" +) + +// bucket is a local cache of the app's default bucket name. +var bucket string // or: var bucket = ".appspot.com" + +func init() { + http.HandleFunc("/", handler) +} + +// demo struct holds information needed to run the various demo functions. +type demo struct { + c context.Context + w http.ResponseWriter + ctx context.Context + // cleanUp is a list of filenames that need cleaning up at the end of the demo. + cleanUp []string + // failed indicates that one or more of the demo steps failed. + failed bool +} + +func (d *demo) errorf(format string, args ...interface{}) { + d.failed = true + log.Errorf(d.c, format, args...) +} + +// handler is the main demo entry point that calls the GCS operations. +func handler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + c := appengine.NewContext(r) + if bucket == "" { + var err error + if bucket, err = file.DefaultBucketName(c); err != nil { + log.Errorf(c, "failed to get default GCS bucket name: %v", err) + return + } + } + hc := &http.Client{ + Transport: &oauth2.Transport{ + Source: google.AppEngineTokenSource(c, storage.ScopeFullControl), + // Note that the App Engine urlfetch service has a limit of 10MB uploads and + // 32MB downloads. + // See https://cloud.google.com/appengine/docs/go/urlfetch/#Go_Quotas_and_limits + // for more information. + Base: &urlfetch.Transport{Context: c}, + }, + } + ctx := cloud.NewContext(appengine.AppID(c), hc) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c)) + fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket) + + d := &demo{ + c: c, + w: w, + ctx: ctx, + } + + n := "demo-testfile-go" + d.createFile(n) + d.readFile(n) + d.copyFile(n) + d.statFile(n) + d.createListFiles() + d.listBucket() + d.listBucketDirMode() + d.defaultACL() + d.putDefaultACLRule() + d.deleteDefaultACLRule() + d.bucketACL() + d.putBucketACLRule() + d.deleteBucketACLRule() + d.acl(n) + d.putACLRule(n) + d.deleteACLRule(n) + d.deleteFiles() + + if d.failed { + io.WriteString(w, "\nDemo failed.\n") + } else { + io.WriteString(w, "\nDemo succeeded.\n") + } +} + +// createFile creates a file in Google Cloud Storage. +func (d *demo) createFile(fileName string) { + fmt.Fprintf(d.w, "Creating file /%v/%v\n", bucket, fileName) + + wc := storage.NewWriter(d.ctx, bucket, fileName) + wc.ContentType = "text/plain" + wc.Metadata = map[string]string{ + "x-goog-meta-foo": "foo", + "x-goog-meta-bar": "bar", + } + d.cleanUp = append(d.cleanUp, fileName) + + if _, err := wc.Write([]byte("abcde\n")); err != nil { + d.errorf("createFile: unable to write data to bucket %q, file %q: %v", bucket, fileName, err) + return + } + if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil { + d.errorf("createFile: unable to write data to bucket %q, file %q: %v", bucket, fileName, err) + return + } + if err := wc.Close(); err != nil { + d.errorf("createFile: unable to close bucket %q, file %q: %v", bucket, fileName, err) + return + } +} + +// readFile reads the named file in Google Cloud Storage. +func (d *demo) readFile(fileName string) { + io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n") + + rc, err := storage.NewReader(d.ctx, bucket, fileName) + if err != nil { + d.errorf("readFile: unable to open file from bucket %q, file %q: %v", bucket, fileName, err) + return + } + defer rc.Close() + slurp, err := ioutil.ReadAll(rc) + if err != nil { + d.errorf("readFile: unable to read data from bucket %q, file %q: %v", bucket, fileName, err) + return + } + + fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0]) + if len(slurp) > 1024 { + fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:]) + } else { + fmt.Fprintf(d.w, "%s\n", slurp) + } +} + +// copyFile copies a file in Google Cloud Storage. +func (d *demo) copyFile(fileName string) { + copyName := fileName + "-copy" + fmt.Fprintf(d.w, "Copying file /%v/%v to /%v/%v:\n", bucket, fileName, bucket, copyName) + + obj, err := storage.CopyObject(d.ctx, bucket, fileName, bucket, copyName, nil) + if err != nil { + d.errorf("copyFile: unable to copy /%v/%v to bucket %q, file %q: %v", bucket, fileName, bucket, copyName, err) + return + } + d.cleanUp = append(d.cleanUp, copyName) + + d.dumpStats(obj) +} + +func (d *demo) dumpStats(obj *storage.Object) { + fmt.Fprintf(d.w, "(filename: /%v/%v, ", obj.Bucket, obj.Name) + fmt.Fprintf(d.w, "ContentType: %q, ", obj.ContentType) + fmt.Fprintf(d.w, "ACL: %#v, ", obj.ACL) + fmt.Fprintf(d.w, "Owner: %v, ", obj.Owner) + fmt.Fprintf(d.w, "ContentEncoding: %q, ", obj.ContentEncoding) + fmt.Fprintf(d.w, "Size: %v, ", obj.Size) + fmt.Fprintf(d.w, "MD5: %q, ", obj.MD5) + fmt.Fprintf(d.w, "CRC32C: %q, ", obj.CRC32C) + fmt.Fprintf(d.w, "Metadata: %#v, ", obj.Metadata) + fmt.Fprintf(d.w, "MediaLink: %q, ", obj.MediaLink) + fmt.Fprintf(d.w, "StorageClass: %q, ", obj.StorageClass) + if !obj.Deleted.IsZero() { + fmt.Fprintf(d.w, "Deleted: %v, ", obj.Deleted) + } + fmt.Fprintf(d.w, "Updated: %v)\n", obj.Updated) +} + +// statFile reads the stats of the named file in Google Cloud Storage. +func (d *demo) statFile(fileName string) { + io.WriteString(d.w, "\nFile stat:\n") + + obj, err := storage.StatObject(d.ctx, bucket, fileName) + if err != nil { + d.errorf("statFile: unable to stat file from bucket %q, file %q: %v", bucket, fileName, err) + return + } + + d.dumpStats(obj) +} + +// createListFiles creates files that will be used by listBucket. +func (d *demo) createListFiles() { + io.WriteString(d.w, "\nCreating more files for listbucket...\n") + for _, n := range []string{"foo1", "foo2", "bar", "bar/1", "bar/2", "boo/"} { + d.createFile(n) + } +} + +// listBucket lists the contents of a bucket in Google Cloud Storage. +func (d *demo) listBucket() { + io.WriteString(d.w, "\nListbucket result:\n") + + query := &storage.Query{Prefix: "foo"} + for query != nil { + objs, err := storage.ListObjects(d.ctx, bucket, query) + if err != nil { + d.errorf("listBucket: unable to list bucket %q: %v", bucket, err) + return + } + query = objs.Next + + for _, obj := range objs.Results { + d.dumpStats(obj) + } + } +} + +func (d *demo) listDir(name, indent string) { + query := &storage.Query{Prefix: name, Delimiter: "/"} + for query != nil { + objs, err := storage.ListObjects(d.ctx, bucket, query) + if err != nil { + d.errorf("listBucketDirMode: unable to list bucket %q: %v", bucket, err) + return + } + query = objs.Next + + for _, obj := range objs.Results { + fmt.Fprint(d.w, indent) + d.dumpStats(obj) + } + for _, dir := range objs.Prefixes { + fmt.Fprintf(d.w, "%v(directory: /%v/%v)\n", indent, bucket, dir) + d.listDir(dir, indent+" ") + } + } +} + +// listBucketDirMode lists the contents of a bucket in dir mode in Google Cloud Storage. +func (d *demo) listBucketDirMode() { + io.WriteString(d.w, "\nListbucket directory mode result:\n") + d.listDir("b", "") +} + +// dumpDefaultACL prints out the default object ACL for this bucket. +func (d *demo) dumpDefaultACL() { + acl, err := storage.DefaultACL(d.ctx, bucket) + if err != nil { + d.errorf("defaultACL: unable to list default object ACL for bucket %q: %v", bucket, err) + return + } + for _, v := range acl { + fmt.Fprintf(d.w, "Entity: %q, Role: %q\n", v.Entity, v.Role) + } +} + +// defaultACL displays the default object ACL for this bucket. +func (d *demo) defaultACL() { + io.WriteString(d.w, "\nDefault object ACL:\n") + d.dumpDefaultACL() +} + +// putDefaultACLRule adds the "allUsers" default object ACL rule for this bucket. +func (d *demo) putDefaultACLRule() { + io.WriteString(d.w, "\nPut Default object ACL Rule:\n") + err := storage.PutDefaultACLRule(d.ctx, bucket, "allUsers", storage.RoleReader) + if err != nil { + d.errorf("putDefaultACLRule: unable to save default object ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpDefaultACL() +} + +// deleteDefaultACLRule deleted the "allUsers" default object ACL rule for this bucket. +func (d *demo) deleteDefaultACLRule() { + io.WriteString(d.w, "\nDelete Default object ACL Rule:\n") + err := storage.DeleteDefaultACLRule(d.ctx, bucket, "allUsers") + if err != nil { + d.errorf("deleteDefaultACLRule: unable to delete default object ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpDefaultACL() +} + +// dumpBucketACL prints out the bucket ACL. +func (d *demo) dumpBucketACL() { + acl, err := storage.BucketACL(d.ctx, bucket) + if err != nil { + d.errorf("dumpBucketACL: unable to list bucket ACL for bucket %q: %v", bucket, err) + return + } + for _, v := range acl { + fmt.Fprintf(d.w, "Entity: %q, Role: %q\n", v.Entity, v.Role) + } +} + +// bucketACL displays the bucket ACL for this bucket. +func (d *demo) bucketACL() { + io.WriteString(d.w, "\nBucket ACL:\n") + d.dumpBucketACL() +} + +// putBucketACLRule adds the "allUsers" bucket ACL rule for this bucket. +func (d *demo) putBucketACLRule() { + io.WriteString(d.w, "\nPut Bucket ACL Rule:\n") + err := storage.PutBucketACLRule(d.ctx, bucket, "allUsers", storage.RoleReader) + if err != nil { + d.errorf("putBucketACLRule: unable to save bucket ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpBucketACL() +} + +// deleteBucketACLRule deleted the "allUsers" bucket ACL rule for this bucket. +func (d *demo) deleteBucketACLRule() { + io.WriteString(d.w, "\nDelete Bucket ACL Rule:\n") + err := storage.DeleteBucketACLRule(d.ctx, bucket, "allUsers") + if err != nil { + d.errorf("deleteBucketACLRule: unable to delete bucket ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpBucketACL() +} + +// dumpACL prints out the ACL of the named file. +func (d *demo) dumpACL(fileName string) { + acl, err := storage.ACL(d.ctx, bucket, fileName) + if err != nil { + d.errorf("dumpACL: unable to list file ACL for bucket %q, file %q: %v", bucket, fileName, err) + return + } + for _, v := range acl { + fmt.Fprintf(d.w, "Entity: %q, Role: %q\n", v.Entity, v.Role) + } +} + +// acl displays the ACL for the named file. +func (d *demo) acl(fileName string) { + fmt.Fprintf(d.w, "\nACL for file %v:\n", fileName) + d.dumpACL(fileName) +} + +// putACLRule adds the "allUsers" ACL rule for the named file. +func (d *demo) putACLRule(fileName string) { + fmt.Fprintf(d.w, "\nPut ACL rule for file %v:\n", fileName) + err := storage.PutACLRule(d.ctx, bucket, fileName, "allUsers", storage.RoleReader) + if err != nil { + d.errorf("putACLRule: unable to save ACL rule for bucket %q, file %q: %v", bucket, fileName, err) + return + } + d.dumpACL(fileName) +} + +// deleteACLRule deleted the "allUsers" ACL rule for the named file. +func (d *demo) deleteACLRule(fileName string) { + fmt.Fprintf(d.w, "\nDelete ACL rule for file %v:\n", fileName) + err := storage.DeleteACLRule(d.ctx, bucket, fileName, "allUsers") + if err != nil { + d.errorf("deleteACLRule: unable to delete ACL rule for bucket %q, file %q: %v", bucket, fileName, err) + return + } + d.dumpACL(fileName) +} + +// deleteFiles deletes all the temporary files from a bucket created by this demo. +func (d *demo) deleteFiles() { + io.WriteString(d.w, "\nDeleting files...\n") + for _, v := range d.cleanUp { + fmt.Fprintf(d.w, "Deleting file %v\n", v) + if err := storage.DeleteObject(d.ctx, bucket, v); err != nil { + d.errorf("deleteFiles: unable to delete bucket %q, file %q: %v", bucket, v, err) + return + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appengine/app.yaml b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appengine/app.yaml new file mode 100644 index 0000000000..e98d030876 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appengine/app.yaml @@ -0,0 +1,8 @@ +application: +version: v1 +runtime: go +api_version: go1 + +handlers: +- url: /.* + script: _go_app diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appenginevm/app.go b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appenginevm/app.go new file mode 100644 index 0000000000..47c7e927bb --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appenginevm/app.go @@ -0,0 +1,396 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main is an example Mananged VM app using the Google Cloud Storage API. +package main + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/appengine" + "google.golang.org/appengine/file" + "google.golang.org/appengine/log" + "google.golang.org/cloud" + "google.golang.org/cloud/storage" +) + +// bucket is a local cache of the app's default bucket name. +var bucket string // or: var bucket = ".appspot.com" + +func main() { + http.HandleFunc("/", handler) + appengine.Main() +} + +// demo struct holds information needed to run the various demo functions. +type demo struct { + c context.Context + w http.ResponseWriter + ctx context.Context + // cleanUp is a list of filenames that need cleaning up at the end of the demo. + cleanUp []string + // failed indicates that one or more of the demo steps failed. + failed bool +} + +func (d *demo) errorf(format string, args ...interface{}) { + d.failed = true + log.Errorf(d.c, format, args...) +} + +// handler is the main demo entry point that calls the GCS operations. +func handler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + c := appengine.NewContext(r) + if bucket == "" { + var err error + if bucket, err = file.DefaultBucketName(c); err != nil { + log.Errorf(c, "failed to get default GCS bucket name: %v", err) + return + } + } + hc := &http.Client{ + Transport: &oauth2.Transport{ + Source: google.AppEngineTokenSource(c, storage.ScopeFullControl), + }, + } + ctx := cloud.NewContext(appengine.AppID(c), hc) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c)) + fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket) + + d := &demo{ + c: c, + w: w, + ctx: ctx, + } + + n := "demo-testfile-go" + d.createFile(n) + d.readFile(n) + d.copyFile(n) + d.statFile(n) + d.createListFiles() + d.listBucket() + d.listBucketDirMode() + d.defaultACL() + d.putDefaultACLRule() + d.deleteDefaultACLRule() + d.bucketACL() + d.putBucketACLRule() + d.deleteBucketACLRule() + d.acl(n) + d.putACLRule(n) + d.deleteACLRule(n) + d.deleteFiles() + + if d.failed { + io.WriteString(w, "\nDemo failed.\n") + } else { + io.WriteString(w, "\nDemo succeeded.\n") + } +} + +// createFile creates a file in Google Cloud Storage. +func (d *demo) createFile(fileName string) { + fmt.Fprintf(d.w, "Creating file /%v/%v\n", bucket, fileName) + + wc := storage.NewWriter(d.ctx, bucket, fileName) + wc.ContentType = "text/plain" + wc.Metadata = map[string]string{ + "x-goog-meta-foo": "foo", + "x-goog-meta-bar": "bar", + } + d.cleanUp = append(d.cleanUp, fileName) + + if _, err := wc.Write([]byte("abcde\n")); err != nil { + d.errorf("createFile: unable to write data to bucket %q, file %q: %v", bucket, fileName, err) + return + } + if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil { + d.errorf("createFile: unable to write data to bucket %q, file %q: %v", bucket, fileName, err) + return + } + if err := wc.Close(); err != nil { + d.errorf("createFile: unable to close bucket %q, file %q: %v", bucket, fileName, err) + return + } +} + +// readFile reads the named file in Google Cloud Storage. +func (d *demo) readFile(fileName string) { + io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n") + + rc, err := storage.NewReader(d.ctx, bucket, fileName) + if err != nil { + d.errorf("readFile: unable to open file from bucket %q, file %q: %v", bucket, fileName, err) + return + } + defer rc.Close() + slurp, err := ioutil.ReadAll(rc) + if err != nil { + d.errorf("readFile: unable to read data from bucket %q, file %q: %v", bucket, fileName, err) + return + } + + fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0]) + if len(slurp) > 1024 { + fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:]) + } else { + fmt.Fprintf(d.w, "%s\n", slurp) + } +} + +// copyFile copies a file in Google Cloud Storage. +func (d *demo) copyFile(fileName string) { + copyName := fileName + "-copy" + fmt.Fprintf(d.w, "Copying file /%v/%v to /%v/%v:\n", bucket, fileName, bucket, copyName) + + obj, err := storage.CopyObject(d.ctx, bucket, fileName, bucket, copyName, nil) + if err != nil { + d.errorf("copyFile: unable to copy /%v/%v to bucket %q, file %q: %v", bucket, fileName, bucket, copyName, err) + return + } + d.cleanUp = append(d.cleanUp, copyName) + + d.dumpStats(obj) +} + +func (d *demo) dumpStats(obj *storage.Object) { + fmt.Fprintf(d.w, "(filename: /%v/%v, ", obj.Bucket, obj.Name) + fmt.Fprintf(d.w, "ContentType: %q, ", obj.ContentType) + fmt.Fprintf(d.w, "ACL: %#v, ", obj.ACL) + fmt.Fprintf(d.w, "Owner: %v, ", obj.Owner) + fmt.Fprintf(d.w, "ContentEncoding: %q, ", obj.ContentEncoding) + fmt.Fprintf(d.w, "Size: %v, ", obj.Size) + fmt.Fprintf(d.w, "MD5: %q, ", obj.MD5) + fmt.Fprintf(d.w, "CRC32C: %q, ", obj.CRC32C) + fmt.Fprintf(d.w, "Metadata: %#v, ", obj.Metadata) + fmt.Fprintf(d.w, "MediaLink: %q, ", obj.MediaLink) + fmt.Fprintf(d.w, "StorageClass: %q, ", obj.StorageClass) + if !obj.Deleted.IsZero() { + fmt.Fprintf(d.w, "Deleted: %v, ", obj.Deleted) + } + fmt.Fprintf(d.w, "Updated: %v)\n", obj.Updated) +} + +// statFile reads the stats of the named file in Google Cloud Storage. +func (d *demo) statFile(fileName string) { + io.WriteString(d.w, "\nFile stat:\n") + + obj, err := storage.StatObject(d.ctx, bucket, fileName) + if err != nil { + d.errorf("statFile: unable to stat file from bucket %q, file %q: %v", bucket, fileName, err) + return + } + + d.dumpStats(obj) +} + +// createListFiles creates files that will be used by listBucket. +func (d *demo) createListFiles() { + io.WriteString(d.w, "\nCreating more files for listbucket...\n") + for _, n := range []string{"foo1", "foo2", "bar", "bar/1", "bar/2", "boo/"} { + d.createFile(n) + } +} + +// listBucket lists the contents of a bucket in Google Cloud Storage. +func (d *demo) listBucket() { + io.WriteString(d.w, "\nListbucket result:\n") + + query := &storage.Query{Prefix: "foo"} + for query != nil { + objs, err := storage.ListObjects(d.ctx, bucket, query) + if err != nil { + d.errorf("listBucket: unable to list bucket %q: %v", bucket, err) + return + } + query = objs.Next + + for _, obj := range objs.Results { + d.dumpStats(obj) + } + } +} + +func (d *demo) listDir(name, indent string) { + query := &storage.Query{Prefix: name, Delimiter: "/"} + for query != nil { + objs, err := storage.ListObjects(d.ctx, bucket, query) + if err != nil { + d.errorf("listBucketDirMode: unable to list bucket %q: %v", bucket, err) + return + } + query = objs.Next + + for _, obj := range objs.Results { + fmt.Fprint(d.w, indent) + d.dumpStats(obj) + } + for _, dir := range objs.Prefixes { + fmt.Fprintf(d.w, "%v(directory: /%v/%v)\n", indent, bucket, dir) + d.listDir(dir, indent+" ") + } + } +} + +// listBucketDirMode lists the contents of a bucket in dir mode in Google Cloud Storage. +func (d *demo) listBucketDirMode() { + io.WriteString(d.w, "\nListbucket directory mode result:\n") + d.listDir("b", "") +} + +// dumpDefaultACL prints out the default object ACL for this bucket. +func (d *demo) dumpDefaultACL() { + acl, err := storage.DefaultACL(d.ctx, bucket) + if err != nil { + d.errorf("defaultACL: unable to list default object ACL for bucket %q: %v", bucket, err) + return + } + for _, v := range acl { + fmt.Fprintf(d.w, "Entity: %q, Role: %q\n", v.Entity, v.Role) + } +} + +// defaultACL displays the default object ACL for this bucket. +func (d *demo) defaultACL() { + io.WriteString(d.w, "\nDefault object ACL:\n") + d.dumpDefaultACL() +} + +// putDefaultACLRule adds the "allUsers" default object ACL rule for this bucket. +func (d *demo) putDefaultACLRule() { + io.WriteString(d.w, "\nPut Default object ACL Rule:\n") + err := storage.PutDefaultACLRule(d.ctx, bucket, "allUsers", storage.RoleReader) + if err != nil { + d.errorf("putDefaultACLRule: unable to save default object ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpDefaultACL() +} + +// deleteDefaultACLRule deleted the "allUsers" default object ACL rule for this bucket. +func (d *demo) deleteDefaultACLRule() { + io.WriteString(d.w, "\nDelete Default object ACL Rule:\n") + err := storage.DeleteDefaultACLRule(d.ctx, bucket, "allUsers") + if err != nil { + d.errorf("deleteDefaultACLRule: unable to delete default object ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpDefaultACL() +} + +// dumpBucketACL prints out the bucket ACL. +func (d *demo) dumpBucketACL() { + acl, err := storage.BucketACL(d.ctx, bucket) + if err != nil { + d.errorf("dumpBucketACL: unable to list bucket ACL for bucket %q: %v", bucket, err) + return + } + for _, v := range acl { + fmt.Fprintf(d.w, "Entity: %q, Role: %q\n", v.Entity, v.Role) + } +} + +// bucketACL displays the bucket ACL for this bucket. +func (d *demo) bucketACL() { + io.WriteString(d.w, "\nBucket ACL:\n") + d.dumpBucketACL() +} + +// putBucketACLRule adds the "allUsers" bucket ACL rule for this bucket. +func (d *demo) putBucketACLRule() { + io.WriteString(d.w, "\nPut Bucket ACL Rule:\n") + err := storage.PutBucketACLRule(d.ctx, bucket, "allUsers", storage.RoleReader) + if err != nil { + d.errorf("putBucketACLRule: unable to save bucket ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpBucketACL() +} + +// deleteBucketACLRule deleted the "allUsers" bucket ACL rule for this bucket. +func (d *demo) deleteBucketACLRule() { + io.WriteString(d.w, "\nDelete Bucket ACL Rule:\n") + err := storage.DeleteBucketACLRule(d.ctx, bucket, "allUsers") + if err != nil { + d.errorf("deleteBucketACLRule: unable to delete bucket ACL rule for bucket %q: %v", bucket, err) + return + } + d.dumpBucketACL() +} + +// dumpACL prints out the ACL of the named file. +func (d *demo) dumpACL(fileName string) { + acl, err := storage.ACL(d.ctx, bucket, fileName) + if err != nil { + d.errorf("dumpACL: unable to list file ACL for bucket %q, file %q: %v", bucket, fileName, err) + return + } + for _, v := range acl { + fmt.Fprintf(d.w, "Entity: %q, Role: %q\n", v.Entity, v.Role) + } +} + +// acl displays the ACL for the named file. +func (d *demo) acl(fileName string) { + fmt.Fprintf(d.w, "\nACL for file %v:\n", fileName) + d.dumpACL(fileName) +} + +// putACLRule adds the "allUsers" ACL rule for the named file. +func (d *demo) putACLRule(fileName string) { + fmt.Fprintf(d.w, "\nPut ACL rule for file %v:\n", fileName) + err := storage.PutACLRule(d.ctx, bucket, fileName, "allUsers", storage.RoleReader) + if err != nil { + d.errorf("putACLRule: unable to save ACL rule for bucket %q, file %q: %v", bucket, fileName, err) + return + } + d.dumpACL(fileName) +} + +// deleteACLRule deleted the "allUsers" ACL rule for the named file. +func (d *demo) deleteACLRule(fileName string) { + fmt.Fprintf(d.w, "\nDelete ACL rule for file %v:\n", fileName) + err := storage.DeleteACLRule(d.ctx, bucket, fileName, "allUsers") + if err != nil { + d.errorf("deleteACLRule: unable to delete ACL rule for bucket %q, file %q: %v", bucket, fileName, err) + return + } + d.dumpACL(fileName) +} + +// deleteFiles deletes all the temporary files from a bucket created by this demo. +func (d *demo) deleteFiles() { + io.WriteString(d.w, "\nDeleting files...\n") + for _, v := range d.cleanUp { + fmt.Fprintf(d.w, "Deleting file %v\n", v) + if err := storage.DeleteObject(d.ctx, bucket, v); err != nil { + d.errorf("deleteFiles: unable to delete bucket %q, file %q: %v", bucket, v, err) + return + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appenginevm/app.yaml b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appenginevm/app.yaml new file mode 100644 index 0000000000..6847fc8715 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/examples/storage/appenginevm/app.yaml @@ -0,0 +1,10 @@ +runtime: go +api_version: go1 +vm: true + +manual_scaling: + instances: 1 + +handlers: +- url: /.* + script: _go_app diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/cloud.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/cloud.go new file mode 100644 index 0000000000..8b0db1b5da --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/cloud.go @@ -0,0 +1,128 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package internal provides support for the cloud packages. +// +// Users should not import this package directly. +package internal + +import ( + "fmt" + "net/http" + "sync" + + "golang.org/x/net/context" +) + +type contextKey struct{} + +func WithContext(parent context.Context, projID string, c *http.Client) context.Context { + if c == nil { + panic("nil *http.Client passed to WithContext") + } + if projID == "" { + panic("empty project ID passed to WithContext") + } + return context.WithValue(parent, contextKey{}, &cloudContext{ + ProjectID: projID, + HTTPClient: c, + }) +} + +const userAgent = "gcloud-golang/0.1" + +type cloudContext struct { + ProjectID string + HTTPClient *http.Client + + mu sync.Mutex // guards svc + svc map[string]interface{} // e.g. "storage" => *rawStorage.Service +} + +// Service returns the result of the fill function if it's never been +// called before for the given name (which is assumed to be an API +// service name, like "datastore"). If it has already been cached, the fill +// func is not run. +// It's safe for concurrent use by multiple goroutines. +func Service(ctx context.Context, name string, fill func(*http.Client) interface{}) interface{} { + return cc(ctx).service(name, fill) +} + +func (c *cloudContext) service(name string, fill func(*http.Client) interface{}) interface{} { + c.mu.Lock() + defer c.mu.Unlock() + + if c.svc == nil { + c.svc = make(map[string]interface{}) + } else if v, ok := c.svc[name]; ok { + return v + } + v := fill(c.HTTPClient) + c.svc[name] = v + return v +} + +// Transport is an http.RoundTripper that appends +// Google Cloud client's user-agent to the original +// request's user-agent header. +type Transport struct { + // Base represents the actual http.RoundTripper + // the requests will be delegated to. + Base http.RoundTripper +} + +// RoundTrip appends a user-agent to the existing user-agent +// header and delegates the request to the base http.RoundTripper. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + req = cloneRequest(req) + ua := req.Header.Get("User-Agent") + if ua == "" { + ua = userAgent + } else { + ua = fmt.Sprintf("%s %s", ua, userAgent) + } + req.Header.Set("User-Agent", ua) + return t.Base.RoundTrip(req) +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header) + for k, s := range r.Header { + r2.Header[k] = s + } + return r2 +} + +func ProjID(ctx context.Context) string { + return cc(ctx).ProjectID +} + +func HTTPClient(ctx context.Context) *http.Client { + return cc(ctx).HTTPClient +} + +// cc returns the internal *cloudContext (cc) state for a context.Context. +// It panics if the user did it wrong. +func cc(ctx context.Context) *cloudContext { + if c, ok := ctx.Value(contextKey{}).(*cloudContext); ok { + return c + } + panic("invalid context.Context type; it should be created with cloud.NewContext") +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/datastore/datastore_v1.pb.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/datastore/datastore_v1.pb.go new file mode 100644 index 0000000000..9cb9be528c --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/datastore/datastore_v1.pb.go @@ -0,0 +1,1633 @@ +// Code generated by protoc-gen-go. +// source: datastore_v1.proto +// DO NOT EDIT! + +/* +Package datastore is a generated protocol buffer package. + +It is generated from these files: + datastore_v1.proto + +It has these top-level messages: + PartitionId + Key + Value + Property + Entity + EntityResult + Query + KindExpression + PropertyReference + PropertyExpression + PropertyOrder + Filter + CompositeFilter + PropertyFilter + GqlQuery + GqlQueryArg + QueryResultBatch + Mutation + MutationResult + ReadOptions + LookupRequest + LookupResponse + RunQueryRequest + RunQueryResponse + BeginTransactionRequest + BeginTransactionResponse + RollbackRequest + RollbackResponse + CommitRequest + CommitResponse + AllocateIdsRequest + AllocateIdsResponse +*/ +package datastore + +import proto "github.com/golang/protobuf/proto" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = math.Inf + +// Specifies what data the 'entity' field contains. +// A ResultType is either implied (for example, in LookupResponse.found it +// is always FULL) or specified by context (for example, in message +// QueryResultBatch, field 'entity_result_type' specifies a ResultType +// for all the values in field 'entity_result'). +type EntityResult_ResultType int32 + +const ( + EntityResult_FULL EntityResult_ResultType = 1 + EntityResult_PROJECTION EntityResult_ResultType = 2 + // The entity may have no key. + // A property value may have meaning 18. + EntityResult_KEY_ONLY EntityResult_ResultType = 3 +) + +var EntityResult_ResultType_name = map[int32]string{ + 1: "FULL", + 2: "PROJECTION", + 3: "KEY_ONLY", +} +var EntityResult_ResultType_value = map[string]int32{ + "FULL": 1, + "PROJECTION": 2, + "KEY_ONLY": 3, +} + +func (x EntityResult_ResultType) Enum() *EntityResult_ResultType { + p := new(EntityResult_ResultType) + *p = x + return p +} +func (x EntityResult_ResultType) String() string { + return proto.EnumName(EntityResult_ResultType_name, int32(x)) +} +func (x *EntityResult_ResultType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(EntityResult_ResultType_value, data, "EntityResult_ResultType") + if err != nil { + return err + } + *x = EntityResult_ResultType(value) + return nil +} + +type PropertyExpression_AggregationFunction int32 + +const ( + PropertyExpression_FIRST PropertyExpression_AggregationFunction = 1 +) + +var PropertyExpression_AggregationFunction_name = map[int32]string{ + 1: "FIRST", +} +var PropertyExpression_AggregationFunction_value = map[string]int32{ + "FIRST": 1, +} + +func (x PropertyExpression_AggregationFunction) Enum() *PropertyExpression_AggregationFunction { + p := new(PropertyExpression_AggregationFunction) + *p = x + return p +} +func (x PropertyExpression_AggregationFunction) String() string { + return proto.EnumName(PropertyExpression_AggregationFunction_name, int32(x)) +} +func (x *PropertyExpression_AggregationFunction) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(PropertyExpression_AggregationFunction_value, data, "PropertyExpression_AggregationFunction") + if err != nil { + return err + } + *x = PropertyExpression_AggregationFunction(value) + return nil +} + +type PropertyOrder_Direction int32 + +const ( + PropertyOrder_ASCENDING PropertyOrder_Direction = 1 + PropertyOrder_DESCENDING PropertyOrder_Direction = 2 +) + +var PropertyOrder_Direction_name = map[int32]string{ + 1: "ASCENDING", + 2: "DESCENDING", +} +var PropertyOrder_Direction_value = map[string]int32{ + "ASCENDING": 1, + "DESCENDING": 2, +} + +func (x PropertyOrder_Direction) Enum() *PropertyOrder_Direction { + p := new(PropertyOrder_Direction) + *p = x + return p +} +func (x PropertyOrder_Direction) String() string { + return proto.EnumName(PropertyOrder_Direction_name, int32(x)) +} +func (x *PropertyOrder_Direction) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(PropertyOrder_Direction_value, data, "PropertyOrder_Direction") + if err != nil { + return err + } + *x = PropertyOrder_Direction(value) + return nil +} + +type CompositeFilter_Operator int32 + +const ( + CompositeFilter_AND CompositeFilter_Operator = 1 +) + +var CompositeFilter_Operator_name = map[int32]string{ + 1: "AND", +} +var CompositeFilter_Operator_value = map[string]int32{ + "AND": 1, +} + +func (x CompositeFilter_Operator) Enum() *CompositeFilter_Operator { + p := new(CompositeFilter_Operator) + *p = x + return p +} +func (x CompositeFilter_Operator) String() string { + return proto.EnumName(CompositeFilter_Operator_name, int32(x)) +} +func (x *CompositeFilter_Operator) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(CompositeFilter_Operator_value, data, "CompositeFilter_Operator") + if err != nil { + return err + } + *x = CompositeFilter_Operator(value) + return nil +} + +type PropertyFilter_Operator int32 + +const ( + PropertyFilter_LESS_THAN PropertyFilter_Operator = 1 + PropertyFilter_LESS_THAN_OR_EQUAL PropertyFilter_Operator = 2 + PropertyFilter_GREATER_THAN PropertyFilter_Operator = 3 + PropertyFilter_GREATER_THAN_OR_EQUAL PropertyFilter_Operator = 4 + PropertyFilter_EQUAL PropertyFilter_Operator = 5 + PropertyFilter_HAS_ANCESTOR PropertyFilter_Operator = 11 +) + +var PropertyFilter_Operator_name = map[int32]string{ + 1: "LESS_THAN", + 2: "LESS_THAN_OR_EQUAL", + 3: "GREATER_THAN", + 4: "GREATER_THAN_OR_EQUAL", + 5: "EQUAL", + 11: "HAS_ANCESTOR", +} +var PropertyFilter_Operator_value = map[string]int32{ + "LESS_THAN": 1, + "LESS_THAN_OR_EQUAL": 2, + "GREATER_THAN": 3, + "GREATER_THAN_OR_EQUAL": 4, + "EQUAL": 5, + "HAS_ANCESTOR": 11, +} + +func (x PropertyFilter_Operator) Enum() *PropertyFilter_Operator { + p := new(PropertyFilter_Operator) + *p = x + return p +} +func (x PropertyFilter_Operator) String() string { + return proto.EnumName(PropertyFilter_Operator_name, int32(x)) +} +func (x *PropertyFilter_Operator) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(PropertyFilter_Operator_value, data, "PropertyFilter_Operator") + if err != nil { + return err + } + *x = PropertyFilter_Operator(value) + return nil +} + +// The possible values for the 'more_results' field. +type QueryResultBatch_MoreResultsType int32 + +const ( + QueryResultBatch_NOT_FINISHED QueryResultBatch_MoreResultsType = 1 + QueryResultBatch_MORE_RESULTS_AFTER_LIMIT QueryResultBatch_MoreResultsType = 2 + // results after the limit. + QueryResultBatch_NO_MORE_RESULTS QueryResultBatch_MoreResultsType = 3 +) + +var QueryResultBatch_MoreResultsType_name = map[int32]string{ + 1: "NOT_FINISHED", + 2: "MORE_RESULTS_AFTER_LIMIT", + 3: "NO_MORE_RESULTS", +} +var QueryResultBatch_MoreResultsType_value = map[string]int32{ + "NOT_FINISHED": 1, + "MORE_RESULTS_AFTER_LIMIT": 2, + "NO_MORE_RESULTS": 3, +} + +func (x QueryResultBatch_MoreResultsType) Enum() *QueryResultBatch_MoreResultsType { + p := new(QueryResultBatch_MoreResultsType) + *p = x + return p +} +func (x QueryResultBatch_MoreResultsType) String() string { + return proto.EnumName(QueryResultBatch_MoreResultsType_name, int32(x)) +} +func (x *QueryResultBatch_MoreResultsType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(QueryResultBatch_MoreResultsType_value, data, "QueryResultBatch_MoreResultsType") + if err != nil { + return err + } + *x = QueryResultBatch_MoreResultsType(value) + return nil +} + +type ReadOptions_ReadConsistency int32 + +const ( + ReadOptions_DEFAULT ReadOptions_ReadConsistency = 0 + ReadOptions_STRONG ReadOptions_ReadConsistency = 1 + ReadOptions_EVENTUAL ReadOptions_ReadConsistency = 2 +) + +var ReadOptions_ReadConsistency_name = map[int32]string{ + 0: "DEFAULT", + 1: "STRONG", + 2: "EVENTUAL", +} +var ReadOptions_ReadConsistency_value = map[string]int32{ + "DEFAULT": 0, + "STRONG": 1, + "EVENTUAL": 2, +} + +func (x ReadOptions_ReadConsistency) Enum() *ReadOptions_ReadConsistency { + p := new(ReadOptions_ReadConsistency) + *p = x + return p +} +func (x ReadOptions_ReadConsistency) String() string { + return proto.EnumName(ReadOptions_ReadConsistency_name, int32(x)) +} +func (x *ReadOptions_ReadConsistency) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(ReadOptions_ReadConsistency_value, data, "ReadOptions_ReadConsistency") + if err != nil { + return err + } + *x = ReadOptions_ReadConsistency(value) + return nil +} + +type BeginTransactionRequest_IsolationLevel int32 + +const ( + BeginTransactionRequest_SNAPSHOT BeginTransactionRequest_IsolationLevel = 0 + // conflict if their mutations conflict. For example: + // Read(A),Write(B) may not conflict with Read(B),Write(A), + // but Read(B),Write(B) does conflict with Read(B),Write(B). + BeginTransactionRequest_SERIALIZABLE BeginTransactionRequest_IsolationLevel = 1 +) + +var BeginTransactionRequest_IsolationLevel_name = map[int32]string{ + 0: "SNAPSHOT", + 1: "SERIALIZABLE", +} +var BeginTransactionRequest_IsolationLevel_value = map[string]int32{ + "SNAPSHOT": 0, + "SERIALIZABLE": 1, +} + +func (x BeginTransactionRequest_IsolationLevel) Enum() *BeginTransactionRequest_IsolationLevel { + p := new(BeginTransactionRequest_IsolationLevel) + *p = x + return p +} +func (x BeginTransactionRequest_IsolationLevel) String() string { + return proto.EnumName(BeginTransactionRequest_IsolationLevel_name, int32(x)) +} +func (x *BeginTransactionRequest_IsolationLevel) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(BeginTransactionRequest_IsolationLevel_value, data, "BeginTransactionRequest_IsolationLevel") + if err != nil { + return err + } + *x = BeginTransactionRequest_IsolationLevel(value) + return nil +} + +type CommitRequest_Mode int32 + +const ( + CommitRequest_TRANSACTIONAL CommitRequest_Mode = 1 + CommitRequest_NON_TRANSACTIONAL CommitRequest_Mode = 2 +) + +var CommitRequest_Mode_name = map[int32]string{ + 1: "TRANSACTIONAL", + 2: "NON_TRANSACTIONAL", +} +var CommitRequest_Mode_value = map[string]int32{ + "TRANSACTIONAL": 1, + "NON_TRANSACTIONAL": 2, +} + +func (x CommitRequest_Mode) Enum() *CommitRequest_Mode { + p := new(CommitRequest_Mode) + *p = x + return p +} +func (x CommitRequest_Mode) String() string { + return proto.EnumName(CommitRequest_Mode_name, int32(x)) +} +func (x *CommitRequest_Mode) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(CommitRequest_Mode_value, data, "CommitRequest_Mode") + if err != nil { + return err + } + *x = CommitRequest_Mode(value) + return nil +} + +// An identifier for a particular subset of entities. +// +// Entities are partitioned into various subsets, each used by different +// datasets and different namespaces within a dataset and so forth. +// +// All input partition IDs are normalized before use. +// A partition ID is normalized as follows: +// If the partition ID is unset or is set to an empty partition ID, replace it +// with the context partition ID. +// Otherwise, if the partition ID has no dataset ID, assign it the context +// partition ID's dataset ID. +// Unless otherwise documented, the context partition ID has the dataset ID set +// to the context dataset ID and no other partition dimension set. +// +// A partition ID is empty if all of its fields are unset. +// +// Partition dimension: +// A dimension may be unset. +// A dimension's value must never be "". +// A dimension's value must match [A-Za-z\d\.\-_]{1,100} +// If the value of any dimension matches regex "__.*__", +// the partition is reserved/read-only. +// A reserved/read-only partition ID is forbidden in certain documented contexts. +// +// Dataset ID: +// A dataset id's value must never be "". +// A dataset id's value must match +// ([a-z\d\-]{1,100}~)?([a-z\d][a-z\d\-\.]{0,99}:)?([a-z\d][a-z\d\-]{0,99} +type PartitionId struct { + // The dataset ID. + DatasetId *string `protobuf:"bytes,3,opt,name=dataset_id" json:"dataset_id,omitempty"` + // The namespace. + Namespace *string `protobuf:"bytes,4,opt,name=namespace" json:"namespace,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *PartitionId) Reset() { *m = PartitionId{} } +func (m *PartitionId) String() string { return proto.CompactTextString(m) } +func (*PartitionId) ProtoMessage() {} + +func (m *PartitionId) GetDatasetId() string { + if m != nil && m.DatasetId != nil { + return *m.DatasetId + } + return "" +} + +func (m *PartitionId) GetNamespace() string { + if m != nil && m.Namespace != nil { + return *m.Namespace + } + return "" +} + +// A unique identifier for an entity. +// If a key's partition id or any of its path kinds or names are +// reserved/read-only, the key is reserved/read-only. +// A reserved/read-only key is forbidden in certain documented contexts. +type Key struct { + // Entities are partitioned into subsets, currently identified by a dataset + // (usually implicitly specified by the project) and namespace ID. + // Queries are scoped to a single partition. + PartitionId *PartitionId `protobuf:"bytes,1,opt,name=partition_id" json:"partition_id,omitempty"` + // The entity path. + // An entity path consists of one or more elements composed of a kind and a + // string or numerical identifier, which identify entities. The first + // element identifies a root entity, the second element identifies + // a child of the root entity, the third element a child of the + // second entity, and so forth. The entities identified by all prefixes of + // the path are called the element's ancestors. + // An entity path is always fully complete: ALL of the entity's ancestors + // are required to be in the path along with the entity identifier itself. + // The only exception is that in some documented cases, the identifier in the + // last path element (for the entity) itself may be omitted. A path can never + // be empty. + PathElement []*Key_PathElement `protobuf:"bytes,2,rep,name=path_element" json:"path_element,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Key) Reset() { *m = Key{} } +func (m *Key) String() string { return proto.CompactTextString(m) } +func (*Key) ProtoMessage() {} + +func (m *Key) GetPartitionId() *PartitionId { + if m != nil { + return m.PartitionId + } + return nil +} + +func (m *Key) GetPathElement() []*Key_PathElement { + if m != nil { + return m.PathElement + } + return nil +} + +// A (kind, ID/name) pair used to construct a key path. +// +// At most one of name or ID may be set. +// If either is set, the element is complete. +// If neither is set, the element is incomplete. +type Key_PathElement struct { + // The kind of the entity. + // A kind matching regex "__.*__" is reserved/read-only. + // A kind must not contain more than 500 characters. + // Cannot be "". + Kind *string `protobuf:"bytes,1,req,name=kind" json:"kind,omitempty"` + // The ID of the entity. + // Never equal to zero. Values less than zero are discouraged and will not + // be supported in the future. + Id *int64 `protobuf:"varint,2,opt,name=id" json:"id,omitempty"` + // The name of the entity. + // A name matching regex "__.*__" is reserved/read-only. + // A name must not be more than 500 characters. + // Cannot be "". + Name *string `protobuf:"bytes,3,opt,name=name" json:"name,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Key_PathElement) Reset() { *m = Key_PathElement{} } +func (m *Key_PathElement) String() string { return proto.CompactTextString(m) } +func (*Key_PathElement) ProtoMessage() {} + +func (m *Key_PathElement) GetKind() string { + if m != nil && m.Kind != nil { + return *m.Kind + } + return "" +} + +func (m *Key_PathElement) GetId() int64 { + if m != nil && m.Id != nil { + return *m.Id + } + return 0 +} + +func (m *Key_PathElement) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +// A message that can hold any of the supported value types and associated +// metadata. +// +// At most one of the Value fields may be set. +// If none are set the value is "null". +// +type Value struct { + // A boolean value. + BooleanValue *bool `protobuf:"varint,1,opt,name=boolean_value" json:"boolean_value,omitempty"` + // An integer value. + IntegerValue *int64 `protobuf:"varint,2,opt,name=integer_value" json:"integer_value,omitempty"` + // A double value. + DoubleValue *float64 `protobuf:"fixed64,3,opt,name=double_value" json:"double_value,omitempty"` + // A timestamp value. + TimestampMicrosecondsValue *int64 `protobuf:"varint,4,opt,name=timestamp_microseconds_value" json:"timestamp_microseconds_value,omitempty"` + // A key value. + KeyValue *Key `protobuf:"bytes,5,opt,name=key_value" json:"key_value,omitempty"` + // A blob key value. + BlobKeyValue *string `protobuf:"bytes,16,opt,name=blob_key_value" json:"blob_key_value,omitempty"` + // A UTF-8 encoded string value. + StringValue *string `protobuf:"bytes,17,opt,name=string_value" json:"string_value,omitempty"` + // A blob value. + BlobValue []byte `protobuf:"bytes,18,opt,name=blob_value" json:"blob_value,omitempty"` + // An entity value. + // May have no key. + // May have a key with an incomplete key path. + // May have a reserved/read-only key. + EntityValue *Entity `protobuf:"bytes,6,opt,name=entity_value" json:"entity_value,omitempty"` + // A list value. + // Cannot contain another list value. + // Cannot also have a meaning and indexing set. + ListValue []*Value `protobuf:"bytes,7,rep,name=list_value" json:"list_value,omitempty"` + // The meaning field is reserved and should not be used. + Meaning *int32 `protobuf:"varint,14,opt,name=meaning" json:"meaning,omitempty"` + // If the value should be indexed. + // + // The indexed property may be set for a + // null value. + // When indexed is true, stringValue + // is limited to 500 characters and the blob value is limited to 500 bytes. + // Exception: If meaning is set to 2, string_value is limited to 2038 + // characters regardless of indexed. + // When indexed is true, meaning 15 and 22 are not allowed, and meaning 16 + // will be ignored on input (and will never be set on output). + // Input values by default have indexed set to + // true; however, you can explicitly set indexed to + // true if you want. (An output value never has + // indexed explicitly set to true.) If a value is + // itself an entity, it cannot have indexed set to + // true. + // Exception: An entity value with meaning 9, 20 or 21 may be indexed. + Indexed *bool `protobuf:"varint,15,opt,name=indexed,def=1" json:"indexed,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Value) Reset() { *m = Value{} } +func (m *Value) String() string { return proto.CompactTextString(m) } +func (*Value) ProtoMessage() {} + +const Default_Value_Indexed bool = true + +func (m *Value) GetBooleanValue() bool { + if m != nil && m.BooleanValue != nil { + return *m.BooleanValue + } + return false +} + +func (m *Value) GetIntegerValue() int64 { + if m != nil && m.IntegerValue != nil { + return *m.IntegerValue + } + return 0 +} + +func (m *Value) GetDoubleValue() float64 { + if m != nil && m.DoubleValue != nil { + return *m.DoubleValue + } + return 0 +} + +func (m *Value) GetTimestampMicrosecondsValue() int64 { + if m != nil && m.TimestampMicrosecondsValue != nil { + return *m.TimestampMicrosecondsValue + } + return 0 +} + +func (m *Value) GetKeyValue() *Key { + if m != nil { + return m.KeyValue + } + return nil +} + +func (m *Value) GetBlobKeyValue() string { + if m != nil && m.BlobKeyValue != nil { + return *m.BlobKeyValue + } + return "" +} + +func (m *Value) GetStringValue() string { + if m != nil && m.StringValue != nil { + return *m.StringValue + } + return "" +} + +func (m *Value) GetBlobValue() []byte { + if m != nil { + return m.BlobValue + } + return nil +} + +func (m *Value) GetEntityValue() *Entity { + if m != nil { + return m.EntityValue + } + return nil +} + +func (m *Value) GetListValue() []*Value { + if m != nil { + return m.ListValue + } + return nil +} + +func (m *Value) GetMeaning() int32 { + if m != nil && m.Meaning != nil { + return *m.Meaning + } + return 0 +} + +func (m *Value) GetIndexed() bool { + if m != nil && m.Indexed != nil { + return *m.Indexed + } + return Default_Value_Indexed +} + +// An entity property. +type Property struct { + // The name of the property. + // A property name matching regex "__.*__" is reserved. + // A reserved property name is forbidden in certain documented contexts. + // The name must not contain more than 500 characters. + // Cannot be "". + Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` + // The value(s) of the property. + // Each value can have only one value property populated. For example, + // you cannot have a values list of { value: { integerValue: 22, + // stringValue: "a" } }, but you can have { value: { listValue: + // [ { integerValue: 22 }, { stringValue: "a" } ] }. + Value *Value `protobuf:"bytes,4,req,name=value" json:"value,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Property) Reset() { *m = Property{} } +func (m *Property) String() string { return proto.CompactTextString(m) } +func (*Property) ProtoMessage() {} + +func (m *Property) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *Property) GetValue() *Value { + if m != nil { + return m.Value + } + return nil +} + +// An entity. +// +// An entity is limited to 1 megabyte when stored. That roughly +// corresponds to a limit of 1 megabyte for the serialized form of this +// message. +type Entity struct { + // The entity's key. + // + // An entity must have a key, unless otherwise documented (for example, + // an entity in Value.entityValue may have no key). + // An entity's kind is its key's path's last element's kind, + // or null if it has no key. + Key *Key `protobuf:"bytes,1,opt,name=key" json:"key,omitempty"` + // The entity's properties. + // Each property's name must be unique for its entity. + Property []*Property `protobuf:"bytes,2,rep,name=property" json:"property,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Entity) Reset() { *m = Entity{} } +func (m *Entity) String() string { return proto.CompactTextString(m) } +func (*Entity) ProtoMessage() {} + +func (m *Entity) GetKey() *Key { + if m != nil { + return m.Key + } + return nil +} + +func (m *Entity) GetProperty() []*Property { + if m != nil { + return m.Property + } + return nil +} + +// The result of fetching an entity from the datastore. +type EntityResult struct { + // The resulting entity. + Entity *Entity `protobuf:"bytes,1,req,name=entity" json:"entity,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *EntityResult) Reset() { *m = EntityResult{} } +func (m *EntityResult) String() string { return proto.CompactTextString(m) } +func (*EntityResult) ProtoMessage() {} + +func (m *EntityResult) GetEntity() *Entity { + if m != nil { + return m.Entity + } + return nil +} + +// A query. +type Query struct { + // The projection to return. If not set the entire entity is returned. + Projection []*PropertyExpression `protobuf:"bytes,2,rep,name=projection" json:"projection,omitempty"` + // The kinds to query (if empty, returns entities from all kinds). + Kind []*KindExpression `protobuf:"bytes,3,rep,name=kind" json:"kind,omitempty"` + // The filter to apply (optional). + Filter *Filter `protobuf:"bytes,4,opt,name=filter" json:"filter,omitempty"` + // The order to apply to the query results (if empty, order is unspecified). + Order []*PropertyOrder `protobuf:"bytes,5,rep,name=order" json:"order,omitempty"` + // The properties to group by (if empty, no grouping is applied to the + // result set). + GroupBy []*PropertyReference `protobuf:"bytes,6,rep,name=group_by" json:"group_by,omitempty"` + // A starting point for the query results. Optional. Query cursors are + // returned in query result batches. + StartCursor []byte `protobuf:"bytes,7,opt,name=start_cursor" json:"start_cursor,omitempty"` + // An ending point for the query results. Optional. Query cursors are + // returned in query result batches. + EndCursor []byte `protobuf:"bytes,8,opt,name=end_cursor" json:"end_cursor,omitempty"` + // The number of results to skip. Applies before limit, but after all other + // constraints (optional, defaults to 0). + Offset *int32 `protobuf:"varint,10,opt,name=offset,def=0" json:"offset,omitempty"` + // The maximum number of results to return. Applies after all other + // constraints. Optional. + Limit *int32 `protobuf:"varint,11,opt,name=limit" json:"limit,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Query) Reset() { *m = Query{} } +func (m *Query) String() string { return proto.CompactTextString(m) } +func (*Query) ProtoMessage() {} + +const Default_Query_Offset int32 = 0 + +func (m *Query) GetProjection() []*PropertyExpression { + if m != nil { + return m.Projection + } + return nil +} + +func (m *Query) GetKind() []*KindExpression { + if m != nil { + return m.Kind + } + return nil +} + +func (m *Query) GetFilter() *Filter { + if m != nil { + return m.Filter + } + return nil +} + +func (m *Query) GetOrder() []*PropertyOrder { + if m != nil { + return m.Order + } + return nil +} + +func (m *Query) GetGroupBy() []*PropertyReference { + if m != nil { + return m.GroupBy + } + return nil +} + +func (m *Query) GetStartCursor() []byte { + if m != nil { + return m.StartCursor + } + return nil +} + +func (m *Query) GetEndCursor() []byte { + if m != nil { + return m.EndCursor + } + return nil +} + +func (m *Query) GetOffset() int32 { + if m != nil && m.Offset != nil { + return *m.Offset + } + return Default_Query_Offset +} + +func (m *Query) GetLimit() int32 { + if m != nil && m.Limit != nil { + return *m.Limit + } + return 0 +} + +// A representation of a kind. +type KindExpression struct { + // The name of the kind. + Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *KindExpression) Reset() { *m = KindExpression{} } +func (m *KindExpression) String() string { return proto.CompactTextString(m) } +func (*KindExpression) ProtoMessage() {} + +func (m *KindExpression) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +// A reference to a property relative to the kind expressions. +// exactly. +type PropertyReference struct { + // The name of the property. + Name *string `protobuf:"bytes,2,req,name=name" json:"name,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *PropertyReference) Reset() { *m = PropertyReference{} } +func (m *PropertyReference) String() string { return proto.CompactTextString(m) } +func (*PropertyReference) ProtoMessage() {} + +func (m *PropertyReference) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +// A representation of a property in a projection. +type PropertyExpression struct { + // The property to project. + Property *PropertyReference `protobuf:"bytes,1,req,name=property" json:"property,omitempty"` + // The aggregation function to apply to the property. Optional. + // Can only be used when grouping by at least one property. Must + // then be set on all properties in the projection that are not + // being grouped by. + AggregationFunction *PropertyExpression_AggregationFunction `protobuf:"varint,2,opt,name=aggregation_function,enum=datastore.PropertyExpression_AggregationFunction" json:"aggregation_function,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *PropertyExpression) Reset() { *m = PropertyExpression{} } +func (m *PropertyExpression) String() string { return proto.CompactTextString(m) } +func (*PropertyExpression) ProtoMessage() {} + +func (m *PropertyExpression) GetProperty() *PropertyReference { + if m != nil { + return m.Property + } + return nil +} + +func (m *PropertyExpression) GetAggregationFunction() PropertyExpression_AggregationFunction { + if m != nil && m.AggregationFunction != nil { + return *m.AggregationFunction + } + return PropertyExpression_FIRST +} + +// The desired order for a specific property. +type PropertyOrder struct { + // The property to order by. + Property *PropertyReference `protobuf:"bytes,1,req,name=property" json:"property,omitempty"` + // The direction to order by. + Direction *PropertyOrder_Direction `protobuf:"varint,2,opt,name=direction,enum=datastore.PropertyOrder_Direction,def=1" json:"direction,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *PropertyOrder) Reset() { *m = PropertyOrder{} } +func (m *PropertyOrder) String() string { return proto.CompactTextString(m) } +func (*PropertyOrder) ProtoMessage() {} + +const Default_PropertyOrder_Direction PropertyOrder_Direction = PropertyOrder_ASCENDING + +func (m *PropertyOrder) GetProperty() *PropertyReference { + if m != nil { + return m.Property + } + return nil +} + +func (m *PropertyOrder) GetDirection() PropertyOrder_Direction { + if m != nil && m.Direction != nil { + return *m.Direction + } + return Default_PropertyOrder_Direction +} + +// A holder for any type of filter. Exactly one field should be specified. +type Filter struct { + // A composite filter. + CompositeFilter *CompositeFilter `protobuf:"bytes,1,opt,name=composite_filter" json:"composite_filter,omitempty"` + // A filter on a property. + PropertyFilter *PropertyFilter `protobuf:"bytes,2,opt,name=property_filter" json:"property_filter,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Filter) Reset() { *m = Filter{} } +func (m *Filter) String() string { return proto.CompactTextString(m) } +func (*Filter) ProtoMessage() {} + +func (m *Filter) GetCompositeFilter() *CompositeFilter { + if m != nil { + return m.CompositeFilter + } + return nil +} + +func (m *Filter) GetPropertyFilter() *PropertyFilter { + if m != nil { + return m.PropertyFilter + } + return nil +} + +// A filter that merges the multiple other filters using the given operation. +type CompositeFilter struct { + // The operator for combining multiple filters. + Operator *CompositeFilter_Operator `protobuf:"varint,1,req,name=operator,enum=datastore.CompositeFilter_Operator" json:"operator,omitempty"` + // The list of filters to combine. + // Must contain at least one filter. + Filter []*Filter `protobuf:"bytes,2,rep,name=filter" json:"filter,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *CompositeFilter) Reset() { *m = CompositeFilter{} } +func (m *CompositeFilter) String() string { return proto.CompactTextString(m) } +func (*CompositeFilter) ProtoMessage() {} + +func (m *CompositeFilter) GetOperator() CompositeFilter_Operator { + if m != nil && m.Operator != nil { + return *m.Operator + } + return CompositeFilter_AND +} + +func (m *CompositeFilter) GetFilter() []*Filter { + if m != nil { + return m.Filter + } + return nil +} + +// A filter on a specific property. +type PropertyFilter struct { + // The property to filter by. + Property *PropertyReference `protobuf:"bytes,1,req,name=property" json:"property,omitempty"` + // The operator to filter by. + Operator *PropertyFilter_Operator `protobuf:"varint,2,req,name=operator,enum=datastore.PropertyFilter_Operator" json:"operator,omitempty"` + // The value to compare the property to. + Value *Value `protobuf:"bytes,3,req,name=value" json:"value,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *PropertyFilter) Reset() { *m = PropertyFilter{} } +func (m *PropertyFilter) String() string { return proto.CompactTextString(m) } +func (*PropertyFilter) ProtoMessage() {} + +func (m *PropertyFilter) GetProperty() *PropertyReference { + if m != nil { + return m.Property + } + return nil +} + +func (m *PropertyFilter) GetOperator() PropertyFilter_Operator { + if m != nil && m.Operator != nil { + return *m.Operator + } + return PropertyFilter_LESS_THAN +} + +func (m *PropertyFilter) GetValue() *Value { + if m != nil { + return m.Value + } + return nil +} + +// A GQL query. +type GqlQuery struct { + QueryString *string `protobuf:"bytes,1,req,name=query_string" json:"query_string,omitempty"` + // When false, the query string must not contain a literal. + AllowLiteral *bool `protobuf:"varint,2,opt,name=allow_literal,def=0" json:"allow_literal,omitempty"` + // A named argument must set field GqlQueryArg.name. + // No two named arguments may have the same name. + // For each non-reserved named binding site in the query string, + // there must be a named argument with that name, + // but not necessarily the inverse. + NameArg []*GqlQueryArg `protobuf:"bytes,3,rep,name=name_arg" json:"name_arg,omitempty"` + // Numbered binding site @1 references the first numbered argument, + // effectively using 1-based indexing, rather than the usual 0. + // A numbered argument must NOT set field GqlQueryArg.name. + // For each binding site numbered i in query_string, + // there must be an ith numbered argument. + // The inverse must also be true. + NumberArg []*GqlQueryArg `protobuf:"bytes,4,rep,name=number_arg" json:"number_arg,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *GqlQuery) Reset() { *m = GqlQuery{} } +func (m *GqlQuery) String() string { return proto.CompactTextString(m) } +func (*GqlQuery) ProtoMessage() {} + +const Default_GqlQuery_AllowLiteral bool = false + +func (m *GqlQuery) GetQueryString() string { + if m != nil && m.QueryString != nil { + return *m.QueryString + } + return "" +} + +func (m *GqlQuery) GetAllowLiteral() bool { + if m != nil && m.AllowLiteral != nil { + return *m.AllowLiteral + } + return Default_GqlQuery_AllowLiteral +} + +func (m *GqlQuery) GetNameArg() []*GqlQueryArg { + if m != nil { + return m.NameArg + } + return nil +} + +func (m *GqlQuery) GetNumberArg() []*GqlQueryArg { + if m != nil { + return m.NumberArg + } + return nil +} + +// A binding argument for a GQL query. +// Exactly one of fields value and cursor must be set. +type GqlQueryArg struct { + // Must match regex "[A-Za-z_$][A-Za-z_$0-9]*". + // Must not match regex "__.*__". + // Must not be "". + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Value *Value `protobuf:"bytes,2,opt,name=value" json:"value,omitempty"` + Cursor []byte `protobuf:"bytes,3,opt,name=cursor" json:"cursor,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *GqlQueryArg) Reset() { *m = GqlQueryArg{} } +func (m *GqlQueryArg) String() string { return proto.CompactTextString(m) } +func (*GqlQueryArg) ProtoMessage() {} + +func (m *GqlQueryArg) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *GqlQueryArg) GetValue() *Value { + if m != nil { + return m.Value + } + return nil +} + +func (m *GqlQueryArg) GetCursor() []byte { + if m != nil { + return m.Cursor + } + return nil +} + +// A batch of results produced by a query. +type QueryResultBatch struct { + // The result type for every entity in entityResults. + EntityResultType *EntityResult_ResultType `protobuf:"varint,1,req,name=entity_result_type,enum=datastore.EntityResult_ResultType" json:"entity_result_type,omitempty"` + // The results for this batch. + EntityResult []*EntityResult `protobuf:"bytes,2,rep,name=entity_result" json:"entity_result,omitempty"` + // A cursor that points to the position after the last result in the batch. + // May be absent. + EndCursor []byte `protobuf:"bytes,4,opt,name=end_cursor" json:"end_cursor,omitempty"` + // The state of the query after the current batch. + MoreResults *QueryResultBatch_MoreResultsType `protobuf:"varint,5,req,name=more_results,enum=datastore.QueryResultBatch_MoreResultsType" json:"more_results,omitempty"` + // The number of results skipped because of Query.offset. + SkippedResults *int32 `protobuf:"varint,6,opt,name=skipped_results" json:"skipped_results,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *QueryResultBatch) Reset() { *m = QueryResultBatch{} } +func (m *QueryResultBatch) String() string { return proto.CompactTextString(m) } +func (*QueryResultBatch) ProtoMessage() {} + +func (m *QueryResultBatch) GetEntityResultType() EntityResult_ResultType { + if m != nil && m.EntityResultType != nil { + return *m.EntityResultType + } + return EntityResult_FULL +} + +func (m *QueryResultBatch) GetEntityResult() []*EntityResult { + if m != nil { + return m.EntityResult + } + return nil +} + +func (m *QueryResultBatch) GetEndCursor() []byte { + if m != nil { + return m.EndCursor + } + return nil +} + +func (m *QueryResultBatch) GetMoreResults() QueryResultBatch_MoreResultsType { + if m != nil && m.MoreResults != nil { + return *m.MoreResults + } + return QueryResultBatch_NOT_FINISHED +} + +func (m *QueryResultBatch) GetSkippedResults() int32 { + if m != nil && m.SkippedResults != nil { + return *m.SkippedResults + } + return 0 +} + +// A set of changes to apply. +// +// No entity in this message may have a reserved property name, +// not even a property in an entity in a value. +// No value in this message may have meaning 18, +// not even a value in an entity in another value. +// +// If entities with duplicate keys are present, an arbitrary choice will +// be made as to which is written. +type Mutation struct { + // Entities to upsert. + // Each upserted entity's key must have a complete path and + // must not be reserved/read-only. + Upsert []*Entity `protobuf:"bytes,1,rep,name=upsert" json:"upsert,omitempty"` + // Entities to update. + // Each updated entity's key must have a complete path and + // must not be reserved/read-only. + Update []*Entity `protobuf:"bytes,2,rep,name=update" json:"update,omitempty"` + // Entities to insert. + // Each inserted entity's key must have a complete path and + // must not be reserved/read-only. + Insert []*Entity `protobuf:"bytes,3,rep,name=insert" json:"insert,omitempty"` + // Insert entities with a newly allocated ID. + // Each inserted entity's key must omit the final identifier in its path and + // must not be reserved/read-only. + InsertAutoId []*Entity `protobuf:"bytes,4,rep,name=insert_auto_id" json:"insert_auto_id,omitempty"` + // Keys of entities to delete. + // Each key must have a complete key path and must not be reserved/read-only. + Delete []*Key `protobuf:"bytes,5,rep,name=delete" json:"delete,omitempty"` + // Ignore a user specified read-only period. Optional. + Force *bool `protobuf:"varint,6,opt,name=force" json:"force,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Mutation) Reset() { *m = Mutation{} } +func (m *Mutation) String() string { return proto.CompactTextString(m) } +func (*Mutation) ProtoMessage() {} + +func (m *Mutation) GetUpsert() []*Entity { + if m != nil { + return m.Upsert + } + return nil +} + +func (m *Mutation) GetUpdate() []*Entity { + if m != nil { + return m.Update + } + return nil +} + +func (m *Mutation) GetInsert() []*Entity { + if m != nil { + return m.Insert + } + return nil +} + +func (m *Mutation) GetInsertAutoId() []*Entity { + if m != nil { + return m.InsertAutoId + } + return nil +} + +func (m *Mutation) GetDelete() []*Key { + if m != nil { + return m.Delete + } + return nil +} + +func (m *Mutation) GetForce() bool { + if m != nil && m.Force != nil { + return *m.Force + } + return false +} + +// The result of applying a mutation. +type MutationResult struct { + // Number of index writes. + IndexUpdates *int32 `protobuf:"varint,1,req,name=index_updates" json:"index_updates,omitempty"` + // Keys for insertAutoId entities. One per entity from the + // request, in the same order. + InsertAutoIdKey []*Key `protobuf:"bytes,2,rep,name=insert_auto_id_key" json:"insert_auto_id_key,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *MutationResult) Reset() { *m = MutationResult{} } +func (m *MutationResult) String() string { return proto.CompactTextString(m) } +func (*MutationResult) ProtoMessage() {} + +func (m *MutationResult) GetIndexUpdates() int32 { + if m != nil && m.IndexUpdates != nil { + return *m.IndexUpdates + } + return 0 +} + +func (m *MutationResult) GetInsertAutoIdKey() []*Key { + if m != nil { + return m.InsertAutoIdKey + } + return nil +} + +// Options shared by read requests. +type ReadOptions struct { + // The read consistency to use. + // Cannot be set when transaction is set. + // Lookup and ancestor queries default to STRONG, global queries default to + // EVENTUAL and cannot be set to STRONG. + ReadConsistency *ReadOptions_ReadConsistency `protobuf:"varint,1,opt,name=read_consistency,enum=datastore.ReadOptions_ReadConsistency,def=0" json:"read_consistency,omitempty"` + // The transaction to use. Optional. + Transaction []byte `protobuf:"bytes,2,opt,name=transaction" json:"transaction,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ReadOptions) Reset() { *m = ReadOptions{} } +func (m *ReadOptions) String() string { return proto.CompactTextString(m) } +func (*ReadOptions) ProtoMessage() {} + +const Default_ReadOptions_ReadConsistency ReadOptions_ReadConsistency = ReadOptions_DEFAULT + +func (m *ReadOptions) GetReadConsistency() ReadOptions_ReadConsistency { + if m != nil && m.ReadConsistency != nil { + return *m.ReadConsistency + } + return Default_ReadOptions_ReadConsistency +} + +func (m *ReadOptions) GetTransaction() []byte { + if m != nil { + return m.Transaction + } + return nil +} + +// The request for Lookup. +type LookupRequest struct { + // Options for this lookup request. Optional. + ReadOptions *ReadOptions `protobuf:"bytes,1,opt,name=read_options" json:"read_options,omitempty"` + // Keys of entities to look up from the datastore. + Key []*Key `protobuf:"bytes,3,rep,name=key" json:"key,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *LookupRequest) Reset() { *m = LookupRequest{} } +func (m *LookupRequest) String() string { return proto.CompactTextString(m) } +func (*LookupRequest) ProtoMessage() {} + +func (m *LookupRequest) GetReadOptions() *ReadOptions { + if m != nil { + return m.ReadOptions + } + return nil +} + +func (m *LookupRequest) GetKey() []*Key { + if m != nil { + return m.Key + } + return nil +} + +// The response for Lookup. +type LookupResponse struct { + // Entities found as ResultType.FULL entities. + Found []*EntityResult `protobuf:"bytes,1,rep,name=found" json:"found,omitempty"` + // Entities not found as ResultType.KEY_ONLY entities. + Missing []*EntityResult `protobuf:"bytes,2,rep,name=missing" json:"missing,omitempty"` + // A list of keys that were not looked up due to resource constraints. + Deferred []*Key `protobuf:"bytes,3,rep,name=deferred" json:"deferred,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *LookupResponse) Reset() { *m = LookupResponse{} } +func (m *LookupResponse) String() string { return proto.CompactTextString(m) } +func (*LookupResponse) ProtoMessage() {} + +func (m *LookupResponse) GetFound() []*EntityResult { + if m != nil { + return m.Found + } + return nil +} + +func (m *LookupResponse) GetMissing() []*EntityResult { + if m != nil { + return m.Missing + } + return nil +} + +func (m *LookupResponse) GetDeferred() []*Key { + if m != nil { + return m.Deferred + } + return nil +} + +// The request for RunQuery. +type RunQueryRequest struct { + // The options for this query. + ReadOptions *ReadOptions `protobuf:"bytes,1,opt,name=read_options" json:"read_options,omitempty"` + // Entities are partitioned into subsets, identified by a dataset (usually + // implicitly specified by the project) and namespace ID. Queries are scoped + // to a single partition. + // This partition ID is normalized with the standard default context + // partition ID, but all other partition IDs in RunQueryRequest are + // normalized with this partition ID as the context partition ID. + PartitionId *PartitionId `protobuf:"bytes,2,opt,name=partition_id" json:"partition_id,omitempty"` + // The query to run. + // Either this field or field gql_query must be set, but not both. + Query *Query `protobuf:"bytes,3,opt,name=query" json:"query,omitempty"` + // The GQL query to run. + // Either this field or field query must be set, but not both. + GqlQuery *GqlQuery `protobuf:"bytes,7,opt,name=gql_query" json:"gql_query,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *RunQueryRequest) Reset() { *m = RunQueryRequest{} } +func (m *RunQueryRequest) String() string { return proto.CompactTextString(m) } +func (*RunQueryRequest) ProtoMessage() {} + +func (m *RunQueryRequest) GetReadOptions() *ReadOptions { + if m != nil { + return m.ReadOptions + } + return nil +} + +func (m *RunQueryRequest) GetPartitionId() *PartitionId { + if m != nil { + return m.PartitionId + } + return nil +} + +func (m *RunQueryRequest) GetQuery() *Query { + if m != nil { + return m.Query + } + return nil +} + +func (m *RunQueryRequest) GetGqlQuery() *GqlQuery { + if m != nil { + return m.GqlQuery + } + return nil +} + +// The response for RunQuery. +type RunQueryResponse struct { + // A batch of query results (always present). + Batch *QueryResultBatch `protobuf:"bytes,1,opt,name=batch" json:"batch,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *RunQueryResponse) Reset() { *m = RunQueryResponse{} } +func (m *RunQueryResponse) String() string { return proto.CompactTextString(m) } +func (*RunQueryResponse) ProtoMessage() {} + +func (m *RunQueryResponse) GetBatch() *QueryResultBatch { + if m != nil { + return m.Batch + } + return nil +} + +// The request for BeginTransaction. +type BeginTransactionRequest struct { + // The transaction isolation level. + IsolationLevel *BeginTransactionRequest_IsolationLevel `protobuf:"varint,1,opt,name=isolation_level,enum=datastore.BeginTransactionRequest_IsolationLevel,def=0" json:"isolation_level,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *BeginTransactionRequest) Reset() { *m = BeginTransactionRequest{} } +func (m *BeginTransactionRequest) String() string { return proto.CompactTextString(m) } +func (*BeginTransactionRequest) ProtoMessage() {} + +const Default_BeginTransactionRequest_IsolationLevel BeginTransactionRequest_IsolationLevel = BeginTransactionRequest_SNAPSHOT + +func (m *BeginTransactionRequest) GetIsolationLevel() BeginTransactionRequest_IsolationLevel { + if m != nil && m.IsolationLevel != nil { + return *m.IsolationLevel + } + return Default_BeginTransactionRequest_IsolationLevel +} + +// The response for BeginTransaction. +type BeginTransactionResponse struct { + // The transaction identifier (always present). + Transaction []byte `protobuf:"bytes,1,opt,name=transaction" json:"transaction,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *BeginTransactionResponse) Reset() { *m = BeginTransactionResponse{} } +func (m *BeginTransactionResponse) String() string { return proto.CompactTextString(m) } +func (*BeginTransactionResponse) ProtoMessage() {} + +func (m *BeginTransactionResponse) GetTransaction() []byte { + if m != nil { + return m.Transaction + } + return nil +} + +// The request for Rollback. +type RollbackRequest struct { + // The transaction identifier, returned by a call to + // beginTransaction. + Transaction []byte `protobuf:"bytes,1,req,name=transaction" json:"transaction,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *RollbackRequest) Reset() { *m = RollbackRequest{} } +func (m *RollbackRequest) String() string { return proto.CompactTextString(m) } +func (*RollbackRequest) ProtoMessage() {} + +func (m *RollbackRequest) GetTransaction() []byte { + if m != nil { + return m.Transaction + } + return nil +} + +// The response for Rollback. +type RollbackResponse struct { + XXX_unrecognized []byte `json:"-"` +} + +func (m *RollbackResponse) Reset() { *m = RollbackResponse{} } +func (m *RollbackResponse) String() string { return proto.CompactTextString(m) } +func (*RollbackResponse) ProtoMessage() {} + +// The request for Commit. +type CommitRequest struct { + // The transaction identifier, returned by a call to + // beginTransaction. Must be set when mode is TRANSACTIONAL. + Transaction []byte `protobuf:"bytes,1,opt,name=transaction" json:"transaction,omitempty"` + // The mutation to perform. Optional. + Mutation *Mutation `protobuf:"bytes,2,opt,name=mutation" json:"mutation,omitempty"` + // The type of commit to perform. Either TRANSACTIONAL or NON_TRANSACTIONAL. + Mode *CommitRequest_Mode `protobuf:"varint,5,opt,name=mode,enum=datastore.CommitRequest_Mode,def=1" json:"mode,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *CommitRequest) Reset() { *m = CommitRequest{} } +func (m *CommitRequest) String() string { return proto.CompactTextString(m) } +func (*CommitRequest) ProtoMessage() {} + +const Default_CommitRequest_Mode CommitRequest_Mode = CommitRequest_TRANSACTIONAL + +func (m *CommitRequest) GetTransaction() []byte { + if m != nil { + return m.Transaction + } + return nil +} + +func (m *CommitRequest) GetMutation() *Mutation { + if m != nil { + return m.Mutation + } + return nil +} + +func (m *CommitRequest) GetMode() CommitRequest_Mode { + if m != nil && m.Mode != nil { + return *m.Mode + } + return Default_CommitRequest_Mode +} + +// The response for Commit. +type CommitResponse struct { + // The result of performing the mutation (if any). + MutationResult *MutationResult `protobuf:"bytes,1,opt,name=mutation_result" json:"mutation_result,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *CommitResponse) Reset() { *m = CommitResponse{} } +func (m *CommitResponse) String() string { return proto.CompactTextString(m) } +func (*CommitResponse) ProtoMessage() {} + +func (m *CommitResponse) GetMutationResult() *MutationResult { + if m != nil { + return m.MutationResult + } + return nil +} + +// The request for AllocateIds. +type AllocateIdsRequest struct { + // A list of keys with incomplete key paths to allocate IDs for. + // No key may be reserved/read-only. + Key []*Key `protobuf:"bytes,1,rep,name=key" json:"key,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *AllocateIdsRequest) Reset() { *m = AllocateIdsRequest{} } +func (m *AllocateIdsRequest) String() string { return proto.CompactTextString(m) } +func (*AllocateIdsRequest) ProtoMessage() {} + +func (m *AllocateIdsRequest) GetKey() []*Key { + if m != nil { + return m.Key + } + return nil +} + +// The response for AllocateIds. +type AllocateIdsResponse struct { + // The keys specified in the request (in the same order), each with + // its key path completed with a newly allocated ID. + Key []*Key `protobuf:"bytes,1,rep,name=key" json:"key,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *AllocateIdsResponse) Reset() { *m = AllocateIdsResponse{} } +func (m *AllocateIdsResponse) String() string { return proto.CompactTextString(m) } +func (*AllocateIdsResponse) ProtoMessage() {} + +func (m *AllocateIdsResponse) GetKey() []*Key { + if m != nil { + return m.Key + } + return nil +} + +func init() { + proto.RegisterEnum("datastore.EntityResult_ResultType", EntityResult_ResultType_name, EntityResult_ResultType_value) + proto.RegisterEnum("datastore.PropertyExpression_AggregationFunction", PropertyExpression_AggregationFunction_name, PropertyExpression_AggregationFunction_value) + proto.RegisterEnum("datastore.PropertyOrder_Direction", PropertyOrder_Direction_name, PropertyOrder_Direction_value) + proto.RegisterEnum("datastore.CompositeFilter_Operator", CompositeFilter_Operator_name, CompositeFilter_Operator_value) + proto.RegisterEnum("datastore.PropertyFilter_Operator", PropertyFilter_Operator_name, PropertyFilter_Operator_value) + proto.RegisterEnum("datastore.QueryResultBatch_MoreResultsType", QueryResultBatch_MoreResultsType_name, QueryResultBatch_MoreResultsType_value) + proto.RegisterEnum("datastore.ReadOptions_ReadConsistency", ReadOptions_ReadConsistency_name, ReadOptions_ReadConsistency_value) + proto.RegisterEnum("datastore.BeginTransactionRequest_IsolationLevel", BeginTransactionRequest_IsolationLevel_name, BeginTransactionRequest_IsolationLevel_value) + proto.RegisterEnum("datastore.CommitRequest_Mode", CommitRequest_Mode_name, CommitRequest_Mode_value) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/datastore/datastore_v1.proto b/Godeps/_workspace/src/google.golang.org/cloud/internal/datastore/datastore_v1.proto new file mode 100644 index 0000000000..d752beaa5e --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/datastore/datastore_v1.proto @@ -0,0 +1,606 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The datastore v1 service proto definitions + +syntax = "proto2"; + +package datastore; +option java_package = "com.google.api.services.datastore"; + + +// An identifier for a particular subset of entities. +// +// Entities are partitioned into various subsets, each used by different +// datasets and different namespaces within a dataset and so forth. +// +// All input partition IDs are normalized before use. +// A partition ID is normalized as follows: +// If the partition ID is unset or is set to an empty partition ID, replace it +// with the context partition ID. +// Otherwise, if the partition ID has no dataset ID, assign it the context +// partition ID's dataset ID. +// Unless otherwise documented, the context partition ID has the dataset ID set +// to the context dataset ID and no other partition dimension set. +// +// A partition ID is empty if all of its fields are unset. +// +// Partition dimension: +// A dimension may be unset. +// A dimension's value must never be "". +// A dimension's value must match [A-Za-z\d\.\-_]{1,100} +// If the value of any dimension matches regex "__.*__", +// the partition is reserved/read-only. +// A reserved/read-only partition ID is forbidden in certain documented contexts. +// +// Dataset ID: +// A dataset id's value must never be "". +// A dataset id's value must match +// ([a-z\d\-]{1,100}~)?([a-z\d][a-z\d\-\.]{0,99}:)?([a-z\d][a-z\d\-]{0,99} +message PartitionId { + // The dataset ID. + optional string dataset_id = 3; + // The namespace. + optional string namespace = 4; +} + +// A unique identifier for an entity. +// If a key's partition id or any of its path kinds or names are +// reserved/read-only, the key is reserved/read-only. +// A reserved/read-only key is forbidden in certain documented contexts. +message Key { + // Entities are partitioned into subsets, currently identified by a dataset + // (usually implicitly specified by the project) and namespace ID. + // Queries are scoped to a single partition. + optional PartitionId partition_id = 1; + + // A (kind, ID/name) pair used to construct a key path. + // + // At most one of name or ID may be set. + // If either is set, the element is complete. + // If neither is set, the element is incomplete. + message PathElement { + // The kind of the entity. + // A kind matching regex "__.*__" is reserved/read-only. + // A kind must not contain more than 500 characters. + // Cannot be "". + required string kind = 1; + // The ID of the entity. + // Never equal to zero. Values less than zero are discouraged and will not + // be supported in the future. + optional int64 id = 2; + // The name of the entity. + // A name matching regex "__.*__" is reserved/read-only. + // A name must not be more than 500 characters. + // Cannot be "". + optional string name = 3; + } + + // The entity path. + // An entity path consists of one or more elements composed of a kind and a + // string or numerical identifier, which identify entities. The first + // element identifies a root entity, the second element identifies + // a child of the root entity, the third element a child of the + // second entity, and so forth. The entities identified by all prefixes of + // the path are called the element's ancestors. + // An entity path is always fully complete: ALL of the entity's ancestors + // are required to be in the path along with the entity identifier itself. + // The only exception is that in some documented cases, the identifier in the + // last path element (for the entity) itself may be omitted. A path can never + // be empty. + repeated PathElement path_element = 2; +} + +// A message that can hold any of the supported value types and associated +// metadata. +// +// At most one of the Value fields may be set. +// If none are set the value is "null". +// +message Value { + // A boolean value. + optional bool boolean_value = 1; + // An integer value. + optional int64 integer_value = 2; + // A double value. + optional double double_value = 3; + // A timestamp value. + optional int64 timestamp_microseconds_value = 4; + // A key value. + optional Key key_value = 5; + // A blob key value. + optional string blob_key_value = 16; + // A UTF-8 encoded string value. + optional string string_value = 17; + // A blob value. + optional bytes blob_value = 18; + // An entity value. + // May have no key. + // May have a key with an incomplete key path. + // May have a reserved/read-only key. + optional Entity entity_value = 6; + // A list value. + // Cannot contain another list value. + // Cannot also have a meaning and indexing set. + repeated Value list_value = 7; + + // The meaning field is reserved and should not be used. + optional int32 meaning = 14; + + // If the value should be indexed. + // + // The indexed property may be set for a + // null value. + // When indexed is true, stringValue + // is limited to 500 characters and the blob value is limited to 500 bytes. + // Exception: If meaning is set to 2, string_value is limited to 2038 + // characters regardless of indexed. + // When indexed is true, meaning 15 and 22 are not allowed, and meaning 16 + // will be ignored on input (and will never be set on output). + // Input values by default have indexed set to + // true; however, you can explicitly set indexed to + // true if you want. (An output value never has + // indexed explicitly set to true.) If a value is + // itself an entity, it cannot have indexed set to + // true. + // Exception: An entity value with meaning 9, 20 or 21 may be indexed. + optional bool indexed = 15 [default = true]; +} + +// An entity property. +message Property { + // The name of the property. + // A property name matching regex "__.*__" is reserved. + // A reserved property name is forbidden in certain documented contexts. + // The name must not contain more than 500 characters. + // Cannot be "". + required string name = 1; + + // The value(s) of the property. + // Each value can have only one value property populated. For example, + // you cannot have a values list of { value: { integerValue: 22, + // stringValue: "a" } }, but you can have { value: { listValue: + // [ { integerValue: 22 }, { stringValue: "a" } ] }. + required Value value = 4; +} + +// An entity. +// +// An entity is limited to 1 megabyte when stored. That roughly +// corresponds to a limit of 1 megabyte for the serialized form of this +// message. +message Entity { + // The entity's key. + // + // An entity must have a key, unless otherwise documented (for example, + // an entity in Value.entityValue may have no key). + // An entity's kind is its key's path's last element's kind, + // or null if it has no key. + optional Key key = 1; + // The entity's properties. + // Each property's name must be unique for its entity. + repeated Property property = 2; +} + +// The result of fetching an entity from the datastore. +message EntityResult { + // Specifies what data the 'entity' field contains. + // A ResultType is either implied (for example, in LookupResponse.found it + // is always FULL) or specified by context (for example, in message + // QueryResultBatch, field 'entity_result_type' specifies a ResultType + // for all the values in field 'entity_result'). + enum ResultType { + FULL = 1; // The entire entity. + PROJECTION = 2; // A projected subset of properties. + // The entity may have no key. + // A property value may have meaning 18. + KEY_ONLY = 3; // Only the key. + } + + // The resulting entity. + required Entity entity = 1; +} + +// A query. +message Query { + // The projection to return. If not set the entire entity is returned. + repeated PropertyExpression projection = 2; + + // The kinds to query (if empty, returns entities from all kinds). + repeated KindExpression kind = 3; + + // The filter to apply (optional). + optional Filter filter = 4; + + // The order to apply to the query results (if empty, order is unspecified). + repeated PropertyOrder order = 5; + + // The properties to group by (if empty, no grouping is applied to the + // result set). + repeated PropertyReference group_by = 6; + + // A starting point for the query results. Optional. Query cursors are + // returned in query result batches. + optional bytes /* serialized QueryCursor */ start_cursor = 7; + + // An ending point for the query results. Optional. Query cursors are + // returned in query result batches. + optional bytes /* serialized QueryCursor */ end_cursor = 8; + + // The number of results to skip. Applies before limit, but after all other + // constraints (optional, defaults to 0). + optional int32 offset = 10 [default=0]; + + // The maximum number of results to return. Applies after all other + // constraints. Optional. + optional int32 limit = 11; +} + +// A representation of a kind. +message KindExpression { + // The name of the kind. + required string name = 1; +} + +// A reference to a property relative to the kind expressions. +// exactly. +message PropertyReference { + // The name of the property. + required string name = 2; +} + +// A representation of a property in a projection. +message PropertyExpression { + enum AggregationFunction { + FIRST = 1; + } + // The property to project. + required PropertyReference property = 1; + // The aggregation function to apply to the property. Optional. + // Can only be used when grouping by at least one property. Must + // then be set on all properties in the projection that are not + // being grouped by. + optional AggregationFunction aggregation_function = 2; +} + +// The desired order for a specific property. +message PropertyOrder { + enum Direction { + ASCENDING = 1; + DESCENDING = 2; + } + // The property to order by. + required PropertyReference property = 1; + // The direction to order by. + optional Direction direction = 2 [default=ASCENDING]; +} + +// A holder for any type of filter. Exactly one field should be specified. +message Filter { + // A composite filter. + optional CompositeFilter composite_filter = 1; + // A filter on a property. + optional PropertyFilter property_filter = 2; +} + +// A filter that merges the multiple other filters using the given operation. +message CompositeFilter { + enum Operator { + AND = 1; + } + + // The operator for combining multiple filters. + required Operator operator = 1; + // The list of filters to combine. + // Must contain at least one filter. + repeated Filter filter = 2; +} + +// A filter on a specific property. +message PropertyFilter { + enum Operator { + LESS_THAN = 1; + LESS_THAN_OR_EQUAL = 2; + GREATER_THAN = 3; + GREATER_THAN_OR_EQUAL = 4; + EQUAL = 5; + + HAS_ANCESTOR = 11; + } + + // The property to filter by. + required PropertyReference property = 1; + // The operator to filter by. + required Operator operator = 2; + // The value to compare the property to. + required Value value = 3; +} + +// A GQL query. +message GqlQuery { + required string query_string = 1; + // When false, the query string must not contain a literal. + optional bool allow_literal = 2 [default = false]; + // A named argument must set field GqlQueryArg.name. + // No two named arguments may have the same name. + // For each non-reserved named binding site in the query string, + // there must be a named argument with that name, + // but not necessarily the inverse. + repeated GqlQueryArg name_arg = 3; + // Numbered binding site @1 references the first numbered argument, + // effectively using 1-based indexing, rather than the usual 0. + // A numbered argument must NOT set field GqlQueryArg.name. + // For each binding site numbered i in query_string, + // there must be an ith numbered argument. + // The inverse must also be true. + repeated GqlQueryArg number_arg = 4; +} + +// A binding argument for a GQL query. +// Exactly one of fields value and cursor must be set. +message GqlQueryArg { + // Must match regex "[A-Za-z_$][A-Za-z_$0-9]*". + // Must not match regex "__.*__". + // Must not be "". + optional string name = 1; + optional Value value = 2; + optional bytes cursor = 3; +} + +// A batch of results produced by a query. +message QueryResultBatch { + // The possible values for the 'more_results' field. + enum MoreResultsType { + NOT_FINISHED = 1; // There are additional batches to fetch from this query. + MORE_RESULTS_AFTER_LIMIT = 2; // The query is finished, but there are more + // results after the limit. + NO_MORE_RESULTS = 3; // The query has been exhausted. + } + + // The result type for every entity in entityResults. + required EntityResult.ResultType entity_result_type = 1; + // The results for this batch. + repeated EntityResult entity_result = 2; + + // A cursor that points to the position after the last result in the batch. + // May be absent. + optional bytes /* serialized QueryCursor */ end_cursor = 4; + + // The state of the query after the current batch. + required MoreResultsType more_results = 5; + + // The number of results skipped because of Query.offset. + optional int32 skipped_results = 6; +} + +// A set of changes to apply. +// +// No entity in this message may have a reserved property name, +// not even a property in an entity in a value. +// No value in this message may have meaning 18, +// not even a value in an entity in another value. +// +// If entities with duplicate keys are present, an arbitrary choice will +// be made as to which is written. +message Mutation { + // Entities to upsert. + // Each upserted entity's key must have a complete path and + // must not be reserved/read-only. + repeated Entity upsert = 1; + // Entities to update. + // Each updated entity's key must have a complete path and + // must not be reserved/read-only. + repeated Entity update = 2; + // Entities to insert. + // Each inserted entity's key must have a complete path and + // must not be reserved/read-only. + repeated Entity insert = 3; + // Insert entities with a newly allocated ID. + // Each inserted entity's key must omit the final identifier in its path and + // must not be reserved/read-only. + repeated Entity insert_auto_id = 4; + // Keys of entities to delete. + // Each key must have a complete key path and must not be reserved/read-only. + repeated Key delete = 5; + // Ignore a user specified read-only period. Optional. + optional bool force = 6; +} + +// The result of applying a mutation. +message MutationResult { + // Number of index writes. + required int32 index_updates = 1; + // Keys for insertAutoId entities. One per entity from the + // request, in the same order. + repeated Key insert_auto_id_key = 2; +} + +// Options shared by read requests. +message ReadOptions { + enum ReadConsistency { + DEFAULT = 0; + STRONG = 1; + EVENTUAL = 2; + } + + // The read consistency to use. + // Cannot be set when transaction is set. + // Lookup and ancestor queries default to STRONG, global queries default to + // EVENTUAL and cannot be set to STRONG. + optional ReadConsistency read_consistency = 1 [default=DEFAULT]; + + // The transaction to use. Optional. + optional bytes /* serialized Transaction */ transaction = 2; +} + +// The request for Lookup. +message LookupRequest { + + // Options for this lookup request. Optional. + optional ReadOptions read_options = 1; + // Keys of entities to look up from the datastore. + repeated Key key = 3; +} + +// The response for Lookup. +message LookupResponse { + + // The order of results in these fields is undefined and has no relation to + // the order of the keys in the input. + + // Entities found as ResultType.FULL entities. + repeated EntityResult found = 1; + + // Entities not found as ResultType.KEY_ONLY entities. + repeated EntityResult missing = 2; + + // A list of keys that were not looked up due to resource constraints. + repeated Key deferred = 3; +} + + +// The request for RunQuery. +message RunQueryRequest { + + // The options for this query. + optional ReadOptions read_options = 1; + + // Entities are partitioned into subsets, identified by a dataset (usually + // implicitly specified by the project) and namespace ID. Queries are scoped + // to a single partition. + // This partition ID is normalized with the standard default context + // partition ID, but all other partition IDs in RunQueryRequest are + // normalized with this partition ID as the context partition ID. + optional PartitionId partition_id = 2; + + // The query to run. + // Either this field or field gql_query must be set, but not both. + optional Query query = 3; + // The GQL query to run. + // Either this field or field query must be set, but not both. + optional GqlQuery gql_query = 7; +} + +// The response for RunQuery. +message RunQueryResponse { + + // A batch of query results (always present). + optional QueryResultBatch batch = 1; + +} + +// The request for BeginTransaction. +message BeginTransactionRequest { + + enum IsolationLevel { + SNAPSHOT = 0; // Read from a consistent snapshot. Concurrent transactions + // conflict if their mutations conflict. For example: + // Read(A),Write(B) may not conflict with Read(B),Write(A), + // but Read(B),Write(B) does conflict with Read(B),Write(B). + SERIALIZABLE = 1; // Read from a consistent snapshot. Concurrent + // transactions conflict if they cannot be serialized. + // For example Read(A),Write(B) does conflict with + // Read(B),Write(A) but Read(A) may not conflict with + // Write(A). + } + + // The transaction isolation level. + optional IsolationLevel isolation_level = 1 [default=SNAPSHOT]; +} + +// The response for BeginTransaction. +message BeginTransactionResponse { + + // The transaction identifier (always present). + optional bytes /* serialized Transaction */ transaction = 1; +} + +// The request for Rollback. +message RollbackRequest { + + // The transaction identifier, returned by a call to + // beginTransaction. + required bytes /* serialized Transaction */ transaction = 1; +} + +// The response for Rollback. +message RollbackResponse { +// Empty +} + +// The request for Commit. +message CommitRequest { + + enum Mode { + TRANSACTIONAL = 1; + NON_TRANSACTIONAL = 2; + } + + // The transaction identifier, returned by a call to + // beginTransaction. Must be set when mode is TRANSACTIONAL. + optional bytes /* serialized Transaction */ transaction = 1; + // The mutation to perform. Optional. + optional Mutation mutation = 2; + // The type of commit to perform. Either TRANSACTIONAL or NON_TRANSACTIONAL. + optional Mode mode = 5 [default=TRANSACTIONAL]; +} + +// The response for Commit. +message CommitResponse { + + // The result of performing the mutation (if any). + optional MutationResult mutation_result = 1; +} + +// The request for AllocateIds. +message AllocateIdsRequest { + + // A list of keys with incomplete key paths to allocate IDs for. + // No key may be reserved/read-only. + repeated Key key = 1; +} + +// The response for AllocateIds. +message AllocateIdsResponse { + + // The keys specified in the request (in the same order), each with + // its key path completed with a newly allocated ID. + repeated Key key = 1; +} + +// Each rpc normalizes the partition IDs of the keys in its input entities, +// and always returns entities with keys with normalized partition IDs. +// (Note that applies to all entities, including entities in values.) +service DatastoreService { + // Look up some entities by key. + rpc Lookup(LookupRequest) returns (LookupResponse) { + }; + // Query for entities. + rpc RunQuery(RunQueryRequest) returns (RunQueryResponse) { + }; + // Begin a new transaction. + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse) { + }; + // Commit a transaction, optionally creating, deleting or modifying some + // entities. + rpc Commit(CommitRequest) returns (CommitResponse) { + }; + // Roll back a transaction. + rpc Rollback(RollbackRequest) returns (RollbackResponse) { + }; + // Allocate IDs for incomplete keys (useful for referencing an entity before + // it is inserted). + rpc AllocateIds(AllocateIdsRequest) returns (AllocateIdsResponse) { + }; +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/opts/option.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/opts/option.go new file mode 100644 index 0000000000..c5ccf4f56d --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/opts/option.go @@ -0,0 +1,24 @@ +// Package opts holds the DialOpts struct, configurable by +// cloud.ClientOptions to set up transports for cloud packages. +// +// This is a separate page to prevent cycles between the core +// cloud packages. +package opts + +import ( + "net/http" + + "golang.org/x/oauth2" + "google.golang.org/grpc" +) + +type DialOpt struct { + Endpoint string + Scopes []string + UserAgent string + + TokenSource oauth2.TokenSource + + HTTPClient *http.Client + GRPCClient *grpc.ClientConn +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/testutil/context.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/testutil/context.go new file mode 100644 index 0000000000..8f8846c356 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/testutil/context.go @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testutil contains helper functions for writing tests. +package testutil + +import ( + "io/ioutil" + "log" + "net/http" + "os" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" +) + +const ( + envProjID = "GCLOUD_TESTS_GOLANG_PROJECT_ID" + envPrivateKey = "GCLOUD_TESTS_GOLANG_KEY" +) + +func ProjID() string { + projID := os.Getenv(envProjID) + if projID == "" { + log.Fatal(envProjID + " must be set. See CONTRIBUTING.md for details.") + } + return projID +} + +func TokenSource(ctx context.Context, scopes ...string) oauth2.TokenSource { + key := os.Getenv(envPrivateKey) + if key == "" { + log.Fatal(envPrivateKey + " must be set. See CONTRIBUTING.md for details.") + } + jsonKey, err := ioutil.ReadFile(key) + if err != nil { + log.Fatalf("Cannot read the JSON key file, err: %v", err) + } + conf, err := google.JWTConfigFromJSON(jsonKey, scopes...) + if err != nil { + log.Fatalf("google.JWTConfigFromJSON: %v", err) + } + return conf.TokenSource(ctx) +} + +// TODO(djd): Delete this function when it's no longer used. +func Context(scopes ...string) context.Context { + ctx := oauth2.NoContext + ts := TokenSource(ctx, scopes...) + return cloud.NewContext(ProjID(), oauth2.NewClient(ctx, ts)) +} + +// TODO(djd): Delete this function when it's no longer used. +func NoAuthContext() context.Context { + return cloud.NewContext(ProjID(), &http.Client{Transport: http.DefaultTransport}) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/cancelreq.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/cancelreq.go new file mode 100644 index 0000000000..ddae71ccef --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/cancelreq.go @@ -0,0 +1,29 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.5 + +package transport + +import "net/http" + +// makeReqCancel returns a closure that cancels the given http.Request +// when called. +func makeReqCancel(req *http.Request) func(http.RoundTripper) { + c := make(chan struct{}) + req.Cancel = c + return func(http.RoundTripper) { + close(c) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/cancelreq_legacy.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/cancelreq_legacy.go new file mode 100644 index 0000000000..c11a4ddebc --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/cancelreq_legacy.go @@ -0,0 +1,31 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !go1.5 + +package transport + +import "net/http" + +// makeReqCancel returns a closure that cancels the given http.Request +// when called. +func makeReqCancel(req *http.Request) func(http.RoundTripper) { + // Go 1.4 and prior do not have a reliable way of cancelling a request. + // Transport.CancelRequest will only work if the request is already in-flight. + return func(r http.RoundTripper) { + if t, ok := r.(*http.Transport); ok { + t.CancelRequest(req) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/dial.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/dial.go new file mode 100644 index 0000000000..29624410d3 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/dial.go @@ -0,0 +1,134 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "errors" + "fmt" + "net/http" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" + "google.golang.org/cloud/internal/opts" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" +) + +// ErrHTTP is returned when on a non-200 HTTP response. +type ErrHTTP struct { + StatusCode int + Body []byte + err error +} + +func (e *ErrHTTP) Error() string { + if e.err == nil { + return fmt.Sprintf("error during call, http status code: %v %s", e.StatusCode, e.Body) + } + return e.err.Error() +} + +// NewHTTPClient returns an HTTP client for use communicating with a Google cloud +// service, configured with the given ClientOptions. It also returns the endpoint +// for the service as specified in the options. +func NewHTTPClient(ctx context.Context, opt ...cloud.ClientOption) (*http.Client, string, error) { + var o opts.DialOpt + for _, opt := range opt { + opt.Resolve(&o) + } + if o.GRPCClient != nil { + return nil, "", errors.New("unsupported GRPC base transport specified") + } + // TODO(djd): Wrap all http.Clients with appropriate internal version to add + // UserAgent header and prepend correct endpoint. + if o.HTTPClient != nil { + return o.HTTPClient, o.Endpoint, nil + } + if o.TokenSource == nil { + var err error + o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...) + if err != nil { + return nil, "", fmt.Errorf("google.DefaultTokenSource: %v", err) + } + } + return oauth2.NewClient(ctx, o.TokenSource), o.Endpoint, nil +} + +// NewProtoClient returns a ProtoClient for communicating with a Google cloud service, +// configured with the given ClientOptions. +func NewProtoClient(ctx context.Context, opt ...cloud.ClientOption) (*ProtoClient, error) { + var o opts.DialOpt + for _, opt := range opt { + opt.Resolve(&o) + } + if o.GRPCClient != nil { + return nil, errors.New("unsupported GRPC base transport specified") + } + var client *http.Client + switch { + case o.HTTPClient != nil: + if o.TokenSource != nil { + return nil, errors.New("at most one of WithTokenSource or WithBaseHTTP may be provided") + } + client = o.HTTPClient + case o.TokenSource != nil: + client = oauth2.NewClient(ctx, o.TokenSource) + default: + var err error + client, err = google.DefaultClient(ctx, o.Scopes...) + if err != nil { + return nil, err + } + } + + return &ProtoClient{ + client: client, + endpoint: o.Endpoint, + userAgent: o.UserAgent, + }, nil +} + +// DialGRPC returns a GRPC connection for use communicating with a Google cloud +// service, configured with the given ClientOptions. +func DialGRPC(ctx context.Context, opt ...cloud.ClientOption) (*grpc.ClientConn, error) { + var o opts.DialOpt + for _, opt := range opt { + opt.Resolve(&o) + } + if o.HTTPClient != nil { + return nil, errors.New("unsupported HTTP base transport specified") + } + if o.GRPCClient != nil { + return o.GRPCClient, nil + } + if o.TokenSource == nil { + var err error + o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("google.DefaultTokenSource: %v", err) + } + } + grpcOpts := []grpc.DialOption{ + grpc.WithPerRPCCredentials(oauth.TokenSource{o.TokenSource}), + grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), + } + if o.UserAgent != "" { + grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent)) + } + return grpc.Dial(o.Endpoint, grpcOpts...) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/proto.go b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/proto.go new file mode 100644 index 0000000000..05b11cde1e --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/internal/transport/proto.go @@ -0,0 +1,80 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "bytes" + "io/ioutil" + "net/http" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" +) + +type ProtoClient struct { + client *http.Client + endpoint string + userAgent string +} + +func (c *ProtoClient) Call(ctx context.Context, method string, req, resp proto.Message) error { + payload, err := proto.Marshal(req) + if err != nil { + return err + } + + httpReq, err := http.NewRequest("POST", c.endpoint+method, bytes.NewReader(payload)) + if err != nil { + return err + } + httpReq.Header.Set("Content-Type", "application/x-protobuf") + if ua := c.userAgent; ua != "" { + httpReq.Header.Set("User-Agent", ua) + } + + errc := make(chan error, 1) + cancel := makeReqCancel(httpReq) + + go func() { + r, err := c.client.Do(httpReq) + if err != nil { + errc <- err + return + } + defer r.Body.Close() + + body, err := ioutil.ReadAll(r.Body) + if r.StatusCode != http.StatusOK { + err = &ErrHTTP{ + StatusCode: r.StatusCode, + Body: body, + err: err, + } + } + if err != nil { + errc <- err + return + } + errc <- proto.Unmarshal(body, resp) + }() + + select { + case <-ctx.Done(): + cancel(c.client.Transport) // Cancel the HTTP request. + return ctx.Err() + case err := <-errc: + return err + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/key.json.enc b/Godeps/_workspace/src/google.golang.org/cloud/key.json.enc new file mode 100644 index 0000000000..2f673a84b1 Binary files /dev/null and b/Godeps/_workspace/src/google.golang.org/cloud/key.json.enc differ diff --git a/Godeps/_workspace/src/google.golang.org/cloud/logging/logging.go b/Godeps/_workspace/src/google.golang.org/cloud/logging/logging.go new file mode 100644 index 0000000000..366fb84bcf --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/logging/logging.go @@ -0,0 +1,468 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logging contains a Google Cloud Logging client. +// +// This package is experimental and subject to API changes. +package logging // import "google.golang.org/cloud/logging" + +import ( + "errors" + "io" + "log" + "sync" + "time" + + "golang.org/x/net/context" + api "google.golang.org/api/logging/v1beta3" + "google.golang.org/cloud" + "google.golang.org/cloud/internal/transport" +) + +// Scope is the OAuth2 scope necessary to use Google Cloud Logging. +const Scope = api.CloudPlatformScope + +// Level is the log level. +type Level int + +const ( + // Default means no assigned severity level. + Default Level = iota + Debug + Info + Warning + Error + Critical + Alert + Emergency + nLevel +) + +var levelName = [nLevel]string{ + Default: "", + Debug: "DEBUG", + Info: "INFO", + Warning: "WARNING", + Error: "ERROR", + Critical: "CRITICAL", + Alert: "ALERT", + Emergency: "EMERGENCY", +} + +func (v Level) String() string { + return levelName[v] +} + +// Client is a Google Cloud Logging client. +// It must be constructed via NewClient. +type Client struct { + svc *api.Service + logs *api.ProjectsLogsEntriesService + projID string + logName string + writer [nLevel]io.Writer + logger [nLevel]*log.Logger + + mu sync.Mutex + queued []*api.LogEntry + curFlush *flushCall // currently in-flight flush + flushTimer *time.Timer // nil before first use + timerActive bool // whether flushTimer is armed + inFlight int // number of log entries sent to API service but not yet ACKed + + // For testing: + timeNow func() time.Time // optional + + // ServiceName may be "appengine.googleapis.com", + // "compute.googleapis.com" or "custom.googleapis.com". + // + // The default is "custom.googleapis.com". + // + // The service name is only used by the API server to + // determine which of the labels are used to index the logs. + ServiceName string + + // CommonLabels are metadata labels that apply to all log + // entries in this request, so that you don't have to repeat + // them in each log entry's metadata.labels field. If any of + // the log entries contains a (key, value) with the same key + // that is in CommonLabels, then the entry's (key, value) + // overrides the one in CommonLabels. + CommonLabels map[string]string + + // BufferLimit is the maximum number of items to keep in memory + // before flushing. Zero means automatic. A value of 1 means to + // flush after each log entry. + // The default is currently 10,000. + BufferLimit int + + // FlushAfter optionally specifies a threshold count at which buffered + // log entries are flushed, even if the BufferInterval has not yet + // been reached. + // The default is currently 10. + FlushAfter int + + // BufferInterval is the maximum amount of time that an item + // should remain buffered in memory before being flushed to + // the logging service. + // The default is currently 1 second. + BufferInterval time.Duration + + // Overflow is a function which runs when the Log function + // overflows its configured buffer limit. If nil, the log + // entry is dropped. The return value from Overflow is + // returned by Log. + Overflow func(*Client, Entry) error +} + +func (c *Client) flushAfter() int { + if v := c.FlushAfter; v > 0 { + return v + } + return 10 +} + +func (c *Client) bufferInterval() time.Duration { + if v := c.BufferInterval; v > 0 { + return v + } + return time.Second +} + +func (c *Client) bufferLimit() int { + if v := c.BufferLimit; v > 0 { + return v + } + return 10000 +} + +func (c *Client) serviceName() string { + if v := c.ServiceName; v != "" { + return v + } + return "custom.googleapis.com" +} + +func (c *Client) now() time.Time { + if now := c.timeNow; now != nil { + return now() + } + return time.Now() +} + +// Writer returns an io.Writer for the provided log level. +// +// Each Write call on the returned Writer generates a log entry. +// +// This Writer accessor does not allocate, so callers do not need to +// cache. +func (c *Client) Writer(v Level) io.Writer { return c.writer[v] } + +// Logger returns a *log.Logger for the provided log level. +// +// A Logger for each Level is pre-allocated by NewClient with an empty +// prefix and no flags. This Logger accessor does not allocate. +// Callers wishing to use alternate flags (such as log.Lshortfile) may +// mutate the returned Logger with SetFlags. Such mutations affect all +// callers in the program. +func (c *Client) Logger(v Level) *log.Logger { return c.logger[v] } + +type levelWriter struct { + level Level + c *Client +} + +func (w levelWriter) Write(p []byte) (n int, err error) { + return len(p), w.c.Log(Entry{ + Level: w.level, + Payload: string(p), + }) +} + +// Entry is a log entry. +type Entry struct { + // Time is the time of the entry. If the zero value, the current time is used. + Time time.Time + + // Level is log entry's severity level. + // The zero value means no assigned severity level. + Level Level + + // Payload must be either a string, []byte, or something that + // marshals via the encoding/json package to a JSON object + // (and not any other type of JSON value). + Payload interface{} + + // Labels optionally specifies key/value labels for the log entry. + // Depending on the Client's ServiceName, these are indexed differently + // by the Cloud Logging Service. + // See https://cloud.google.com/logging/docs/logs_index + // The Client.Log method takes ownership of this map. + Labels map[string]string + + // TODO: de-duping id +} + +func (c *Client) apiEntry(e Entry) (*api.LogEntry, error) { + t := e.Time + if t.IsZero() { + t = c.now() + } + + ent := &api.LogEntry{ + Metadata: &api.LogEntryMetadata{ + Timestamp: t.UTC().Format(time.RFC3339Nano), + ServiceName: c.serviceName(), + Severity: e.Level.String(), + Labels: e.Labels, + }, + } + switch p := e.Payload.(type) { + case string: + ent.TextPayload = p + case []byte: + ent.TextPayload = string(p) + default: + ent.StructPayload = api.LogEntryStructPayload(p) + } + return ent, nil +} + +// LogSync logs e synchronously without any buffering. +// This is mostly intended for debugging or critical errors. +func (c *Client) LogSync(e Entry) error { + ent, err := c.apiEntry(e) + if err != nil { + return err + } + _, err = c.logs.Write(c.projID, c.logName, &api.WriteLogEntriesRequest{ + CommonLabels: c.CommonLabels, + Entries: []*api.LogEntry{ent}, + }).Do() + return err +} + +var ErrOverflow = errors.New("logging: log entry overflowed buffer limits") + +// Log queues an entry to be sent to the logging service, subject to the +// Client's parameters. By default, the log will be flushed within +// one second. +// Log only returns an error if the entry is invalid or the queue is at +// capacity. If the queue is at capacity and the entry can't be added, +// Log returns either ErrOverflow when c.Overflow is nil, or the +// value returned by c.Overflow. +func (c *Client) Log(e Entry) error { + ent, err := c.apiEntry(e) + if err != nil { + return err + } + + c.mu.Lock() + buffered := len(c.queued) + c.inFlight + + if buffered >= c.bufferLimit() { + c.mu.Unlock() + if fn := c.Overflow; fn != nil { + return fn(c, e) + } + return ErrOverflow + } + defer c.mu.Unlock() + + c.queued = append(c.queued, ent) + if len(c.queued) >= c.flushAfter() { + c.scheduleFlushLocked(0) + return nil + } + c.scheduleFlushLocked(c.bufferInterval()) + return nil +} + +// c.mu must be held. +// +// d will be one of two values: either c.BufferInterval (or its +// default value) or 0. +func (c *Client) scheduleFlushLocked(d time.Duration) { + if c.inFlight > 0 { + // For now to keep things simple, only allow one HTTP + // request in flight at a time. + return + } + switch { + case c.flushTimer == nil: + // First flush. + c.timerActive = true + c.flushTimer = time.AfterFunc(d, c.timeoutFlush) + case c.timerActive && d == 0: + // Make it happen sooner. For example, this is the + // case of transitioning from a 1 second flush after + // the 1st item to an immediate flush after the 10th + // item. + c.flushTimer.Reset(0) + case !c.timerActive: + c.timerActive = true + c.flushTimer.Reset(d) + default: + // else timer was already active, also at d > 0, + // so we don't touch it and let it fire as previously + // scheduled. + } +} + +// timeoutFlush runs in its own goroutine (from time.AfterFunc) and +// flushes c.queued. +func (c *Client) timeoutFlush() { + c.mu.Lock() + c.timerActive = false + c.mu.Unlock() + if err := c.Flush(); err != nil { + // schedule another try + // TODO: smarter back-off? + c.mu.Lock() + c.scheduleFlushLocked(5 * time.Second) + c.mu.Unlock() + } +} + +// Ping reports whether the client's connection to Google Cloud +// Logging and the authentication configuration are valid. +func (c *Client) Ping() error { + _, err := c.logs.Write(c.projID, c.logName, &api.WriteLogEntriesRequest{ + Entries: []*api.LogEntry{}, + }).Do() + return err +} + +// Flush flushes any buffered log entries. +func (c *Client) Flush() error { + var numFlush int + c.mu.Lock() + for { + // We're already flushing (or we just started flushing + // ourselves), so wait for it to finish. + if f := c.curFlush; f != nil { + wasEmpty := len(c.queued) == 0 + c.mu.Unlock() + <-f.donec // wait for it + numFlush++ + // Terminate whenever there's an error, we've + // already flushed twice (one that was already + // in-flight when flush was called, and then + // one we instigated), or the queue was empty + // when we released the locked (meaning this + // in-flight flush removes everything present + // when Flush was called, and we don't need to + // kick off a new flush for things arriving + // afterward) + if f.err != nil || numFlush == 2 || wasEmpty { + return f.err + } + // Otherwise, re-obtain the lock and loop, + // starting over with seeing if a flush is in + // progress, which might've been started by a + // different goroutine before aquiring this + // lock again. + c.mu.Lock() + continue + } + + // Terminal case: + if len(c.queued) == 0 { + c.mu.Unlock() + return nil + } + + c.startFlushLocked() + } +} + +// requires c.mu be held. +func (c *Client) startFlushLocked() { + if c.curFlush != nil { + panic("internal error: flush already in flight") + } + if len(c.queued) == 0 { + panic("internal error: no items queued") + } + logEntries := c.queued + c.inFlight = len(logEntries) + c.queued = nil + + flush := &flushCall{ + donec: make(chan struct{}), + } + c.curFlush = flush + go func() { + defer close(flush.donec) + _, err := c.logs.Write(c.projID, c.logName, &api.WriteLogEntriesRequest{ + CommonLabels: c.CommonLabels, + Entries: logEntries, + }).Do() + flush.err = err + c.mu.Lock() + defer c.mu.Unlock() + c.inFlight = 0 + c.curFlush = nil + if err != nil { + c.queued = append(c.queued, logEntries...) + } else if len(c.queued) > 0 { + c.scheduleFlushLocked(c.bufferInterval()) + } + }() + +} + +const prodAddr = "https://logging.googleapis.com/" + +const userAgent = "gcloud-golang-logging/20150922" + +// NewClient returns a new log client, logging to the named log in the +// provided project. +// +// The exported fields on the returned client may be modified before +// the client is used for logging. Once log entries are in flight, +// the fields must not be modified. +func NewClient(ctx context.Context, projectID, logName string, opts ...cloud.ClientOption) (*Client, error) { + httpClient, endpoint, err := transport.NewHTTPClient(ctx, append([]cloud.ClientOption{ + cloud.WithEndpoint(prodAddr), + cloud.WithScopes(api.CloudPlatformScope), + cloud.WithUserAgent(userAgent), + }, opts...)...) + if err != nil { + return nil, err + } + svc, err := api.New(httpClient) + if err != nil { + return nil, err + } + svc.BasePath = endpoint + c := &Client{ + svc: svc, + logs: api.NewProjectsLogsEntriesService(svc), + logName: logName, + projID: projectID, + } + for i := range c.writer { + level := Level(i) + c.writer[level] = levelWriter{level, c} + c.logger[level] = log.New(c.writer[level], "", 0) + } + return c, nil +} + +// flushCall is an in-flight or completed flush. +type flushCall struct { + donec chan struct{} // closed when response is in + err error // error is valid after wg is Done +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/logging/logging_test.go b/Godeps/_workspace/src/google.golang.org/cloud/logging/logging_test.go new file mode 100644 index 0000000000..f5ff9f7f55 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/logging/logging_test.go @@ -0,0 +1,319 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging + +import ( + "errors" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + + "google.golang.org/cloud" +) + +func TestLogPayload(t *testing.T) { + lt := newLogTest(t) + defer lt.ts.Close() + + tests := []struct { + name string + entry Entry + want string + }{ + { + name: "string", + entry: Entry{ + Time: time.Unix(0, 0), + Payload: "some log string", + }, + want: `{"entries":[{"metadata":{"serviceName":"custom.googleapis.com","timestamp":"1970-01-01T00:00:00Z"},"textPayload":"some log string"}]}`, + }, + { + name: "[]byte", + entry: Entry{ + Time: time.Unix(0, 0), + Payload: []byte("some log bytes"), + }, + want: `{"entries":[{"metadata":{"serviceName":"custom.googleapis.com","timestamp":"1970-01-01T00:00:00Z"},"textPayload":"some log bytes"}]}`, + }, + { + name: "struct", + entry: Entry{ + Time: time.Unix(0, 0), + Payload: struct { + Foo string `json:"foo"` + Bar int `json:"bar,omitempty"` + }{ + Foo: "foovalue", + }, + }, + want: `{"entries":[{"metadata":{"serviceName":"custom.googleapis.com","timestamp":"1970-01-01T00:00:00Z"},"structPayload":{"foo":"foovalue"}}]}`, + }, + { + name: "map[string]interface{}", + entry: Entry{ + Time: time.Unix(0, 0), + Payload: map[string]interface{}{ + "string": "foo", + "int": 42, + }, + }, + want: `{"entries":[{"metadata":{"serviceName":"custom.googleapis.com","timestamp":"1970-01-01T00:00:00Z"},"structPayload":{"int":42,"string":"foo"}}]}`, + }, + { + name: "map[string]interface{}", + entry: Entry{ + Time: time.Unix(0, 0), + Payload: customJSONObject{}, + }, + want: `{"entries":[{"metadata":{"serviceName":"custom.googleapis.com","timestamp":"1970-01-01T00:00:00Z"},"structPayload":{"custom":"json"}}]}`, + }, + } + for _, tt := range tests { + lt.startGetRequest() + if err := lt.c.LogSync(tt.entry); err != nil { + t.Errorf("%s: LogSync = %v", tt.name, err) + continue + } + got := lt.getRequest() + if got != tt.want { + t.Errorf("%s: mismatch\n got: %s\nwant: %s\n", tt.name, got, tt.want) + } + } +} + +func TestBufferInterval(t *testing.T) { + lt := newLogTest(t) + defer lt.ts.Close() + + lt.c.CommonLabels = map[string]string{ + "common1": "one", + "common2": "two", + } + lt.c.BufferInterval = 1 * time.Millisecond // immediately, basically. + lt.c.FlushAfter = 100 // but we'll only send 1 + + lt.startGetRequest() + lt.c.Logger(Debug).Printf("log line 1") + got := lt.getRequest() + want := `{"commonLabels":{"common1":"one","common2":"two"},"entries":[{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:01Z"},"textPayload":"log line 1\n"}]}` + if got != want { + t.Errorf(" got: %s\nwant: %s\n", got, want) + } +} + +func TestFlushAfter(t *testing.T) { + lt := newLogTest(t) + defer lt.ts.Close() + + lt.c.CommonLabels = map[string]string{ + "common1": "one", + "common2": "two", + } + lt.c.BufferInterval = getRequestTimeout * 2 + lt.c.FlushAfter = 2 + + lt.c.Logger(Debug).Printf("log line 1") + lt.startGetRequest() + lt.c.Logger(Debug).Printf("log line 2") + got := lt.getRequest() + want := `{"commonLabels":{"common1":"one","common2":"two"},"entries":[{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:01Z"},"textPayload":"log line 1\n"},{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:02Z"},"textPayload":"log line 2\n"}]}` + if got != want { + t.Errorf(" got: %s\nwant: %s\n", got, want) + } +} + +func TestFlush(t *testing.T) { + lt := newLogTest(t) + defer lt.ts.Close() + lt.c.BufferInterval = getRequestTimeout * 2 + lt.c.FlushAfter = 100 // but we'll only send 1, requiring a Flush + + lt.c.Logger(Debug).Printf("log line 1") + lt.startGetRequest() + if err := lt.c.Flush(); err != nil { + t.Fatal(err) + } + got := lt.getRequest() + want := `{"entries":[{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:01Z"},"textPayload":"log line 1\n"}]}` + if got != want { + t.Errorf(" got: %s\nwant: %s\n", got, want) + } +} + +func TestOverflow(t *testing.T) { + lt := newLogTest(t) + defer lt.ts.Close() + + lt.c.FlushAfter = 1 + lt.c.BufferLimit = 5 + lt.c.BufferInterval = 1 * time.Millisecond // immediately, basically. + + someErr := errors.New("some specific error value") + lt.c.Overflow = func(c *Client, e Entry) error { + return someErr + } + + unblock := make(chan bool, 1) + inHandler := make(chan bool, 1) + lt.handlerc <- http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + inHandler <- true + <-unblock + ioutil.ReadAll(r.Body) + io.WriteString(w, "{}") // WriteLogEntriesResponse + }) + + lt.c.Logger(Debug).Printf("log line 1") + <-inHandler + lt.c.Logger(Debug).Printf("log line 2") + lt.c.Logger(Debug).Printf("log line 3") + lt.c.Logger(Debug).Printf("log line 4") + lt.c.Logger(Debug).Printf("log line 5") + + queued, inFlight := lt.c.stats() + if want := 4; queued != want { + t.Errorf("queued = %d; want %d", queued, want) + } + if want := 1; inFlight != want { + t.Errorf("inFlight = %d; want %d", inFlight, want) + } + + if err := lt.c.Log(Entry{Payload: "to overflow"}); err != someErr { + t.Errorf("Log(overflow Log entry) = %v; want someErr", err) + } + lt.startGetRequest() + unblock <- true + got := lt.getRequest() + want := `{"entries":[{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:02Z"},"textPayload":"log line 2\n"},{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:03Z"},"textPayload":"log line 3\n"},{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:04Z"},"textPayload":"log line 4\n"},{"metadata":{"serviceName":"custom.googleapis.com","severity":"DEBUG","timestamp":"1970-01-01T00:00:05Z"},"textPayload":"log line 5\n"}]}` + if got != want { + t.Errorf(" got: %s\nwant: %s\n", got, want) + } + if err := lt.c.Flush(); err != nil { + t.Fatal(err) + } + queued, inFlight = lt.c.stats() + if want := 0; queued != want { + t.Errorf("queued = %d; want %d", queued, want) + } + if want := 0; inFlight != want { + t.Errorf("inFlight = %d; want %d", inFlight, want) + } +} + +func (c *Client) stats() (queued, inFlight int) { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.queued), c.inFlight +} + +type customJSONObject struct{} + +func (customJSONObject) MarshalJSON() ([]byte, error) { + return []byte(`{"custom":"json"}`), nil +} + +type logTest struct { + t *testing.T + ts *httptest.Server + c *Client + handlerc chan<- http.Handler + + bodyc chan string +} + +func newLogTest(t *testing.T) *logTest { + handlerc := make(chan http.Handler, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case h := <-handlerc: + h.ServeHTTP(w, r) + default: + slurp, _ := ioutil.ReadAll(r.Body) + t.Errorf("Unexpected HTTP request received: %s", slurp) + w.WriteHeader(500) + io.WriteString(w, "unexpected HTTP request") + } + })) + c, err := NewClient(context.Background(), "PROJ-ID", "LOG-NAME", + cloud.WithEndpoint(ts.URL), + cloud.WithTokenSource(dummyTokenSource{}), // prevent DefaultTokenSource + ) + if err != nil { + t.Fatal(err) + } + var clock struct { + sync.Mutex + now time.Time + } + c.timeNow = func() time.Time { + clock.Lock() + defer clock.Unlock() + if clock.now.IsZero() { + clock.now = time.Unix(0, 0) + } + clock.now = clock.now.Add(1 * time.Second) + return clock.now + } + return &logTest{ + t: t, + ts: ts, + c: c, + handlerc: handlerc, + } +} + +func (lt *logTest) startGetRequest() { + bodyc := make(chan string, 1) + lt.bodyc = bodyc + lt.handlerc <- http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + slurp, err := ioutil.ReadAll(r.Body) + if err != nil { + bodyc <- "ERROR: " + err.Error() + } else { + bodyc <- string(slurp) + } + io.WriteString(w, "{}") // a complete WriteLogEntriesResponse JSON struct + }) +} + +const getRequestTimeout = 5 * time.Second + +func (lt *logTest) getRequest() string { + if lt.bodyc == nil { + lt.t.Fatalf("getRequest called without previous startGetRequest") + } + select { + case v := <-lt.bodyc: + return strings.TrimSpace(v) + case <-time.After(getRequestTimeout): + lt.t.Fatalf("timeout waiting for request") + panic("unreachable") + } +} + +// dummyTokenSource returns fake oauth2 tokens for local testing. +type dummyTokenSource struct{} + +func (dummyTokenSource) Token() (*oauth2.Token, error) { + return new(oauth2.Token), nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/option.go b/Godeps/_workspace/src/google.golang.org/cloud/option.go new file mode 100644 index 0000000000..d4a5aea294 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/option.go @@ -0,0 +1,100 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud + +import ( + "net/http" + + "golang.org/x/oauth2" + "google.golang.org/cloud/internal/opts" + "google.golang.org/grpc" +) + +// ClientOption is used when construct clients for each cloud service. +type ClientOption interface { + // Resolve configures the given DialOpts for this option. + Resolve(*opts.DialOpt) +} + +// WithTokenSource returns a ClientOption that specifies an OAuth2 token +// source to be used as the basis for authentication. +func WithTokenSource(s oauth2.TokenSource) ClientOption { + return withTokenSource{s} +} + +type withTokenSource struct{ ts oauth2.TokenSource } + +func (w withTokenSource) Resolve(o *opts.DialOpt) { + o.TokenSource = w.ts +} + +// WithEndpoint returns a ClientOption that overrides the default endpoint +// to be used for a service. +func WithEndpoint(url string) ClientOption { + return withEndpoint(url) +} + +type withEndpoint string + +func (w withEndpoint) Resolve(o *opts.DialOpt) { + o.Endpoint = string(w) +} + +// WithScopes returns a ClientOption that overrides the default OAuth2 scopes +// to be used for a service. +func WithScopes(scope ...string) ClientOption { + return withScopes(scope) +} + +type withScopes []string + +func (w withScopes) Resolve(o *opts.DialOpt) { + o.Scopes = []string(w) +} + +// WithUserAgent returns a ClientOption that sets the User-Agent. +func WithUserAgent(ua string) ClientOption { + return withUA(ua) +} + +type withUA string + +func (w withUA) Resolve(o *opts.DialOpt) { o.UserAgent = string(w) } + +// WithBaseHTTP returns a ClientOption that specifies the HTTP client to +// use as the basis of communications. This option may only be used with +// services that support HTTP as their communication transport. +func WithBaseHTTP(client *http.Client) ClientOption { + return withBaseHTTP{client} +} + +type withBaseHTTP struct{ client *http.Client } + +func (w withBaseHTTP) Resolve(o *opts.DialOpt) { + o.HTTPClient = w.client +} + +// WithBaseGRPC returns a ClientOption that specifies the GRPC client +// connection to use as the basis of communications. This option many only be +// used with services that support HRPC as their communication transport. +func WithBaseGRPC(client *grpc.ClientConn) ClientOption { + return withBaseGRPC{client} +} + +type withBaseGRPC struct{ client *grpc.ClientConn } + +func (w withBaseGRPC) Resolve(o *opts.DialOpt) { + o.GRPCClient = w.client +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/pubsub/example_test.go b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/example_test.go new file mode 100644 index 0000000000..939ca18231 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/example_test.go @@ -0,0 +1,80 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pubsub_test + +import ( + "io/ioutil" + "log" + "testing" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" + "google.golang.org/cloud/pubsub" +) + +// TODO(jbd): Remove after Go 1.4. +// Related to https://codereview.appspot.com/107320046 +func TestA(t *testing.T) {} + +func Example_auth() context.Context { + // Initialize an authorized context with Google Developers Console + // JSON key. Read the google package examples to learn more about + // different authorization flows you can use. + // http://godoc.org/golang.org/x/oauth2/google + jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json") + if err != nil { + log.Fatal(err) + } + conf, err := google.JWTConfigFromJSON( + jsonKey, + pubsub.ScopeCloudPlatform, + pubsub.ScopePubSub, + ) + if err != nil { + log.Fatal(err) + } + ctx := cloud.NewContext("project-id", conf.Client(oauth2.NoContext)) + // See the other samples to learn how to use the context. + return ctx +} + +func ExamplePublish() { + ctx := Example_auth() + + msgIDs, err := pubsub.Publish(ctx, "topic1", &pubsub.Message{ + Data: []byte("hello world"), + }) + if err != nil { + log.Fatal(err) + } + log.Printf("Published a message with a message id: %s\n", msgIDs[0]) +} + +func ExamplePull() { + ctx := Example_auth() + + // E.g. c.CreateSub("sub1", "topic1", time.Duration(0), "") + msgs, err := pubsub.Pull(ctx, "sub1", 1) + if err != nil { + log.Fatal(err) + } + log.Printf("New message arrived: %v\n", msgs[0]) + if err := pubsub.Ack(ctx, "sub1", msgs[0].AckID); err != nil { + log.Fatal(err) + } + log.Println("Acknowledged message") +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/pubsub/integration_test.go b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/integration_test.go new file mode 100644 index 0000000000..9ce68af9d6 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/integration_test.go @@ -0,0 +1,144 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build integration + +package pubsub + +import ( + "fmt" + "testing" + "time" + + "google.golang.org/cloud/internal/testutil" +) + +func TestAll(t *testing.T) { + ctx := testutil.Context(ScopePubSub, ScopeCloudPlatform) + now := time.Now() + topic := fmt.Sprintf("topic-%d", now.Unix()) + subscription := fmt.Sprintf("subscription-%d", now.Unix()) + + if err := CreateTopic(ctx, topic); err != nil { + t.Errorf("CreateTopic error: %v", err) + } + + if err := CreateSub(ctx, subscription, topic, time.Duration(0), ""); err != nil { + t.Errorf("CreateSub error: %v", err) + } + + exists, err := TopicExists(ctx, topic) + if err != nil { + t.Fatalf("TopicExists error: %v", err) + } + if !exists { + t.Errorf("topic %s should exist, but it doesn't", topic) + } + + exists, err = SubExists(ctx, subscription) + if err != nil { + t.Fatalf("SubExists error: %v", err) + } + if !exists { + t.Errorf("subscription %s should exist, but it doesn't", subscription) + } + + max := 10 + msgs := make([]*Message, max) + expectedMsgs := make(map[string]bool, max) + for i := 0; i < max; i++ { + text := fmt.Sprintf("a message with an index %d", i) + attrs := make(map[string]string) + attrs["foo"] = "bar" + msgs[i] = &Message{ + Data: []byte(text), + Attributes: attrs, + } + expectedMsgs[text] = false + } + + ids, err := Publish(ctx, topic, msgs...) + if err != nil { + t.Fatalf("Publish (1) error: %v", err) + } + + if len(ids) != max { + t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), max) + } + + expectedIDs := make(map[string]bool, max) + for _, id := range ids { + expectedIDs[id] = false + } + + received, err := PullWait(ctx, subscription, max) + if err != nil { + t.Fatalf("PullWait error: %v", err) + } + + if len(received) != max { + t.Errorf("unexpected number of messages received; %d, want %d", len(received), max) + } + + for _, msg := range received { + expectedMsgs[string(msg.Data)] = true + expectedIDs[msg.ID] = true + if msg.Attributes["foo"] != "bar" { + t.Errorf("message attribute foo is expected to be 'bar', found '%s'", msg.Attributes["foo"]) + } + } + + for msg, found := range expectedMsgs { + if !found { + t.Errorf("message '%s' should be received", msg) + } + } + + for id, found := range expectedIDs { + if !found { + t.Errorf("message with the message id '%s' should be received", id) + } + } + + // base64 test + data := "=@~" + msg := &Message{ + Data: []byte(data), + } + _, err = Publish(ctx, topic, msg) + if err != nil { + t.Fatalf("Publish (2) error: %v", err) + } + + received, err = PullWait(ctx, subscription, 1) + if err != nil { + t.Fatalf("PullWait error: %v", err) + } + if len(received) != 1 { + t.Fatalf("unexpected number of messages received; %d, want %d", len(received), 1) + } + if string(received[0].Data) != data { + t.Errorf("unexpexted message received; %s, want %s", string(received[0].Data), data) + } + + err = DeleteSub(ctx, subscription) + if err != nil { + t.Errorf("DeleteSub error: %v", err) + } + + err = DeleteTopic(ctx, topic) + if err != nil { + t.Errorf("DeleteTopic error: %v", err) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/pubsub/pubsub.go b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/pubsub.go new file mode 100644 index 0000000000..23d4ce01bc --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/pubsub.go @@ -0,0 +1,323 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pubsub contains a Google Cloud Pub/Sub client. +// +// This package is experimental and may make backwards-incompatible changes. +// +// More information about Google Cloud Pub/Sub is available at +// https://cloud.google.com/pubsub/docs +package pubsub // import "google.golang.org/cloud/pubsub" + +import ( + "encoding/base64" + "errors" + "fmt" + "net/http" + "time" + + "google.golang.org/api/googleapi" + raw "google.golang.org/api/pubsub/v1beta2" + "google.golang.org/cloud" + "google.golang.org/cloud/internal" + "google.golang.org/cloud/internal/transport" + + "golang.org/x/net/context" +) + +const ( + // ScopePubSub grants permissions to view and manage Pub/Sub + // topics and subscriptions. + ScopePubSub = "https://www.googleapis.com/auth/pubsub" + + // ScopeCloudPlatform grants permissions to view and manage your data + // across Google Cloud Platform services. + ScopeCloudPlatform = "https://www.googleapis.com/auth/cloud-platform" +) + +const prodAddr = "https://pubsub.googleapis.com/" +const userAgent = "gcloud-golang-pubsub/20151008" + +// batchLimit is maximun size of a single batch. +const batchLimit = 1000 + +// Message represents a Pub/Sub message. +type Message struct { + // ID identifies this message. + ID string + + // AckID is the identifier to acknowledge this message. + AckID string + + // Data is the actual data in the message. + Data []byte + + // Attributes represents the key-value pairs the current message + // is labelled with. + Attributes map[string]string +} + +// Client is a Google Pub/Sub client, which may be used to perform Pub/Sub operations with a project. +// Note: Some operations are not yet available via Client, and must be performed via the legacy standalone functions. +// It must be constructed via NewClient. +type Client struct { + projectID string + s *raw.Service +} + +// NewClient create a new PubSub client. +func NewClient(ctx context.Context, projectID string, opts ...cloud.ClientOption) (*Client, error) { + o := []cloud.ClientOption{ + cloud.WithEndpoint(prodAddr), + cloud.WithScopes(raw.PubsubScope, raw.CloudPlatformScope), + cloud.WithUserAgent(userAgent), + } + o = append(o, opts...) + httpClient, endpoint, err := transport.NewHTTPClient(ctx, o...) + if err != nil { + return nil, fmt.Errorf("dialing: %v", err) + } + s, err := raw.New(httpClient) + if err != nil { + return nil, err + } + + s.BasePath = endpoint + c := &Client{ + projectID: projectID, + s: s, + } + + return c, nil +} + +// TODO(jbd): Add subscription and topic listing. + +// CreateSub creates a Pub/Sub subscription on the backend. +// A subscription should subscribe to an existing topic. +// +// The messages that haven't acknowledged will be pushed back to the +// subscription again when the default acknowledgement deadline is +// reached. You can override the default deadline by providing a +// non-zero deadline. Deadline must not be specified to +// precision greater than one second. +// +// As new messages are being queued on the subscription, you +// may recieve push notifications regarding to the new arrivals. +// To receive notifications of new messages in the queue, +// specify an endpoint callback URL. +// If endpoint is an empty string the backend will not notify the +// client of new messages. +// +// If the subscription already exists an error will be returned. +func CreateSub(ctx context.Context, name string, topic string, deadline time.Duration, endpoint string) error { + sub := &raw.Subscription{ + Topic: fullTopicName(internal.ProjID(ctx), topic), + } + if int64(deadline) > 0 { + if !isSec(deadline) { + return errors.New("pubsub: deadline must not be specified to precision greater than one second") + } + sub.AckDeadlineSeconds = int64(deadline / time.Second) + } + if endpoint != "" { + sub.PushConfig = &raw.PushConfig{PushEndpoint: endpoint} + } + _, err := rawService(ctx).Projects.Subscriptions.Create(fullSubName(internal.ProjID(ctx), name), sub).Do() + return err +} + +// DeleteSub deletes the subscription. +func DeleteSub(ctx context.Context, name string) error { + _, err := rawService(ctx).Projects.Subscriptions.Delete(fullSubName(internal.ProjID(ctx), name)).Do() + return err +} + +// ModifyAckDeadline modifies the acknowledgement deadline +// for the messages retrieved from the specified subscription. +// Deadline must not be specified to precision greater than one second. +func ModifyAckDeadline(ctx context.Context, sub string, id string, deadline time.Duration) error { + if !isSec(deadline) { + return errors.New("pubsub: deadline must not be specified to precision greater than one second") + } + _, err := rawService(ctx).Projects.Subscriptions.ModifyAckDeadline(fullSubName(internal.ProjID(ctx), sub), &raw.ModifyAckDeadlineRequest{ + AckDeadlineSeconds: int64(deadline / time.Second), + AckId: id, + }).Do() + return err +} + +// ModifyPushEndpoint modifies the URL endpoint to modify the resource +// to handle push notifications coming from the Pub/Sub backend +// for the specified subscription. +func ModifyPushEndpoint(ctx context.Context, sub, endpoint string) error { + _, err := rawService(ctx).Projects.Subscriptions.ModifyPushConfig(fullSubName(internal.ProjID(ctx), sub), &raw.ModifyPushConfigRequest{ + PushConfig: &raw.PushConfig{ + PushEndpoint: endpoint, + }, + }).Do() + return err +} + +// SubExists returns true if subscription exists. +func SubExists(ctx context.Context, name string) (bool, error) { + _, err := rawService(ctx).Projects.Subscriptions.Get(fullSubName(internal.ProjID(ctx), name)).Do() + if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// Ack acknowledges one or more Pub/Sub messages on the +// specified subscription. +func Ack(ctx context.Context, sub string, id ...string) error { + for idx, ackID := range id { + if ackID == "" { + return fmt.Errorf("pubsub: empty ackID detected at index %d", idx) + } + } + _, err := rawService(ctx).Projects.Subscriptions.Acknowledge(fullSubName(internal.ProjID(ctx), sub), &raw.AcknowledgeRequest{ + AckIds: id, + }).Do() + return err +} + +func toMessage(resp *raw.ReceivedMessage) (*Message, error) { + if resp.Message == nil { + return &Message{AckID: resp.AckId}, nil + } + data, err := base64.StdEncoding.DecodeString(resp.Message.Data) + if err != nil { + return nil, err + } + return &Message{ + AckID: resp.AckId, + Data: data, + Attributes: resp.Message.Attributes, + ID: resp.Message.MessageId, + }, nil +} + +// Pull pulls messages from the subscription. It returns up to n +// number of messages, and n could not be larger than 100. +func Pull(ctx context.Context, sub string, n int) ([]*Message, error) { + return pull(ctx, sub, n, true) +} + +// PullWait pulls messages from the subscription. If there are not +// enough messages left in the subscription queue, it will block until +// at least n number of messages arrive or timeout occurs, and n could +// not be larger than 100. +func PullWait(ctx context.Context, sub string, n int) ([]*Message, error) { + return pull(ctx, sub, n, false) +} + +func pull(ctx context.Context, sub string, n int, retImmediately bool) ([]*Message, error) { + if n < 1 || n > batchLimit { + return nil, fmt.Errorf("pubsub: cannot pull less than one, more than %d messages, but %d was given", batchLimit, n) + } + resp, err := rawService(ctx).Projects.Subscriptions.Pull(fullSubName(internal.ProjID(ctx), sub), &raw.PullRequest{ + ReturnImmediately: retImmediately, + MaxMessages: int64(n), + }).Do() + if err != nil { + return nil, err + } + msgs := make([]*Message, len(resp.ReceivedMessages)) + for i := 0; i < len(resp.ReceivedMessages); i++ { + msg, err := toMessage(resp.ReceivedMessages[i]) + if err != nil { + return nil, fmt.Errorf("pubsub: cannot decode the retrieved message at index: %d, PullResponse: %+v", i, resp.ReceivedMessages[i]) + } + msgs[i] = msg + } + return msgs, nil +} + +// CreateTopic creates a new topic with the specified name on the backend. +// It will return an error if topic already exists. +func CreateTopic(ctx context.Context, name string) error { + _, err := rawService(ctx).Projects.Topics.Create(fullTopicName(internal.ProjID(ctx), name), &raw.Topic{}).Do() + return err +} + +// DeleteTopic deletes the specified topic. +func DeleteTopic(ctx context.Context, name string) error { + _, err := rawService(ctx).Projects.Topics.Delete(fullTopicName(internal.ProjID(ctx), name)).Do() + return err +} + +// TopicExists returns true if a topic exists with the specified name. +func TopicExists(ctx context.Context, name string) (bool, error) { + _, err := rawService(ctx).Projects.Topics.Get(fullTopicName(internal.ProjID(ctx), name)).Do() + if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// Publish publish messages to the topic's subscribers. It returns +// message IDs upon success. +func Publish(ctx context.Context, topic string, msgs ...*Message) ([]string, error) { + var rawMsgs []*raw.PubsubMessage + if len(msgs) == 0 { + return nil, errors.New("pubsub: no messages to publish") + } + if len(msgs) > batchLimit { + return nil, fmt.Errorf("pubsub: %d messages given, but maximum batch size is %d", len(msgs), batchLimit) + } + rawMsgs = make([]*raw.PubsubMessage, len(msgs)) + for i, msg := range msgs { + rawMsgs[i] = &raw.PubsubMessage{ + Data: base64.StdEncoding.EncodeToString(msg.Data), + Attributes: msg.Attributes, + } + } + resp, err := rawService(ctx).Projects.Topics.Publish(fullTopicName(internal.ProjID(ctx), topic), &raw.PublishRequest{ + Messages: rawMsgs, + }).Do() + if err != nil { + return nil, err + } + return resp.MessageIds, nil +} + +// fullSubName returns the fully qualified name for a subscription. +// E.g. /subscriptions/project-id/subscription-name. +func fullSubName(proj, name string) string { + return fmt.Sprintf("projects/%s/subscriptions/%s", proj, name) +} + +// fullTopicName returns the fully qualified name for a topic. +// E.g. /topics/project-id/topic-name. +func fullTopicName(proj, name string) string { + return fmt.Sprintf("projects/%s/topics/%s", proj, name) +} + +func isSec(dur time.Duration) bool { + return dur%time.Second == 0 +} + +func rawService(ctx context.Context) *raw.Service { + return internal.Service(ctx, "pubsub", func(hc *http.Client) interface{} { + svc, _ := raw.New(hc) + return svc + }).(*raw.Service) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/pubsub/pubsub_test.go b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/pubsub_test.go new file mode 100644 index 0000000000..df73f11f2b --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/pubsub/pubsub_test.go @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pubsub + +import ( + "net/http" + "strings" + "testing" + "time" + + "google.golang.org/cloud" +) + +func TestIsSec(t *testing.T) { + tests := map[time.Duration]bool{ + time.Second: true, + 5 * time.Second: true, + time.Hour: true, + time.Millisecond: false, + time.Second + time.Microsecond: false, + } + for dur, expected := range tests { + if isSec(dur) != expected { + t.Errorf("%v is more precise than a second", dur) + } + } +} + +func TestEmptyAckID(t *testing.T) { + ctx := cloud.NewContext("project-id", &http.Client{}) + id := []string{"test", ""} + err := Ack(ctx, "sub", id...) + + if err == nil || !strings.Contains(err.Error(), "index 1") { + t.Errorf("Ack should report an error indicating the id is empty. Got: %v", err) + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/acl.go b/Godeps/_workspace/src/google.golang.org/cloud/storage/acl.go new file mode 100644 index 0000000000..71c5800a86 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/acl.go @@ -0,0 +1,176 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + + "golang.org/x/net/context" + raw "google.golang.org/api/storage/v1" +) + +// ACLRole is the the access permission for the entity. +type ACLRole string + +const ( + RoleOwner ACLRole = "OWNER" + RoleReader ACLRole = "READER" +) + +// ACLEntity is an entity holding an ACL permission. +// +// It could be in the form of: +// "user-", "user-","group-", "group-", +// "domain-" and "project-team-". +// +// Or one of the predefined constants: AllUsers, AllAuthenticatedUsers. +type ACLEntity string + +const ( + AllUsers ACLEntity = "allUsers" + AllAuthenticatedUsers ACLEntity = "allAuthenticatedUsers" +) + +// ACLRule represents an access control list rule entry for a Google Cloud Storage object or bucket. +// A bucket is a Google Cloud Storage container whose name is globally unique and contains zero or +// more objects. An object is a blob of data that is stored in a bucket. +type ACLRule struct { + // Entity identifies the entity holding the current rule's permissions. + Entity ACLEntity + + // Role is the the access permission for the entity. + Role ACLRole +} + +// DefaultACL returns the default object ACL entries for the named bucket. +func DefaultACL(ctx context.Context, bucket string) ([]ACLRule, error) { + acls, err := rawService(ctx).DefaultObjectAccessControls.List(bucket).Context(ctx).Do() + if err != nil { + return nil, fmt.Errorf("storage: error listing default object ACL for bucket %q: %v", bucket, err) + } + r := make([]ACLRule, 0, len(acls.Items)) + for _, v := range acls.Items { + if m, ok := v.(map[string]interface{}); ok { + entity, ok1 := m["entity"].(string) + role, ok2 := m["role"].(string) + if ok1 && ok2 { + r = append(r, ACLRule{Entity: ACLEntity(entity), Role: ACLRole(role)}) + } + } + } + return r, nil +} + +// PutDefaultACLRule saves the named default object ACL entity with the provided role for the named bucket. +func PutDefaultACLRule(ctx context.Context, bucket string, entity ACLEntity, role ACLRole) error { + acl := &raw.ObjectAccessControl{ + Bucket: bucket, + Entity: string(entity), + Role: string(role), + } + _, err := rawService(ctx).DefaultObjectAccessControls.Update(bucket, string(entity), acl).Context(ctx).Do() + if err != nil { + return fmt.Errorf("storage: error updating default ACL rule for bucket %q, entity %q: %v", bucket, entity, err) + } + return nil +} + +// DeleteDefaultACLRule deletes the named default ACL entity for the named bucket. +func DeleteDefaultACLRule(ctx context.Context, bucket string, entity ACLEntity) error { + err := rawService(ctx).DefaultObjectAccessControls.Delete(bucket, string(entity)).Context(ctx).Do() + if err != nil { + return fmt.Errorf("storage: error deleting default ACL rule for bucket %q, entity %q: %v", bucket, entity, err) + } + return nil +} + +// BucketACL returns the ACL entries for the named bucket. +func BucketACL(ctx context.Context, bucket string) ([]ACLRule, error) { + acls, err := rawService(ctx).BucketAccessControls.List(bucket).Context(ctx).Do() + if err != nil { + return nil, fmt.Errorf("storage: error listing bucket ACL for bucket %q: %v", bucket, err) + } + r := make([]ACLRule, len(acls.Items)) + for i, v := range acls.Items { + r[i].Entity = ACLEntity(v.Entity) + r[i].Role = ACLRole(v.Role) + } + return r, nil +} + +// PutBucketACLRule saves the named ACL entity with the provided role for the named bucket. +func PutBucketACLRule(ctx context.Context, bucket string, entity ACLEntity, role ACLRole) error { + acl := &raw.BucketAccessControl{ + Bucket: bucket, + Entity: string(entity), + Role: string(role), + } + _, err := rawService(ctx).BucketAccessControls.Update(bucket, string(entity), acl).Context(ctx).Do() + if err != nil { + return fmt.Errorf("storage: error updating bucket ACL rule for bucket %q, entity %q: %v", bucket, entity, err) + } + return nil +} + +// DeleteBucketACLRule deletes the named ACL entity for the named bucket. +func DeleteBucketACLRule(ctx context.Context, bucket string, entity ACLEntity) error { + err := rawService(ctx).BucketAccessControls.Delete(bucket, string(entity)).Context(ctx).Do() + if err != nil { + return fmt.Errorf("storage: error deleting bucket ACL rule for bucket %q, entity %q: %v", bucket, entity, err) + } + return nil +} + +// ACL returns the ACL entries for the named object. +func ACL(ctx context.Context, bucket, object string) ([]ACLRule, error) { + acls, err := rawService(ctx).ObjectAccessControls.List(bucket, object).Context(ctx).Do() + if err != nil { + return nil, fmt.Errorf("storage: error listing object ACL for bucket %q, file %q: %v", bucket, object, err) + } + r := make([]ACLRule, 0, len(acls.Items)) + for _, v := range acls.Items { + if m, ok := v.(map[string]interface{}); ok { + entity, ok1 := m["entity"].(string) + role, ok2 := m["role"].(string) + if ok1 && ok2 { + r = append(r, ACLRule{Entity: ACLEntity(entity), Role: ACLRole(role)}) + } + } + } + return r, nil +} + +// PutACLRule saves the named ACL entity with the provided role for the named object. +func PutACLRule(ctx context.Context, bucket, object string, entity ACLEntity, role ACLRole) error { + acl := &raw.ObjectAccessControl{ + Bucket: bucket, + Entity: string(entity), + Role: string(role), + } + _, err := rawService(ctx).ObjectAccessControls.Update(bucket, object, string(entity), acl).Context(ctx).Do() + if err != nil { + return fmt.Errorf("storage: error updating object ACL rule for bucket %q, file %q, entity %q: %v", bucket, object, entity, err) + } + return nil +} + +// DeleteACLRule deletes the named ACL entity for the named object. +func DeleteACLRule(ctx context.Context, bucket, object string, entity ACLEntity) error { + err := rawService(ctx).ObjectAccessControls.Delete(bucket, object, string(entity)).Context(ctx).Do() + if err != nil { + return fmt.Errorf("storage: error deleting object ACL rule for bucket %q, file %q, entity %q: %v", bucket, object, entity, err) + } + return nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/example_test.go b/Godeps/_workspace/src/google.golang.org/cloud/storage/example_test.go new file mode 100644 index 0000000000..38c543021c --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/example_test.go @@ -0,0 +1,150 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage_test + +import ( + "io/ioutil" + "log" + "testing" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" + "google.golang.org/cloud/storage" +) + +// TODO(jbd): Remove after Go 1.4. +// Related to https://codereview.appspot.com/107320046 +func TestA(t *testing.T) {} + +func Example_auth() context.Context { + // Initialize an authorized context with Google Developers Console + // JSON key. Read the google package examples to learn more about + // different authorization flows you can use. + // http://godoc.org/golang.org/x/oauth2/google + jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json") + if err != nil { + log.Fatal(err) + } + conf, err := google.JWTConfigFromJSON( + jsonKey, + storage.ScopeFullControl, + ) + if err != nil { + log.Fatal(err) + } + ctx := cloud.NewContext("project-id", conf.Client(oauth2.NoContext)) + // Use the context (see other examples) + return ctx +} + +func ExampleListObjects() { + ctx := Example_auth() + + var query *storage.Query + for { + // If you are using this package on App Engine Managed VMs runtime, + // you can init a bucket client with your app's default bucket name. + // See http://godoc.org/google.golang.org/appengine/file#DefaultBucketName. + objects, err := storage.ListObjects(ctx, "bucketname", query) + if err != nil { + log.Fatal(err) + } + for _, obj := range objects.Results { + log.Printf("object name: %s, size: %v", obj.Name, obj.Size) + } + // if there are more results, objects.Next + // will be non-nil. + query = objects.Next + if query == nil { + break + } + } + + log.Println("paginated through all object items in the bucket you specified.") +} + +func ExampleNewReader() { + ctx := Example_auth() + + rc, err := storage.NewReader(ctx, "bucketname", "filename1") + if err != nil { + log.Fatal(err) + } + slurp, err := ioutil.ReadAll(rc) + rc.Close() + if err != nil { + log.Fatal(err) + } + + log.Println("file contents:", slurp) +} + +func ExampleNewWriter() { + ctx := Example_auth() + + wc := storage.NewWriter(ctx, "bucketname", "filename1") + wc.ContentType = "text/plain" + wc.ACL = []storage.ACLRule{{storage.AllUsers, storage.RoleReader}} + if _, err := wc.Write([]byte("hello world")); err != nil { + log.Fatal(err) + } + if err := wc.Close(); err != nil { + log.Fatal(err) + } + log.Println("updated object:", wc.Object()) +} + +func ExampleCopyObject() { + ctx := Example_auth() + + o, err := storage.CopyObject(ctx, "bucketname", "file1", "another-bucketname", "file2", nil) + if err != nil { + log.Fatal(err) + } + log.Println("copied file:", o) +} + +func ExampleDeleteObject() { + // To delete multiple objects in a bucket, first ListObjects then delete them. + ctx := Example_auth() + + // If you are using this package on App Engine Managed VMs runtime, + // you can init a bucket client with your app's default bucket name. + // See http://godoc.org/google.golang.org/appengine/file#DefaultBucketName. + const bucket = "bucketname" + + var query *storage.Query // Set up query as desired. + for { + objects, err := storage.ListObjects(ctx, bucket, query) + if err != nil { + log.Fatal(err) + } + for _, obj := range objects.Results { + log.Printf("deleting object name: %q, size: %v", obj.Name, obj.Size) + if err := storage.DeleteObject(ctx, bucket, obj.Name); err != nil { + log.Fatalf("unable to delete %q: %v", obj.Name, err) + } + } + // if there are more results, objects.Next will be non-nil. + query = objects.Next + if query == nil { + break + } + } + + log.Println("deleted all object items in the bucket you specified.") +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/integration_test.go b/Godeps/_workspace/src/google.golang.org/cloud/storage/integration_test.go new file mode 100644 index 0000000000..9c691964f6 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/integration_test.go @@ -0,0 +1,327 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build integration + +package storage + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "testing" + "time" + + "google.golang.org/cloud/internal" + "google.golang.org/cloud/internal/testutil" +) + +var ( + bucket string + contents = make(map[string][]byte) + objects = []string{"obj1", "obj2", "obj/with/slashes"} + aclObjects = []string{"acl1", "acl2"} + copyObj = "copy-object" +) + +const envBucket = "GCLOUD_TESTS_GOLANG_PROJECT_ID" + +func TestObjects(t *testing.T) { + ctx := testutil.Context(ScopeFullControl) + bucket = os.Getenv(envBucket) + + // Cleanup. + cleanup(t, "obj") + + const defaultType = "text/plain" + + // Test Writer. + for _, obj := range objects { + t.Logf("Writing %v", obj) + wc := NewWriter(ctx, bucket, obj) + wc.ContentType = defaultType + c := randomContents() + if _, err := wc.Write(c); err != nil { + t.Errorf("Write for %v failed with %v", obj, err) + } + if err := wc.Close(); err != nil { + t.Errorf("Close for %v failed with %v", obj, err) + } + contents[obj] = c + } + + // Test Reader. + for _, obj := range objects { + t.Logf("Creating a reader to read %v", obj) + rc, err := NewReader(ctx, bucket, obj) + if err != nil { + t.Errorf("Can't create a reader for %v, errored with %v", obj, err) + } + slurp, err := ioutil.ReadAll(rc) + if err != nil { + t.Errorf("Can't ReadAll object %v, errored with %v", obj, err) + } + if got, want := slurp, contents[obj]; !bytes.Equal(got, want) { + t.Errorf("Contents (%v) = %q; want %q", obj, got, want) + } + rc.Close() + + // Test SignedURL + opts := &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "GET", + MD5: []byte("202cb962ac59075b964b07152d234b70"), + Expires: time.Date(2020, time.October, 2, 10, 0, 0, 0, time.UTC), + ContentType: "application/json", + Headers: []string{"x-header1", "x-header2"}, + } + u, err := SignedURL(bucket, obj, opts) + if err != nil { + t.Fatalf("SignedURL(%q, %q) errored with %v", bucket, obj, err) + } + hc := internal.HTTPClient(ctx) + res, err := hc.Get(u) + if err != nil { + t.Fatalf("Can't get URL %q: %v", u, err) + } + slurp, err = ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("Can't ReadAll signed object %v, errored with %v", obj, err) + } + if got, want := slurp, contents[obj]; !bytes.Equal(got, want) { + t.Errorf("Contents (%v) = %q; want %q", obj, got, want) + } + res.Body.Close() + } + + // Test NotFound. + _, err := NewReader(ctx, bucket, "obj-not-exists") + if err != ErrObjectNotExist { + t.Errorf("Object should not exist, err found to be %v", err) + } + + name := objects[0] + + // Test StatObject. + o, err := StatObject(ctx, bucket, name) + if err != nil { + t.Error(err) + } + if got, want := o.Name, name; got != want { + t.Errorf("Name (%v) = %q; want %q", name, got, want) + } + if got, want := o.ContentType, defaultType; got != want { + t.Errorf("ContentType (%v) = %q; want %q", name, got, want) + } + + // Test object copy. + copy, err := CopyObject(ctx, bucket, name, bucket, copyObj, nil) + if err != nil { + t.Errorf("CopyObject failed with %v", err) + } + if copy.Name != copyObj { + t.Errorf("Copy object's name = %q; want %q", copy.Name, copyObj) + } + if copy.Bucket != bucket { + t.Errorf("Copy object's bucket = %q; want %q", copy.Bucket, bucket) + } + + // Test UpdateAttrs. + updated, err := UpdateAttrs(ctx, bucket, name, ObjectAttrs{ + ContentType: "text/html", + ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, + }) + if err != nil { + t.Errorf("UpdateAttrs failed with %v", err) + } + if want := "text/html"; updated.ContentType != want { + t.Errorf("updated.ContentType == %q; want %q", updated.ContentType, want) + } + + // Test checksums. + checksumCases := []struct { + name string + contents [][]byte + size int64 + md5 string + crc32c uint32 + }{ + { + name: "checksum-object", + contents: [][]byte{[]byte("hello"), []byte("world")}, + size: 10, + md5: "fc5e038d38a57032085441e7fe7010b0", + crc32c: 1456190592, + }, + { + name: "zero-object", + contents: [][]byte{}, + size: 0, + md5: "d41d8cd98f00b204e9800998ecf8427e", + crc32c: 0, + }, + } + for _, c := range checksumCases { + wc := NewWriter(ctx, bucket, c.name) + for _, data := range c.contents { + if _, err := wc.Write(data); err != nil { + t.Errorf("Write(%q) failed with %q", data, err) + } + } + if err = wc.Close(); err != nil { + t.Errorf("%q: close failed with %q", c.name, err) + } + obj := wc.Object() + if got, want := obj.Size, c.size; got != want { + t.Errorf("Object (%q) Size = %v; want %v", c.name, got, want) + } + if got, want := fmt.Sprintf("%x", obj.MD5), c.md5; got != want { + t.Errorf("Object (%q) MD5 = %q; want %q", c.name, got, want) + } + if got, want := obj.CRC32C, c.crc32c; got != want { + t.Errorf("Object (%q) CRC32C = %v; want %v", c.name, got, want) + } + } + + // Test public ACL. + publicObj := objects[0] + if err = PutACLRule(ctx, bucket, publicObj, AllUsers, RoleReader); err != nil { + t.Errorf("PutACLRule failed with %v", err) + } + publicCtx := testutil.NoAuthContext() + rc, err := NewReader(publicCtx, bucket, publicObj) + if err != nil { + t.Error(err) + } + slurp, err := ioutil.ReadAll(rc) + if err != nil { + t.Errorf("ReadAll failed with %v", err) + } + if string(slurp) != string(contents[publicObj]) { + t.Errorf("Public object's content is expected to be %s, found %s", contents[publicObj], slurp) + } + rc.Close() + + // Test writer error handling. + wc := NewWriter(publicCtx, bucket, publicObj) + if _, err := wc.Write([]byte("hello")); err != nil { + t.Errorf("Write unexpectedly failed with %v", err) + } + if err = wc.Close(); err == nil { + t.Error("Close expected an error, found none") + } + + // DeleteObject object. + // The rest of the other object will be deleted during + // the initial cleanup. This tests exists, so we still can cover + // deletion if there are no objects on the bucket to clean. + if err := DeleteObject(ctx, bucket, copyObj); err != nil { + t.Errorf("Deletion of %v failed with %v", copyObj, err) + } + _, err = StatObject(ctx, bucket, copyObj) + if err != ErrObjectNotExist { + t.Errorf("Copy is expected to be deleted, stat errored with %v", err) + } +} + +func TestACL(t *testing.T) { + ctx := testutil.Context(ScopeFullControl) + cleanup(t, "acl") + entity := ACLEntity("domain-google.com") + if err := PutDefaultACLRule(ctx, bucket, entity, RoleReader); err != nil { + t.Errorf("Can't put default ACL rule for the bucket, errored with %v", err) + } + for _, obj := range aclObjects { + t.Logf("Writing %v", obj) + wc := NewWriter(ctx, bucket, obj) + c := randomContents() + if _, err := wc.Write(c); err != nil { + t.Errorf("Write for %v failed with %v", obj, err) + } + if err := wc.Close(); err != nil { + t.Errorf("Close for %v failed with %v", obj, err) + } + } + name := aclObjects[0] + acl, err := ACL(ctx, bucket, name) + if err != nil { + t.Errorf("Can't retrieve ACL of %v", name) + } + aclFound := false + for _, rule := range acl { + if rule.Entity == entity && rule.Role == RoleReader { + aclFound = true + } + } + if !aclFound { + t.Error("Expected to find an ACL rule for google.com domain users, but not found") + } + if err := DeleteACLRule(ctx, bucket, name, entity); err != nil { + t.Errorf("Can't delete the ACL rule for the entity: %v", entity) + } + + if err := PutBucketACLRule(ctx, bucket, "user-jbd@google.com", RoleReader); err != nil { + t.Errorf("Error while putting bucket ACL rule: %v", err) + } + bACL, err := BucketACL(ctx, bucket) + if err != nil { + t.Errorf("Error while getting the ACL of the bucket: %v", err) + } + bACLFound := false + for _, rule := range bACL { + if rule.Entity == "user-jbd@google.com" && rule.Role == RoleReader { + bACLFound = true + } + } + if !bACLFound { + t.Error("Expected to find an ACL rule for jbd@google.com user, but not found") + } + if err := DeleteBucketACLRule(ctx, bucket, "user-jbd@google.com"); err != nil { + t.Errorf("Error while deleting bucket ACL rule: %v", err) + } +} + +func cleanup(t *testing.T, prefix string) { + ctx := testutil.Context(ScopeFullControl) + var q *Query = &Query{ + Prefix: prefix, + } + for { + o, err := ListObjects(ctx, bucket, q) + if err != nil { + t.Fatalf("Cleanup List for bucket %v failed with error: %v", bucket, err) + } + for _, obj := range o.Results { + t.Logf("Cleanup deletion of %v", obj.Name) + if err = DeleteObject(ctx, bucket, obj.Name); err != nil { + t.Fatalf("Cleanup Delete for object %v failed with %v", obj.Name, err) + } + } + if o.Next == nil { + break + } + q = o.Next + } +} + +func randomContents() []byte { + h := md5.New() + io.WriteString(h, fmt.Sprintf("hello world%d", rand.Intn(100000))) + return h.Sum(nil) +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/storage.go b/Godeps/_workspace/src/google.golang.org/cloud/storage/storage.go new file mode 100644 index 0000000000..8aa70ff429 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/storage.go @@ -0,0 +1,350 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package storage contains a Google Cloud Storage client. +// +// This package is experimental and may make backwards-incompatible changes. +package storage // import "google.golang.org/cloud/storage" + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "google.golang.org/cloud/internal" + + "golang.org/x/net/context" + "google.golang.org/api/googleapi" + raw "google.golang.org/api/storage/v1" +) + +var ( + ErrBucketNotExist = errors.New("storage: bucket doesn't exist") + ErrObjectNotExist = errors.New("storage: object doesn't exist") +) + +const ( + // ScopeFullControl grants permissions to manage your + // data and permissions in Google Cloud Storage. + ScopeFullControl = raw.DevstorageFullControlScope + + // ScopeReadOnly grants permissions to + // view your data in Google Cloud Storage. + ScopeReadOnly = raw.DevstorageReadOnlyScope + + // ScopeReadWrite grants permissions to manage your + // data in Google Cloud Storage. + ScopeReadWrite = raw.DevstorageReadWriteScope +) + +// TODO(jbd): Add storage.buckets.list. +// TODO(jbd): Add storage.buckets.insert. +// TODO(jbd): Add storage.buckets.update. +// TODO(jbd): Add storage.buckets.delete. + +// TODO(jbd): Add storage.objects.watch. + +// BucketInfo returns the metadata for the specified bucket. +func BucketInfo(ctx context.Context, name string) (*Bucket, error) { + resp, err := rawService(ctx).Buckets.Get(name).Projection("full").Context(ctx).Do() + if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { + return nil, ErrBucketNotExist + } + if err != nil { + return nil, err + } + return newBucket(resp), nil +} + +// ListObjects lists objects from the bucket. You can specify a query +// to filter the results. If q is nil, no filtering is applied. +func ListObjects(ctx context.Context, bucket string, q *Query) (*Objects, error) { + c := rawService(ctx).Objects.List(bucket) + c.Projection("full") + if q != nil { + c.Delimiter(q.Delimiter) + c.Prefix(q.Prefix) + c.Versions(q.Versions) + c.PageToken(q.Cursor) + if q.MaxResults > 0 { + c.MaxResults(int64(q.MaxResults)) + } + } + resp, err := c.Context(ctx).Do() + if err != nil { + return nil, err + } + objects := &Objects{ + Results: make([]*Object, len(resp.Items)), + Prefixes: make([]string, len(resp.Prefixes)), + } + for i, item := range resp.Items { + objects.Results[i] = newObject(item) + } + for i, prefix := range resp.Prefixes { + objects.Prefixes[i] = prefix + } + if resp.NextPageToken != "" { + next := Query{} + if q != nil { + // keep the other filtering + // criteria if there is a query + next = *q + } + next.Cursor = resp.NextPageToken + objects.Next = &next + } + return objects, nil +} + +// SignedURLOptions allows you to restrict the access to the signed URL. +type SignedURLOptions struct { + // GoogleAccessID represents the authorizer of the signed URL generation. + // It is typically the Google service account client email address from + // the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". + // Required. + GoogleAccessID string + + // PrivateKey is the Google service account private key. It is obtainable + // from the Google Developers Console. + // At https://console.developers.google.com/project//apiui/credential, + // create a service account client ID or reuse one of your existing service account + // credentials. Click on the "Generate new P12 key" to generate and download + // a new private key. Once you download the P12 file, use the following command + // to convert it into a PEM file. + // + // $ openssl pkcs12 -in key.p12 -passin pass:notasecret -out key.pem -nodes + // + // Provide the contents of the PEM file as a byte slice. + // Required. + PrivateKey []byte + + // Method is the HTTP method to be used with the signed URL. + // Signed URLs can be used with GET, HEAD, PUT, and DELETE requests. + // Required. + Method string + + // Expires is the expiration time on the signed URL. It must be + // a datetime in the future. + // Required. + Expires time.Time + + // ContentType is the content type header the client must provide + // to use the generated signed URL. + // Optional. + ContentType string + + // Headers is a list of extention headers the client must provide + // in order to use the generated signed URL. + // Optional. + Headers []string + + // MD5 is the base64 encoded MD5 checksum of the file. + // If provided, the client should provide the exact value on the request + // header in order to use the signed URL. + // Optional. + MD5 []byte +} + +// SignedURL returns a URL for the specified object. Signed URLs allow +// the users access to a restricted resource for a limited time without having a +// Google account or signing in. For more information about the signed +// URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs. +func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { + if opts == nil { + return "", errors.New("storage: missing required SignedURLOptions") + } + if opts.GoogleAccessID == "" || opts.PrivateKey == nil { + return "", errors.New("storage: missing required credentials to generate a signed URL") + } + if opts.Method == "" { + return "", errors.New("storage: missing required method option") + } + if opts.Expires.IsZero() { + return "", errors.New("storage: missing required expires option") + } + key, err := parseKey(opts.PrivateKey) + if err != nil { + return "", err + } + h := sha256.New() + fmt.Fprintf(h, "%s\n", opts.Method) + fmt.Fprintf(h, "%s\n", opts.MD5) + fmt.Fprintf(h, "%s\n", opts.ContentType) + fmt.Fprintf(h, "%d\n", opts.Expires.Unix()) + fmt.Fprintf(h, "%s", strings.Join(opts.Headers, "\n")) + fmt.Fprintf(h, "/%s/%s", bucket, name) + b, err := rsa.SignPKCS1v15( + rand.Reader, + key, + crypto.SHA256, + h.Sum(nil), + ) + if err != nil { + return "", err + } + encoded := base64.StdEncoding.EncodeToString(b) + u := &url.URL{ + Scheme: "https", + Host: "storage.googleapis.com", + Path: fmt.Sprintf("/%s/%s", bucket, name), + } + q := u.Query() + q.Set("GoogleAccessId", opts.GoogleAccessID) + q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix())) + q.Set("Signature", string(encoded)) + u.RawQuery = q.Encode() + return u.String(), nil +} + +// StatObject returns meta information about the specified object. +func StatObject(ctx context.Context, bucket, name string) (*Object, error) { + o, err := rawService(ctx).Objects.Get(bucket, name).Projection("full").Context(ctx).Do() + if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { + return nil, ErrObjectNotExist + } + if err != nil { + return nil, err + } + return newObject(o), nil +} + +// UpdateAttrs updates an object with the provided attributes. +// All zero-value attributes are ignored. +func UpdateAttrs(ctx context.Context, bucket, name string, attrs ObjectAttrs) (*Object, error) { + o, err := rawService(ctx).Objects.Patch(bucket, name, attrs.toRawObject(bucket)).Projection("full").Context(ctx).Do() + if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { + return nil, ErrObjectNotExist + } + if err != nil { + return nil, err + } + return newObject(o), nil +} + +// DeleteObject deletes the single specified object. +func DeleteObject(ctx context.Context, bucket, name string) error { + return rawService(ctx).Objects.Delete(bucket, name).Context(ctx).Do() +} + +// CopyObject copies the source object to the destination. +// The copied object's attributes are overwritten by attrs if non-nil. +func CopyObject(ctx context.Context, srcBucket, srcName string, destBucket, destName string, attrs *ObjectAttrs) (*Object, error) { + if srcBucket == "" || destBucket == "" { + return nil, errors.New("storage: srcBucket and destBucket must both be non-empty") + } + if srcName == "" || destName == "" { + return nil, errors.New("storage: srcName and destName must be non-empty") + } + var rawObject *raw.Object + if attrs != nil { + attrs.Name = destName + if attrs.ContentType == "" { + return nil, errors.New("storage: attrs.ContentType must be non-empty") + } + rawObject = attrs.toRawObject(destBucket) + } + o, err := rawService(ctx).Objects.Copy( + srcBucket, srcName, destBucket, destName, rawObject).Projection("full").Context(ctx).Do() + if err != nil { + return nil, err + } + return newObject(o), nil +} + +// NewReader creates a new io.ReadCloser to read the contents +// of the object. +func NewReader(ctx context.Context, bucket, name string) (io.ReadCloser, error) { + hc := internal.HTTPClient(ctx) + u := &url.URL{ + Scheme: "https", + Host: "storage.googleapis.com", + Path: fmt.Sprintf("/%s/%s", bucket, name), + } + res, err := hc.Get(u.String()) + if err != nil { + return nil, err + } + if res.StatusCode == http.StatusNotFound { + res.Body.Close() + return nil, ErrObjectNotExist + } + if res.StatusCode < 200 || res.StatusCode > 299 { + res.Body.Close() + return res.Body, fmt.Errorf("storage: can't read object %v/%v, status code: %v", bucket, name, res.Status) + } + return res.Body, nil +} + +// NewWriter returns a storage Writer that writes to the GCS object +// identified by the specified name. +// If such an object doesn't exist, it creates one. +// Attributes can be set on the object by modifying the returned Writer's +// ObjectAttrs field before the first call to Write. The name parameter to this +// function is ignored if the Name field of the ObjectAttrs field is set to a +// non-empty string. +// +// It is the caller's responsibility to call Close when writing is done. +// +// The object is not available and any previous object with the same +// name is not replaced on Cloud Storage until Close is called. +func NewWriter(ctx context.Context, bucket, name string) *Writer { + return &Writer{ + ctx: ctx, + bucket: bucket, + name: name, + donec: make(chan struct{}), + } +} + +func rawService(ctx context.Context) *raw.Service { + return internal.Service(ctx, "storage", func(hc *http.Client) interface{} { + svc, _ := raw.New(hc) + return svc + }).(*raw.Service) +} + +// parseKey converts the binary contents of a private key file +// to an *rsa.PrivateKey. It detects whether the private key is in a +// PEM container or not. If so, it extracts the the private key +// from PEM container before conversion. It only supports PEM +// containers with no passphrase. +func parseKey(key []byte) (*rsa.PrivateKey, error) { + if block, _ := pem.Decode(key); block != nil { + key = block.Bytes + } + parsedKey, err := x509.ParsePKCS8PrivateKey(key) + if err != nil { + parsedKey, err = x509.ParsePKCS1PrivateKey(key) + if err != nil { + return nil, err + } + } + parsed, ok := parsedKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("oauth2: private key is invalid") + } + return parsed, nil +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/storage_test.go b/Godeps/_workspace/src/google.golang.org/cloud/storage/storage_test.go new file mode 100644 index 0000000000..4d239e5c72 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/storage_test.go @@ -0,0 +1,238 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "io/ioutil" + "log" + "strings" + "testing" + "time" + + "golang.org/x/net/context" +) + +func TestSignedURL(t *testing.T) { + expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00") + url, err := SignedURL("bucket-name", "object-name", &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "GET", + MD5: []byte("202cb962ac59075b964b07152d234b70"), + Expires: expires, + ContentType: "application/json", + Headers: []string{"x-header1", "x-header2"}, + }) + if err != nil { + t.Error(err) + } + want := "https://storage.googleapis.com/bucket-name/object-name?" + + "Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" + + "ITqNWQHr7ayIj%2B0Ds5%2FzUT2cWMQQouuFmu6L11Zd3kfNKvm3sjyGIzO" + + "gZsSUoter1SxP7BcrCzgqIZ9fQmgQnuIpqqLL4kcGmTbKsQS6hTknpJM%2F" + + "2lS4NY6UH1VXBgm2Tce28kz8rnmqG6svcGvtWuOgJsETeSIl1R9nAEIDCEq" + + "ZJzoOiru%2BODkHHkpoFjHWAwHugFHX%2B9EX4SxaytiN3oEy48HpYGWV0I" + + "h8NvU1hmeWzcLr41GnTADeCn7Eg%2Fb5H2GCNO70Cz%2Bw2fn%2BofLCUeR" + + "YQd%2FhES8oocv5kpHZkstc8s8uz3aKMsMauzZ9MOmGy%2F6VULBgIVvi6a" + + "AwEBIYOw%3D%3D" + if url != want { + t.Fatalf("Unexpected signed URL; found %v", url) + } +} + +func TestSignedURL_PEMPrivateKey(t *testing.T) { + expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00") + url, err := SignedURL("bucket-name", "object-name", &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("pem"), + Method: "GET", + MD5: []byte("202cb962ac59075b964b07152d234b70"), + Expires: expires, + ContentType: "application/json", + Headers: []string{"x-header1", "x-header2"}, + }) + if err != nil { + t.Error(err) + } + want := "https://storage.googleapis.com/bucket-name/object-name?" + + "Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" + + "B7XkS4dfmVDoe%2FoDeXZkWlYmg8u2kI0SizTrzL5%2B9RmKnb5j7Kf34DZ" + + "JL8Hcjr1MdPFLNg2QV4lEH86Gqgqt%2Fv3jFOTRl4wlzcRU%2FvV5c5HU8M" + + "qW0FZ0IDbqod2RdsMONLEO6yQWV2HWFrMLKl2yMFlWCJ47et%2BFaHe6v4Z" + + "EBc0%3D" + if url != want { + t.Fatalf("Unexpected signed URL; found %v", url) + } +} + +func TestSignedURL_MissingOptions(t *testing.T) { + pk := dummyKey("rsa") + var tests = []struct { + opts *SignedURLOptions + errMsg string + }{ + { + &SignedURLOptions{}, + "missing required credentials", + }, + { + &SignedURLOptions{GoogleAccessID: "access_id"}, + "missing required credentials", + }, + { + &SignedURLOptions{ + GoogleAccessID: "access_id", + PrivateKey: pk, + }, + "missing required method", + }, + { + &SignedURLOptions{ + GoogleAccessID: "access_id", + PrivateKey: pk, + Method: "PUT", + }, + "missing required expires", + }, + } + for _, test := range tests { + _, err := SignedURL("bucket", "name", test.opts) + if !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("expected err: %v, found: %v", test.errMsg, err) + } + } +} + +func dummyKey(kind string) []byte { + slurp, err := ioutil.ReadFile(fmt.Sprintf("./testdata/dummy_%s", kind)) + if err != nil { + log.Fatal(err) + } + return slurp +} + +func TestCopyObjectMissingFields(t *testing.T) { + var tests = []struct { + srcBucket, srcName, destBucket, destName string + errMsg string + }{ + { + "mybucket", "", "mybucket", "destname", + "srcName and destName must be non-empty", + }, + { + "mybucket", "srcname", "mybucket", "", + "srcName and destName must be non-empty", + }, + { + "", "srcfile", "mybucket", "destname", + "srcBucket and destBucket must both be non-empty", + }, + { + "mybucket", "srcfile", "", "destname", + "srcBucket and destBucket must both be non-empty", + }, + } + for i, test := range tests { + _, err := CopyObject(context.TODO(), test.srcBucket, test.srcName, test.destBucket, test.destName, nil) + if !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("CopyObject test #%v: err = %v, want %v", i, err, test.errMsg) + } + } +} + +func TestObjectNames(t *testing.T) { + // Naming requirements: https://cloud.google.com/storage/docs/bucket-naming + const maxLegalLength = 1024 + + type testT struct { + name, want string + } + tests := []testT{ + // Embedded characters important in URLs. + {"foo % bar", "foo%20%25%20bar"}, + {"foo ? bar", "foo%20%3F%20bar"}, + {"foo / bar", "foo%20/%20bar"}, + {"foo %?/ bar", "foo%20%25%3F/%20bar"}, + + // Non-Roman scripts + {"타코", "%ED%83%80%EC%BD%94"}, + {"δΈ–η•Œ", "%E4%B8%96%E7%95%8C"}, + + // Longest legal name + {strings.Repeat("a", maxLegalLength), strings.Repeat("a", maxLegalLength)}, + + // Line terminators besides CR and LF: https://en.wikipedia.org/wiki/Newline#Unicode + {"foo \u000b bar", "foo%20%0B%20bar"}, + {"foo \u000c bar", "foo%20%0C%20bar"}, + {"foo \u0085 bar", "foo%20%C2%85%20bar"}, + {"foo \u2028 bar", "foo%20%E2%80%A8%20bar"}, + {"foo \u2029 bar", "foo%20%E2%80%A9%20bar"}, + + // Null byte. + {"foo \u0000 bar", "foo%20%00%20bar"}, + + // Non-control characters that are discouraged, but not forbidden, according to the documentation. + {"foo # bar", "foo%20%23%20bar"}, + {"foo []*? bar", "foo%20%5B%5D%2A%3F%20bar"}, + + // Angstrom symbol singleton and normalized forms: http://unicode.org/reports/tr15/ + {"foo \u212b bar", "foo%20%E2%84%AB%20bar"}, + {"foo \u0041\u030a bar", "foo%20A%CC%8A%20bar"}, + {"foo \u00c5 bar", "foo%20%C3%85%20bar"}, + + // Hangul separating jamo: http://www.unicode.org/versions/Unicode7.0.0/ch18.pdf (Table 18-10) + {"foo \u3131\u314f bar", "foo%20%E3%84%B1%E3%85%8F%20bar"}, + {"foo \u1100\u1161 bar", "foo%20%E1%84%80%E1%85%A1%20bar"}, + {"foo \uac00 bar", "foo%20%EA%B0%80%20bar"}, + } + + // C0 control characters not forbidden by the docs. + var runes []rune + for r := rune(0x01); r <= rune(0x1f); r++ { + if r != '\u000a' && r != '\u000d' { + runes = append(runes, r) + } + } + tests = append(tests, testT{fmt.Sprintf("foo %s bar", string(runes)), "foo%20%01%02%03%04%05%06%07%08%09%0B%0C%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20bar"}) + + // C1 control characters, plus DEL. + runes = nil + for r := rune(0x7f); r <= rune(0x9f); r++ { + runes = append(runes, r) + } + tests = append(tests, testT{fmt.Sprintf("foo %s bar", string(runes)), "foo%20%7F%C2%80%C2%81%C2%82%C2%83%C2%84%C2%85%C2%86%C2%87%C2%88%C2%89%C2%8A%C2%8B%C2%8C%C2%8D%C2%8E%C2%8F%C2%90%C2%91%C2%92%C2%93%C2%94%C2%95%C2%96%C2%97%C2%98%C2%99%C2%9A%C2%9B%C2%9C%C2%9D%C2%9E%C2%9F%20bar"}) + + opts := &SignedURLOptions{ + GoogleAccessID: "xxx@clientid", + PrivateKey: dummyKey("rsa"), + Method: "GET", + MD5: []byte("202cb962ac59075b964b07152d234b70"), + Expires: time.Date(2002, time.October, 2, 10, 0, 0, 0, time.UTC), + ContentType: "application/json", + Headers: []string{"x-header1", "x-header2"}, + } + + for _, test := range tests { + g, err := SignedURL("bucket-name", test.name, opts) + if err != nil { + t.Errorf("SignedURL(%q) err=%v, want nil", test.name, err) + } + if w := "/bucket-name/" + test.want; !strings.Contains(g, w) { + t.Errorf("SignedURL(%q)=%q, want substring %q", test.name, g, w) + } + } +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/testdata/dummy_pem b/Godeps/_workspace/src/google.golang.org/cloud/storage/testdata/dummy_pem new file mode 100644 index 0000000000..3428d4497c --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/testdata/dummy_pem @@ -0,0 +1,39 @@ +Bag Attributes + friendlyName: privatekey + localKeyID: 54 69 6D 65 20 31 34 31 36 38 35 32 30 30 34 37 37 32 +Key Attributes: +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQCtCWMoJ2Bok2QoGFyU7A6IlGprO9QfUTT0jNrLkIbM5OWNIuDx +64+PEaTS5g5m+2Hz/lmd5jJKanAH4dY9LZzsaYAPq1K17Gcmg1hEisYeKsgOcjYY +kwRkV+natCTsC+tfWmS0voRh0jA1rI1J4MikceoHtgWdEuoHrrptRVpWKwIDAQAB +AoGAKp3uQvx3vSnX+BwP6Um+RpsvHpwMoW3xue1bEdnVqW8SrlERz+NxZw40ZxDs +KSbuuBZD4iTI7BUM5JQVnNm4FQY1YrPlWZLyI73Bj8RKTXrPdJheM/0r7xjiIXbQ +7w4cUSM9rVugnI/rxF2kPIQTGYI+EG/6+P+k6VvgPmC0T/ECQQDUPskiS18WaY+i +Koalbrb3GakaBoHrC1b4ln4CAv7fq7H4WvFvqi/2rxLhHYq31iwxYy8s7J7Sba1+ +5vwJ2TxZAkEA0LVfs3Q2VWZ+cM3bv0aYTalMXg6wT+LoNvk9HnOb0zQYajF3qm4G +ZFdfEqvOkje0zQ4fcihARKyda/VY84UGIwJBAIZa0FvjNmgrnn7bSKzEbxHwrnkJ +EYjGfuGR8mY3mzvfpiM+/oLfSslvfhX+62cALq18yco4ZzlxsFgaxAU//NECQDcS +NN94YcHlGqYPW9W7/gI4EwOaoqFhwV6II71+SfbP/0U+KlJZV+xwNZEKrqZcdqPI +/zkzL8ovNha/laokRrsCQQCyoPHGcBWj+VFbNoyQnX4tghc6rOY7n4pmpgQvU825 +TAM9vnYtSkKK/V56kEDNBO5LwiRsir95IUNclqqMKR1C +-----END RSA PRIVATE KEY----- +Bag Attributes + friendlyName: privatekey + localKeyID: 54 69 6D 65 20 31 34 31 36 38 35 32 30 30 34 37 37 32 +subject=/CN=1079432350659-nvog0vmn9s6pqr3kr4v2avbc7nkhoa11.apps.googleusercontent.com +issuer=/CN=1079432350659-nvog0vmn9s6pqr3kr4v2avbc7nkhoa11.apps.googleusercontent.com +-----BEGIN CERTIFICATE----- +MIICXTCCAcagAwIBAgIIHxTMQUVJRZ0wDQYJKoZIhvcNAQEFBQAwVDFSMFAGA1UE +AxNJMTA3OTQzMjM1MDY1OS1udm9nMHZtbjlzNnBxcjNrcjR2MmF2YmM3bmtob2Ex +MS5hcHBzLmdvb2dsZXVzZXJjb250ZW50LmNvbTAeFw0xNDExMjQxODAwMDRaFw0y +NDExMjExODAwMDRaMFQxUjBQBgNVBAMTSTEwNzk0MzIzNTA2NTktbnZvZzB2bW45 +czZwcXIza3I0djJhdmJjN25raG9hMTEuYXBwcy5nb29nbGV1c2VyY29udGVudC5j +b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK0JYygnYGiTZCgYXJTsDoiU +ams71B9RNPSM2suQhszk5Y0i4PHrj48RpNLmDmb7YfP+WZ3mMkpqcAfh1j0tnOxp +gA+rUrXsZyaDWESKxh4qyA5yNhiTBGRX6dq0JOwL619aZLS+hGHSMDWsjUngyKRx +6ge2BZ0S6geuum1FWlYrAgMBAAGjODA2MAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/ +BAQDAgeAMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMA0GCSqGSIb3DQEBBQUAA4GB +ACVvKkZkomHq3uffOQwdZ4VJYuxrvDGnZu/ExW9WngO2teEsjxABL41TNnRYHN5T +lMC19poFA2tR/DySDLJ2XNs/hSvyQUL6HHCncVdR4Srpie88j48peY1MZSMP51Jv +qagbbP5K5DSEu02/zZaV0kaCvLEN0KAtj/noDuOOnQU2 +-----END CERTIFICATE----- diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/testdata/dummy_rsa b/Godeps/_workspace/src/google.golang.org/cloud/storage/testdata/dummy_rsa new file mode 100644 index 0000000000..4ce6678dbd --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/testdata/dummy_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAx4fm7dngEmOULNmAs1IGZ9Apfzh+BkaQ1dzkmbUgpcoghucE +DZRnAGd2aPyB6skGMXUytWQvNYav0WTR00wFtX1ohWTfv68HGXJ8QXCpyoSKSSFY +fuP9X36wBSkSX9J5DVgiuzD5VBdzUISSmapjKm+DcbRALjz6OUIPEWi1Tjl6p5RK +1w41qdbmt7E5/kGhKLDuT7+M83g4VWhgIvaAXtnhklDAggilPPa8ZJ1IFe31lNlr +k4DRk38nc6sEutdf3RL7QoH7FBusI7uXV03DC6dwN1kP4GE7bjJhcRb/7jYt7CQ9 +/E9Exz3c0yAp0yrTg0Fwh+qxfH9dKwN52S7SBwIDAQABAoIBAQCaCs26K07WY5Jt +3a2Cw3y2gPrIgTCqX6hJs7O5ByEhXZ8nBwsWANBUe4vrGaajQHdLj5OKfsIDrOvn +2NI1MqflqeAbu/kR32q3tq8/Rl+PPiwUsW3E6Pcf1orGMSNCXxeducF2iySySzh3 +nSIhCG5uwJDWI7a4+9KiieFgK1pt/Iv30q1SQS8IEntTfXYwANQrfKUVMmVF9aIK +6/WZE2yd5+q3wVVIJ6jsmTzoDCX6QQkkJICIYwCkglmVy5AeTckOVwcXL0jqw5Kf +5/soZJQwLEyBoQq7Kbpa26QHq+CJONetPP8Ssy8MJJXBT+u/bSseMb3Zsr5cr43e +DJOhwsThAoGBAPY6rPKl2NT/K7XfRCGm1sbWjUQyDShscwuWJ5+kD0yudnT/ZEJ1 +M3+KS/iOOAoHDdEDi9crRvMl0UfNa8MAcDKHflzxg2jg/QI+fTBjPP5GOX0lkZ9g +z6VePoVoQw2gpPFVNPPTxKfk27tEzbaffvOLGBEih0Kb7HTINkW8rIlzAoGBAM9y +1yr+jvfS1cGFtNU+Gotoihw2eMKtIqR03Yn3n0PK1nVCDKqwdUqCypz4+ml6cxRK +J8+Pfdh7D+ZJd4LEG6Y4QRDLuv5OA700tUoSHxMSNn3q9As4+T3MUyYxWKvTeu3U +f2NWP9ePU0lV8ttk7YlpVRaPQmc1qwooBA/z/8AdAoGAW9x0HWqmRICWTBnpjyxx +QGlW9rQ9mHEtUotIaRSJ6K/F3cxSGUEkX1a3FRnp6kPLcckC6NlqdNgNBd6rb2rA +cPl/uSkZP42Als+9YMoFPU/xrrDPbUhu72EDrj3Bllnyb168jKLa4VBOccUvggxr +Dm08I1hgYgdN5huzs7y6GeUCgYEAj+AZJSOJ6o1aXS6rfV3mMRve9bQ9yt8jcKXw +5HhOCEmMtaSKfnOF1Ziih34Sxsb7O2428DiX0mV/YHtBnPsAJidL0SdLWIapBzeg +KHArByIRkwE6IvJvwpGMdaex1PIGhx5i/3VZL9qiq/ElT05PhIb+UXgoWMabCp84 +OgxDK20CgYAeaFo8BdQ7FmVX2+EEejF+8xSge6WVLtkaon8bqcn6P0O8lLypoOhd +mJAYH8WU+UAy9pecUnDZj14LAGNVmYcse8HFX71MoshnvCTFEPVo4rZxIAGwMpeJ +5jgQ3slYLpqrGlcbLgUXBUgzEO684Wk/UV9DFPlHALVqCfXQ9dpJPg== +-----END RSA PRIVATE KEY----- diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/types.go b/Godeps/_workspace/src/google.golang.org/cloud/storage/types.go new file mode 100644 index 0000000000..060deb6ad7 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/types.go @@ -0,0 +1,417 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "encoding/base64" + "io" + "sync" + "time" + + "golang.org/x/net/context" + raw "google.golang.org/api/storage/v1" +) + +// Bucket represents a Google Cloud Storage bucket. +type Bucket struct { + // Name is the name of the bucket. + Name string + + // ACL is the list of access control rules on the bucket. + ACL []ACLRule + + // DefaultObjectACL is the list of access controls to + // apply to new objects when no object ACL is provided. + DefaultObjectACL []ACLRule + + // Location is the location of the bucket. It defaults to "US". + Location string + + // Metageneration is the metadata generation of the bucket. + // Read-only. + Metageneration int64 + + // StorageClass is the storage class of the bucket. This defines + // how objects in the bucket are stored and determines the SLA + // and the cost of storage. Typical values are "STANDARD" and + // "DURABLE_REDUCED_AVAILABILITY". Defaults to "STANDARD". + StorageClass string + + // Created is the creation time of the bucket. + // Read-only. + Created time.Time +} + +func newBucket(b *raw.Bucket) *Bucket { + if b == nil { + return nil + } + bucket := &Bucket{ + Name: b.Name, + Location: b.Location, + Metageneration: b.Metageneration, + StorageClass: b.StorageClass, + Created: convertTime(b.TimeCreated), + } + acl := make([]ACLRule, len(b.Acl)) + for i, rule := range b.Acl { + acl[i] = ACLRule{ + Entity: ACLEntity(rule.Entity), + Role: ACLRole(rule.Role), + } + } + bucket.ACL = acl + objACL := make([]ACLRule, len(b.DefaultObjectAcl)) + for i, rule := range b.DefaultObjectAcl { + objACL[i] = ACLRule{ + Entity: ACLEntity(rule.Entity), + Role: ACLRole(rule.Role), + } + } + bucket.DefaultObjectACL = objACL + return bucket +} + +// ObjectAttrs is the user-editable object attributes. +type ObjectAttrs struct { + // Name is the name of the object. + Name string + + // ContentType is the MIME type of the object's content. + // Optional. + ContentType string + + // ContentLanguage is the optional RFC 1766 Content-Language of + // the object's content sent in response headers. + ContentLanguage string + + // ContentEncoding is the optional Content-Encoding of the object + // sent it the response headers. + ContentEncoding string + + // CacheControl is the optional Cache-Control header of the object + // sent in the response headers. + CacheControl string + + // ContentDisposition is the optional Content-Disposition header of the object + // sent in the response headers. + ContentDisposition string + + // ACL is the list of access control rules for the object. + // Optional. If nil or empty, existing ACL rules are preserved. + ACL []ACLRule + + // Metadata represents user-provided metadata, in key/value pairs. + // It can be nil if the current metadata values needs to preserved. + Metadata map[string]string +} + +func (o ObjectAttrs) toRawObject(bucket string) *raw.Object { + var acl []*raw.ObjectAccessControl + if len(o.ACL) > 0 { + acl = make([]*raw.ObjectAccessControl, len(o.ACL)) + for i, rule := range o.ACL { + acl[i] = &raw.ObjectAccessControl{ + Entity: string(rule.Entity), + Role: string(rule.Role), + } + } + } + return &raw.Object{ + Bucket: bucket, + Name: o.Name, + ContentType: o.ContentType, + ContentEncoding: o.ContentEncoding, + ContentLanguage: o.ContentLanguage, + CacheControl: o.CacheControl, + ContentDisposition: o.ContentDisposition, + Acl: acl, + Metadata: o.Metadata, + } +} + +// Object represents a Google Cloud Storage (GCS) object. +type Object struct { + // Bucket is the name of the bucket containing this GCS object. + Bucket string + + // Name is the name of the object within the bucket. + Name string + + // ContentType is the MIME type of the object's content. + ContentType string + + // ContentLanguage is the content language of the object's content. + ContentLanguage string + + // CacheControl is the Cache-Control header to be sent in the response + // headers when serving the object data. + CacheControl string + + // ACL is the list of access control rules for the object. + ACL []ACLRule + + // Owner is the owner of the object. + // + // If non-zero, it is in the form of "user-". + Owner string + + // Size is the length of the object's content. + Size int64 + + // ContentEncoding is the encoding of the object's content. + ContentEncoding string + + // MD5 is the MD5 hash of the object's content. + MD5 []byte + + // CRC32C is the CRC32 checksum of the object's content using + // the Castagnoli93 polynomial. + CRC32C uint32 + + // MediaLink is an URL to the object's content. + MediaLink string + + // Metadata represents user-provided metadata, in key/value pairs. + // It can be nil if no metadata is provided. + Metadata map[string]string + + // Generation is the generation number of the object's content. + Generation int64 + + // MetaGeneration is the version of the metadata for this + // object at this generation. This field is used for preconditions + // and for detecting changes in metadata. A metageneration number + // is only meaningful in the context of a particular generation + // of a particular object. + MetaGeneration int64 + + // StorageClass is the storage class of the bucket. + // This value defines how objects in the bucket are stored and + // determines the SLA and the cost of storage. Typical values are + // "STANDARD" and "DURABLE_REDUCED_AVAILABILITY". + // It defaults to "STANDARD". + StorageClass string + + // Deleted is the time the object was deleted. + // If not deleted, it is the zero value. + Deleted time.Time + + // Updated is the creation or modification time of the object. + // For buckets with versioning enabled, changing an object's + // metadata does not change this property. + Updated time.Time +} + +// convertTime converts a time in RFC3339 format to time.Time. +// If any error occurs in parsing, the zero-value time.Time is silently returned. +func convertTime(t string) time.Time { + var r time.Time + if t != "" { + r, _ = time.Parse(time.RFC3339, t) + } + return r +} + +func newObject(o *raw.Object) *Object { + if o == nil { + return nil + } + acl := make([]ACLRule, len(o.Acl)) + for i, rule := range o.Acl { + acl[i] = ACLRule{ + Entity: ACLEntity(rule.Entity), + Role: ACLRole(rule.Role), + } + } + owner := "" + if o.Owner != nil { + owner = o.Owner.Entity + } + md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) + var crc32c uint32 + d, err := base64.StdEncoding.DecodeString(o.Crc32c) + if err == nil && len(d) == 4 { + crc32c = uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]) + } + return &Object{ + Bucket: o.Bucket, + Name: o.Name, + ContentType: o.ContentType, + ContentLanguage: o.ContentLanguage, + CacheControl: o.CacheControl, + ACL: acl, + Owner: owner, + ContentEncoding: o.ContentEncoding, + Size: int64(o.Size), + MD5: md5, + CRC32C: crc32c, + MediaLink: o.MediaLink, + Metadata: o.Metadata, + Generation: o.Generation, + MetaGeneration: o.Metageneration, + StorageClass: o.StorageClass, + Deleted: convertTime(o.TimeDeleted), + Updated: convertTime(o.Updated), + } +} + +// Query represents a query to filter objects from a bucket. +type Query struct { + // Delimiter returns results in a directory-like fashion. + // Results will contain only objects whose names, aside from the + // prefix, do not contain delimiter. Objects whose names, + // aside from the prefix, contain delimiter will have their name, + // truncated after the delimiter, returned in prefixes. + // Duplicate prefixes are omitted. + // Optional. + Delimiter string + + // Prefix is the prefix filter to query objects + // whose names begin with this prefix. + // Optional. + Prefix string + + // Versions indicates whether multiple versions of the same + // object will be included in the results. + Versions bool + + // Cursor is a previously-returned page token + // representing part of the larger set of results to view. + // Optional. + Cursor string + + // MaxResults is the maximum number of items plus prefixes + // to return. As duplicate prefixes are omitted, + // fewer total results may be returned than requested. + // The default page limit is used if it is negative or zero. + MaxResults int +} + +// Objects represents a list of objects returned from +// a bucket look-p request and a query to retrieve more +// objects from the next pages. +type Objects struct { + // Results represent a list of object results. + Results []*Object + + // Next is the continuation query to retrieve more + // results with the same filtering criteria. If there + // are no more results to retrieve, it is nil. + Next *Query + + // Prefixes represents prefixes of objects + // matching-but-not-listed up to and including + // the requested delimiter. + Prefixes []string +} + +// contentTyper implements ContentTyper to enable an +// io.ReadCloser to specify its MIME type. +type contentTyper struct { + io.Reader + t string +} + +func (c *contentTyper) ContentType() string { + return c.t +} + +// A Writer writes a Cloud Storage object. +type Writer struct { + // ObjectAttrs are optional attributes to set on the object. Any attributes + // must be initialized before the first Write call. Nil or zero-valued + // attributes are ignored. + ObjectAttrs + + ctx context.Context + bucket string + name string + + once sync.Once + + opened bool + r io.Reader + pw *io.PipeWriter + + donec chan struct{} // closed after err and obj are set. + err error + obj *Object +} + +func (w *Writer) open() { + attrs := w.ObjectAttrs + // Always set the name, otherwise the backend + // rejects the request and responds with an HTTP 400. + if attrs.Name == "" { + attrs.Name = w.name + } + pr, pw := io.Pipe() + w.r = &contentTyper{pr, attrs.ContentType} + w.pw = pw + w.opened = true + + go func() { + resp, err := rawService(w.ctx).Objects.Insert( + w.bucket, attrs.toRawObject(w.bucket)).Media(w.r).Projection("full").Context(w.ctx).Do() + w.err = err + if err == nil { + w.obj = newObject(resp) + } else { + pr.CloseWithError(w.err) + } + close(w.donec) + }() +} + +// Write appends to w. +func (w *Writer) Write(p []byte) (n int, err error) { + if w.err != nil { + return 0, w.err + } + if !w.opened { + w.open() + } + return w.pw.Write(p) +} + +// Close completes the write operation and flushes any buffered data. +// If Close doesn't return an error, metadata about the written object +// can be retrieved by calling Object. +func (w *Writer) Close() error { + if !w.opened { + w.open() + } + if err := w.pw.Close(); err != nil { + return err + } + <-w.donec + return w.err +} + +// CloseWithError aborts the write operation with the provided error. +// CloseWithError always returns nil. +func (w *Writer) CloseWithError(err error) error { + if !w.opened { + return nil + } + return w.pw.CloseWithError(err) +} + +// Object returns metadata about a successfully-written object. +// It's only valid to call it after Close returns nil. +func (w *Writer) Object() *Object { + return w.obj +} diff --git a/Godeps/_workspace/src/google.golang.org/cloud/storage/types_test.go b/Godeps/_workspace/src/google.golang.org/cloud/storage/types_test.go new file mode 100644 index 0000000000..02b71314c7 --- /dev/null +++ b/Godeps/_workspace/src/google.golang.org/cloud/storage/types_test.go @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/http" + "testing" + + "google.golang.org/cloud" +) + +type fakeTransport struct{} + +func (t *fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("error handling request") +} + +func TestErrorOnObjectsInsertCall(t *testing.T) { + ctx := cloud.NewContext("project-id", &http.Client{ + Transport: &fakeTransport{}}) + wc := NewWriter(ctx, "bucketname", "filename1") + wc.ContentType = "text/plain" + if _, err := wc.Write([]byte("hello world")); err == nil { + t.Errorf("expected error on write, got nil") + } + if err := wc.Close(); err == nil { + t.Errorf("expected error on close, got nil") + } +} diff --git a/docs/drivers/gce.md b/docs/drivers/gce.md index ab973bc512..1215f01aec 100644 --- a/docs/drivers/gce.md +++ b/docs/drivers/gce.md @@ -10,9 +10,12 @@ parent="smn_machine_drivers" # Google Compute Engine -Create machines on [Google Compute Engine](https://cloud.google.com/compute/). You will need a Google account and project name. See https://cloud.google.com/compute/docs/projects for details on projects. +Create machines on [Google Compute Engine](https://cloud.google.com/compute/). You will need a Google account and project name. +See https://cloud.google.com/compute/docs/projects for details on projects. -The Google driver uses oAuth. When creating the machine, you will have your browser opened to authorize. Once authorized, paste the code given in the prompt to launch the instance. +The Google driver uses [Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials). +So if `docker-machine` is started from a GCE host, authentication will happen automatically. +Otherwise, you have to run `gcloud auth login` once before using `docker-machine`. Options: @@ -20,7 +23,6 @@ Options: - `--google-machine-type`: The type of instance. - `--google-username`: The username to use for the instance. - `--google-project`: **required** The name of your project to use when launching the instance. - - `--google-auth-token`: Your oAuth token for the Google Cloud API. - `--google-scopes`: The scopes for OAuth 2.0 to Access Google APIs. See [Google Compute Engine Doc](https://cloud.google.com/storage/docs/authentication). - `--google-disk-size`: The disk size of instance. - `--google-disk-type`: The disk type of instance. @@ -28,17 +30,16 @@ Options: - `--google-preemptible`: Instance preemptibility. - `--google-tags`: Instance tags (comma-separated). -The GCE driver will use the `ubuntu-1404-trusty-v20150316` instance type unless otherwise specified. +The GCE driver will use the `ubuntu-1404-trusty-v20150909a` instance type unless otherwise specified. Environment variables and default values: | CLI option | Environment variable | Default | |---------------------------|-----------------------|--------------------------------------| | `--google-zone` | `GOOGLE_ZONE` | `us-central1-a` | -| `--google-machine-type` | `GOOGLE_MACHINE_TYPE` | `f1-micro` | +| `--google-machine-type` | `GOOGLE_MACHINE_TYPE` | `n1-standard-1` | | `--google-username` | `GOOGLE_USERNAME` | `docker-user` | | **`--google-project`** | `GOOGLE_PROJECT` | - | -| `--google-auth-token` | `GOOGLE_AUTH_TOKEN` | - | | `--google-scopes` | `GOOGLE_SCOPES` | `devstorage.read_only,logging.write` | | `--google-disk-size` | `GOOGLE_DISK_SIZE` | `10` | | `--google-disk-type` | `GOOGLE_DISK_TYPE` | `pd-standard` | diff --git a/drivers/google/auth_util.go b/drivers/google/auth_util.go deleted file mode 100644 index 12c0dedecf..0000000000 --- a/drivers/google/auth_util.go +++ /dev/null @@ -1,127 +0,0 @@ -package google - -import ( - "encoding/gob" - "fmt" - "net/http" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - "time" - - "code.google.com/p/goauth2/oauth" - "github.com/docker/machine/libmachine/log" - raw "google.golang.org/api/compute/v1" -) - -const ( - AuthURL = "https://accounts.google.com/o/oauth2/auth" - TokenURL = "https://accounts.google.com/o/oauth2/token" - ClientId = "22738965389-8arp8bah3uln9eoenproamovfjj1ac33.apps.googleusercontent.com" - ClientSecret = "qApc3amTyr5wI74vVrRWAfC_" - RedirectURI = "urn:ietf:wg:oauth:2.0:oob" -) - -func newGCEService(storePath, authTokenPath string) (*raw.Service, error) { - client := newOauthClient(storePath, authTokenPath) - service, err := raw.New(client) - return service, err -} - -func newOauthClient(storePath, authTokenPath string) *http.Client { - config := &oauth.Config{ - ClientId: ClientId, - ClientSecret: ClientSecret, - Scope: raw.ComputeScope, - AuthURL: AuthURL, - TokenURL: TokenURL, - } - - token := token(storePath, authTokenPath, config) - t := oauth.Transport{ - Token: token, - Config: config, - Transport: http.DefaultTransport, - } - return t.Client() -} - -func token(storePath, authTokenPath string, config *oauth.Config) *oauth.Token { - tokenPath := authTokenPath - if authTokenPath == "" { - tokenPath = filepath.Join(storePath, "gce_token") - } - log.Debugf("using auth token: %s", tokenPath) - token, err := tokenFromCache(tokenPath) - if err != nil { - token = tokenFromWeb(config) - saveToken(storePath, token) - } - return token -} - -func tokenFromCache(tokenPath string) (*oauth.Token, error) { - f, err := os.Open(tokenPath) - if err != nil { - return nil, err - } - token := new(oauth.Token) - err = gob.NewDecoder(f).Decode(token) - return token, err -} - -func tokenFromWeb(config *oauth.Config) *oauth.Token { - randState := fmt.Sprintf("st%d", time.Now().UnixNano()) - - config.RedirectURL = RedirectURI - authURL := config.AuthCodeURL(randState) - - log.Info("Opening auth URL in browser.") - log.Info(authURL) - log.Info("If the URL doesn't open please open it manually and copy the code here.") - openURL(authURL) - code := getCodeFromStdin() - - log.Infof("Got code: %s", code) - - t := &oauth.Transport{ - Config: config, - Transport: http.DefaultTransport, - } - _, err := t.Exchange(code) - if err != nil { - log.Warnf("Token exchange error: %v", err) - } - return t.Token -} - -func getCodeFromStdin() string { - fmt.Print("Enter code: ") - var code string - fmt.Scanln(&code) - return strings.Trim(code, "\n") -} - -func openURL(url string) { - try := []string{"xdg-open", "google-chrome", "open"} - for _, bin := range try { - err := exec.Command(bin, url).Run() - if err == nil { - return - } - } -} - -func saveToken(storePath string, token *oauth.Token) { - tokenPath := path.Join(storePath, "gce_token") - log.Infof("Saving token in %v", tokenPath) - f, err := os.Create(tokenPath) - if err != nil { - log.Infof("Warning: failed to cache oauth token: %v", err) - return - } - defer f.Close() - gob.NewEncoder(f).Encode(token) -} diff --git a/drivers/google/compute_util.go b/drivers/google/compute_util.go index 6079dfdf32..39bbb6164a 100644 --- a/drivers/google/compute_util.go +++ b/drivers/google/compute_util.go @@ -11,29 +11,31 @@ import ( "github.com/docker/machine/libmachine/log" "github.com/docker/machine/libmachine/ssh" raw "google.golang.org/api/compute/v1" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" ) // ComputeUtil is used to wrap the raw GCE API code and store common parameters. type ComputeUtil struct { - zone string - instanceName string - userName string - project string - diskTypeURL string - address string - preemptible bool - service *raw.Service - zoneURL string - authTokenPath string - globalURL string - ipAddress string - SwarmMaster bool - SwarmHost string + zone string + instanceName string + userName string + project string + diskTypeURL string + address string + preemptible bool + service *raw.Service + zoneURL string + globalURL string + ipAddress string + SwarmMaster bool + SwarmHost string } const ( apiURL = "https://www.googleapis.com/compute/v1/projects/" - imageName = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-1404-trusty-v20150316" + imageName = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-1404-trusty-v20150909a" firewallRule = "docker-machines" port = "2376" firewallTargetTag = "docker-machine" @@ -43,24 +45,29 @@ const ( // NewComputeUtil creates and initializes a ComputeUtil. func newComputeUtil(driver *Driver) (*ComputeUtil, error) { - service, err := newGCEService(driver.ResolveStorePath("."), driver.AuthTokenPath) + client, err := google.DefaultClient(oauth2.NoContext, raw.ComputeScope) if err != nil { return nil, err } + + service, err := raw.New(client) + if err != nil { + return nil, err + } + c := ComputeUtil{ - authTokenPath: driver.AuthTokenPath, - zone: driver.Zone, - instanceName: driver.MachineName, - userName: driver.SSHUser, - project: driver.Project, - diskTypeURL: driver.DiskType, - address: driver.Address, - preemptible: driver.Preemptible, - service: service, - zoneURL: apiURL + driver.Project + "/zones/" + driver.Zone, - globalURL: apiURL + driver.Project + "/global", - SwarmMaster: driver.SwarmMaster, - SwarmHost: driver.SwarmHost, + zone: driver.Zone, + instanceName: driver.MachineName, + userName: driver.SSHUser, + project: driver.Project, + diskTypeURL: driver.DiskType, + address: driver.Address, + preemptible: driver.Preemptible, + service: service, + zoneURL: apiURL + driver.Project + "/zones/" + driver.Zone, + globalURL: apiURL + driver.Project + "/global", + SwarmMaster: driver.SwarmMaster, + SwarmHost: driver.SwarmHost, } return &c, nil } @@ -267,12 +274,8 @@ func (c *ComputeUtil) createInstance(d *Driver) error { return err } log.Infof("Waiting for SSH Key") - err = c.waitForRegionalOp(op.Name) - if err != nil { - return err - } - return nil + return c.waitForRegionalOp(op.Name) } // parseTags computes the tags for the instance. diff --git a/drivers/google/google.go b/drivers/google/google.go index 4f1f3c1ba4..43f2b29648 100644 --- a/drivers/google/google.go +++ b/drivers/google/google.go @@ -14,22 +14,21 @@ import ( // Driver is a struct compatible with the docker.hosts.drivers.Driver interface. type Driver struct { *drivers.BaseDriver - Zone string - MachineType string - DiskType string - Address string - Preemptible bool - Scopes string - DiskSize int - AuthTokenPath string - Project string - Tags string + Zone string + MachineType string + DiskType string + Address string + Preemptible bool + Scopes string + DiskSize int + Project string + Tags string } const ( defaultZone = "us-central1-a" defaultUser = "docker-user" - defaultMachineType = "f1-micro" + defaultMachineType = "n1-standard-1" defaultScopes = "https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write" defaultDiskType = "pd-standard" defaultDiskSize = 10 @@ -62,11 +61,6 @@ func (d *Driver) GetCreateFlags() []mcnflag.Flag { Usage: "GCE Project", EnvVar: "GOOGLE_PROJECT", }, - mcnflag.StringFlag{ - Name: "google-auth-token", - Usage: "GCE oAuth token", - EnvVar: "GOOGLE_AUTH_TOKEN", - }, mcnflag.StringFlag{ Name: "google-scopes", Usage: "GCE Scopes (comma-separated if multiple scopes)", @@ -140,24 +134,25 @@ func (d *Driver) DriverName() string { // SetConfigFromFlags initializes the driver based on the command line flags. func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { + d.Project = flags.String("google-project") + if d.Project == "" { + return fmt.Errorf("Please specify the Google Cloud Project name using the option --google-project.") + } + d.Zone = flags.String("google-zone") d.MachineType = flags.String("google-machine-type") d.DiskSize = flags.Int("google-disk-size") d.DiskType = flags.String("google-disk-type") d.Address = flags.String("google-address") d.Preemptible = flags.Bool("google-preemptible") - d.AuthTokenPath = flags.String("google-auth-token") - d.Project = flags.String("google-project") d.Scopes = flags.String("google-scopes") d.Tags = flags.String("google-tags") d.SwarmMaster = flags.Bool("swarm-master") d.SwarmHost = flags.String("swarm-host") d.SwarmDiscovery = flags.String("swarm-discovery") - if d.Project == "" { - return fmt.Errorf("Please specify the Google Cloud Project name using the option --google-project.") - } d.SSHUser = flags.String("google-username") d.SSHPort = 22 + return nil } @@ -183,7 +178,12 @@ func (d *Driver) Create() error { return err } - return c.createInstance(d) + err = c.createInstance(d) + if err != nil { + return fmt.Errorf("You might have to run `gcloud auth login` to get the authorization credentials: %s", err) + } + + return err } // GetURL returns the URL of the remote docker daemon.