From e83bc563341b00d5a331c0c3419b084ca6b36ab3 Mon Sep 17 00:00:00 2001 From: Andrew Gerrand Date: Wed, 23 Sep 2015 12:17:19 +1000 Subject: [PATCH] godoc/{dl,proxy,short}: move packages out of Google's internal repo These were built inside Google but have been in production for years. Move them into the public tools repository so that they can be more easily maintained. This is step one to moving the entire golang.org deployment process out of Google's internal source repository. Change-Id: I72f875c5020b3f58f1c0cea1d19268e0f991833f Reviewed-on: https://go-review.googlesource.com/14842 Reviewed-by: Chris Broadfoot --- cmd/godoc/appinit.go | 10 +- cmd/godoc/dl.go | 4 +- cmd/godoc/handlers.go | 4 +- cmd/godoc/play.go | 2 + godoc/dl/dl.go | 450 +++++++++++++++++++++++++++++++++++++++ godoc/dl/dl_test.go | 70 ++++++ godoc/dl/tmpl.go | 267 +++++++++++++++++++++++ godoc/proxy/appengine.go | 13 ++ godoc/proxy/proxy.go | 169 +++++++++++++++ godoc/short/short.go | 171 +++++++++++++++ godoc/short/tmpl.go | 119 +++++++++++ 11 files changed, 1275 insertions(+), 4 deletions(-) create mode 100644 godoc/dl/dl.go create mode 100644 godoc/dl/dl_test.go create mode 100644 godoc/dl/tmpl.go create mode 100644 godoc/proxy/appengine.go create mode 100644 godoc/proxy/proxy.go create mode 100644 godoc/short/short.go create mode 100644 godoc/short/tmpl.go diff --git a/cmd/godoc/appinit.go b/cmd/godoc/appinit.go index c2c166aec6..25618e0615 100644 --- a/cmd/godoc/appinit.go +++ b/cmd/godoc/appinit.go @@ -16,12 +16,15 @@ import ( "regexp" "golang.org/x/tools/godoc" + "golang.org/x/tools/godoc/dl" + "golang.org/x/tools/godoc/proxy" + "golang.org/x/tools/godoc/short" "golang.org/x/tools/godoc/static" "golang.org/x/tools/godoc/vfs" "golang.org/x/tools/godoc/vfs/mapfs" "golang.org/x/tools/godoc/vfs/zipfs" - "appengine" + "google.golang.org/appengine" ) func init() { @@ -64,7 +67,10 @@ func init() { pres.NotesRx = regexp.MustCompile("BUG") readTemplates(pres, true) - registerHandlers(pres) + mux := registerHandlers(pres) + dl.RegisterHandlers(mux) + proxy.RegisterHandlers(mux) + short.RegisterHandlers(mux) log.Println("godoc initialization complete") } diff --git a/cmd/godoc/dl.go b/cmd/godoc/dl.go index bd73831624..40e66584bc 100644 --- a/cmd/godoc/dl.go +++ b/cmd/godoc/dl.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build !appengine + package main import "net/http" @@ -10,5 +12,5 @@ import "net/http" // This file will not be included when deploying godoc to golang.org. func init() { - http.Handle("/dl/", http.RedirectHandler("http://golang.org/dl/", http.StatusFound)) + http.Handle("/dl/", http.RedirectHandler("https://golang.org/dl/", http.StatusFound)) } diff --git a/cmd/godoc/handlers.go b/cmd/godoc/handlers.go index 5919a6feb2..25e4cd801a 100644 --- a/cmd/godoc/handlers.go +++ b/cmd/godoc/handlers.go @@ -61,7 +61,7 @@ func (h hostEnforcerHandler) validHost(host string) bool { return false } -func registerHandlers(pres *godoc.Presentation) { +func registerHandlers(pres *godoc.Presentation) *http.ServeMux { if pres == nil { panic("nil Presentation") } @@ -73,6 +73,8 @@ func registerHandlers(pres *godoc.Presentation) { mux.Handle("/pkg/C/", redirect.Handler("/cmd/cgo/")) redirect.Register(mux) http.Handle("/", hostEnforcerHandler{mux}) + + return mux } func readTemplate(name string) *template.Template { diff --git a/cmd/godoc/play.go b/cmd/godoc/play.go index a56ffe28bb..b8c55345c4 100644 --- a/cmd/godoc/play.go +++ b/cmd/godoc/play.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build !appengine + package main import ( diff --git a/godoc/dl/dl.go b/godoc/dl/dl.go new file mode 100644 index 0000000000..ec0f4c3ba0 --- /dev/null +++ b/godoc/dl/dl.go @@ -0,0 +1,450 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// Package dl implements a simple downloads frontend server. +// +// It accepts HTTP POST requests to create a new download metadata entity, and +// lists entities with sorting and filtering. +// It is designed to run only on the instance of godoc that serves golang.org. +package dl + +import ( + "crypto/hmac" + "crypto/md5" + "encoding/json" + "fmt" + "html/template" + "io" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "golang.org/x/net/context" + + "google.golang.org/appengine" + "google.golang.org/appengine/datastore" + "google.golang.org/appengine/log" + "google.golang.org/appengine/memcache" + "google.golang.org/appengine/user" + "google.golang.org/cloud/compute/metadata" +) + +const ( + gcsBaseURL = "https://storage.googleapis.com/golang/" + cacheKey = "download_list_3" // increment if listTemplateData changes + cacheDuration = time.Hour +) + +var builderKey string + +func init() { + builderKey, _ = metadata.ProjectAttributeValue("builder-key") +} + +func RegisterHandlers(mux *http.ServeMux) { + mux.Handle("/dl", http.RedirectHandler("/dl/", http.StatusFound)) + mux.HandleFunc("/dl/", getHandler) // also serves listHandler + mux.HandleFunc("/dl/upload", uploadHandler) + mux.HandleFunc("/dl/init", initHandler) +} + +type File struct { + Filename string + OS string + Arch string + Version string + Checksum string `datastore:",noindex"` + Size int64 `datastore:",noindex"` + Kind string // "archive", "installer", "source" + Uploaded time.Time +} + +func (f File) PrettyOS() string { + if f.OS == "darwin" { + switch { + case strings.Contains(f.Filename, "osx10.8"): + return "OS X 10.8+" + case strings.Contains(f.Filename, "osx10.6"): + return "OS X 10.6+" + } + } + return pretty(f.OS) +} + +func (f File) PrettySize() string { + const mb = 1 << 20 + if f.Size == 0 { + return "" + } + if f.Size < mb { + // All Go releases are >1mb, but handle this case anyway. + return fmt.Sprintf("%v bytes", f.Size) + } + return fmt.Sprintf("%.0fMB", float64(f.Size)/mb) +} + +func (f File) Highlight() bool { + switch { + case f.Kind == "source": + return true + case f.Arch == "amd64" && f.OS == "linux": + return true + case f.Arch == "amd64" && f.Kind == "installer": + switch f.OS { + case "windows": + return true + case "darwin": + if !strings.Contains(f.Filename, "osx10.6") { + return true + } + } + } + return false +} + +func (f File) URL() string { + return gcsBaseURL + f.Filename +} + +type Release struct { + Version string + Stable bool + Files []File + Visible bool // show files on page load +} + +type Feature struct { + // The File field will be filled in by the first stable File + // whose name matches the given fileRE. + File + fileRE *regexp.Regexp + + Platform string // "Microsoft Windows", "Mac OS X", "Linux" + Requirements string // "Windows XP and above, 64-bit Intel Processor" +} + +// featuredFiles lists the platforms and files to be featured +// at the top of the downloads page. +var featuredFiles = []Feature{ + { + Platform: "Microsoft Windows", + Requirements: "Windows XP or later, Intel 64-bit processor", + fileRE: regexp.MustCompile(`\.windows-amd64\.msi$`), + }, + { + Platform: "Apple OS X", + Requirements: "OS X 10.8 or later, Intel 64-bit processor", + fileRE: regexp.MustCompile(`\.darwin-amd64(-osx10\.8)?\.pkg$`), + }, + { + Platform: "Linux", + Requirements: "Linux 2.6.23 or later, Intel 64-bit processor", + fileRE: regexp.MustCompile(`\.linux-amd64\.tar\.gz$`), + }, + { + Platform: "Source", + fileRE: regexp.MustCompile(`\.src\.tar\.gz$`), + }, +} + +// data to send to the template; increment cacheKey if you change this. +type listTemplateData struct { + Featured []Feature + Stable, Unstable []Release + LoginURL string +} + +var ( + listTemplate = template.Must(template.New("").Funcs(templateFuncs).Parse(templateHTML)) + templateFuncs = template.FuncMap{"pretty": pretty} +) + +func listHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + var ( + c = appengine.NewContext(r) + d listTemplateData + ) + if _, err := memcache.Gob.Get(c, cacheKey, &d); err != nil { + if err == memcache.ErrCacheMiss { + log.Debugf(c, "cache miss") + } else { + log.Errorf(c, "cache get error: %v", err) + } + + var fs []File + _, err := datastore.NewQuery("File").Ancestor(rootKey(c)).GetAll(c, &fs) + if err != nil { + log.Errorf(c, "error listing: %v", err) + return + } + d.Stable, d.Unstable = filesToReleases(fs) + if len(d.Stable) > 0 { + d.Featured = filesToFeatured(d.Stable[0].Files) + } + + d.LoginURL, _ = user.LoginURL(c, "/dl") + if user.Current(c) != nil { + d.LoginURL, _ = user.LogoutURL(c, "/dl") + } + + item := &memcache.Item{Key: cacheKey, Object: &d, Expiration: cacheDuration} + if err := memcache.Gob.Set(c, item); err != nil { + log.Errorf(c, "cache set error: %v", err) + } + } + if err := listTemplate.ExecuteTemplate(w, "root", d); err != nil { + log.Errorf(c, "error executing template: %v", err) + } +} + +func filesToFeatured(fs []File) (featured []Feature) { + for _, feature := range featuredFiles { + for _, file := range fs { + if feature.fileRE.MatchString(file.Filename) { + feature.File = file + featured = append(featured, feature) + break + } + } + } + return +} + +func filesToReleases(fs []File) (stable, unstable []Release) { + sort.Sort(fileOrder(fs)) + + var r *Release + var stableMaj, stableMin int + add := func() { + if r == nil { + return + } + if r.Stable { + if len(stable) == 0 { + // Display files for latest stable release. + stableMaj, stableMin, _ = parseVersion(r.Version) + r.Visible = len(stable) == 0 + } + stable = append(stable, *r) + return + } + if len(unstable) != 0 { + // Only show one (latest) unstable version. + return + } + maj, min, _ := parseVersion(r.Version) + if maj < stableMaj || maj == stableMaj && min <= stableMin { + // Display unstable version only if newer than the + // latest stable release. + return + } + r.Visible = true + unstable = append(unstable, *r) + } + for _, f := range fs { + if r == nil || f.Version != r.Version { + add() + r = &Release{ + Version: f.Version, + Stable: isStable(f.Version), + } + } + r.Files = append(r.Files, f) + } + add() + return +} + +// isStable reports whether the version string v is a stable version. +func isStable(v string) bool { + return !strings.Contains(v, "beta") && !strings.Contains(v, "rc") +} + +type fileOrder []File + +func (s fileOrder) Len() int { return len(s) } +func (s fileOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s fileOrder) Less(i, j int) bool { + a, b := s[i], s[j] + if av, bv := a.Version, b.Version; av != bv { + return versionLess(av, bv) + } + if a.OS != b.OS { + return a.OS < b.OS + } + if a.Arch != b.Arch { + return a.Arch < b.Arch + } + if a.Kind != b.Kind { + return a.Kind < b.Kind + } + return a.Filename < b.Filename +} + +func versionLess(a, b string) bool { + // Put stable releases first. + if isStable(a) != isStable(b) { + return isStable(a) + } + maja, mina, ta := parseVersion(a) + majb, minb, tb := parseVersion(b) + if maja == majb { + if mina == minb { + return ta >= tb + } + return mina >= minb + } + return maja >= majb +} + +func parseVersion(v string) (maj, min int, tail string) { + if i := strings.Index(v, "beta"); i > 0 { + tail = v[i:] + v = v[:i] + } + if i := strings.Index(v, "rc"); i > 0 { + tail = v[i:] + v = v[:i] + } + p := strings.Split(strings.TrimPrefix(v, "go1."), ".") + maj, _ = strconv.Atoi(p[0]) + if len(p) < 2 { + return + } + min, _ = strconv.Atoi(p[1]) + return +} + +func uploadHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + c := appengine.NewContext(r) + + // Authenticate using a user token (same as gomote). + user := r.FormValue("user") + if !validUser(user) { + http.Error(w, "bad user", http.StatusForbidden) + return + } + if builderKey == "" { + http.Error(w, "no builder-key found in project metadata", http.StatusInternalServerError) + return + } + if r.FormValue("key") != userKey(c, user) { + http.Error(w, "bad key", http.StatusForbidden) + return + } + + var f File + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&f); err != nil { + log.Errorf(c, "error decoding upload JSON: %v", err) + http.Error(w, "Something broke", http.StatusInternalServerError) + return + } + if f.Filename == "" { + http.Error(w, "Must provide Filename", http.StatusBadRequest) + return + } + if f.Uploaded.IsZero() { + f.Uploaded = time.Now() + } + k := datastore.NewKey(c, "File", f.Filename, 0, rootKey(c)) + if _, err := datastore.Put(c, k, &f); err != nil { + log.Errorf(c, "putting File entity: %v", err) + http.Error(w, "could not put File entity", http.StatusInternalServerError) + return + } + if err := memcache.Delete(c, cacheKey); err != nil { + log.Errorf(c, "cache delete error: %v", err) + } + io.WriteString(w, "OK") +} + +func getHandler(w http.ResponseWriter, r *http.Request) { + name := strings.TrimPrefix(r.URL.Path, "/dl/") + if name == "" { + listHandler(w, r) + return + } + if !fileRe.MatchString(name) { + http.NotFound(w, r) + return + } + http.Redirect(w, r, gcsBaseURL+name, http.StatusFound) +} + +func validUser(user string) bool { + switch user { + case "adg", "bradfitz", "cbro": + return true + } + return false +} + +func userKey(c context.Context, user string) string { + h := hmac.New(md5.New, []byte(builderKey)) + h.Write([]byte("user-" + user)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +var fileRe = regexp.MustCompile(`^go[0-9a-z.]+\.[0-9a-z.-]+\.(tar\.gz|pkg|msi|zip)$`) + +func initHandler(w http.ResponseWriter, r *http.Request) { + var fileRoot struct { + Root string + } + c := appengine.NewContext(r) + k := rootKey(c) + err := datastore.RunInTransaction(c, func(c context.Context) error { + err := datastore.Get(c, k, &fileRoot) + if err != nil && err != datastore.ErrNoSuchEntity { + return err + } + _, err = datastore.Put(c, k, &fileRoot) + return err + }, nil) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + io.WriteString(w, "OK") +} + +// rootKey is the ancestor of all File entities. +func rootKey(c context.Context) *datastore.Key { + return datastore.NewKey(c, "FileRoot", "root", 0, nil) +} + +// pretty returns a human-readable version of the given OS, Arch, or Kind. +func pretty(s string) string { + t, ok := prettyStrings[s] + if !ok { + return s + } + return t +} + +var prettyStrings = map[string]string{ + "darwin": "OS X", + "freebsd": "FreeBSD", + "linux": "Linux", + "windows": "Windows", + + "386": "32-bit", + "amd64": "64-bit", + + "archive": "Archive", + "installer": "Installer", + "source": "Source", +} diff --git a/godoc/dl/dl_test.go b/godoc/dl/dl_test.go new file mode 100644 index 0000000000..207604e86f --- /dev/null +++ b/godoc/dl/dl_test.go @@ -0,0 +1,70 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package dl + +import ( + "sort" + "strings" + "testing" +) + +func TestParseVersion(t *testing.T) { + for _, c := range []struct { + in string + maj, min int + tail string + }{ + {"go1.5", 5, 0, ""}, + {"go1.5beta1", 5, 0, "beta1"}, + {"go1.5.1", 5, 1, ""}, + {"go1.5.1rc1", 5, 1, "rc1"}, + } { + maj, min, tail := parseVersion(c.in) + if maj != c.maj || min != c.min || tail != c.tail { + t.Errorf("parseVersion(%q) = %v, %v, %q; want %v, %v, %q", + c.in, maj, min, tail, c.maj, c.min, c.tail) + } + } +} + +func TestFileOrder(t *testing.T) { + fs := []File{ + {Filename: "go1.3.src.tar.gz", Version: "go1.3", OS: "", Arch: "", Kind: "source"}, + {Filename: "go1.3.1.src.tar.gz", Version: "go1.3.1", OS: "", Arch: "", Kind: "source"}, + {Filename: "go1.3.linux-amd64.tar.gz", Version: "go1.3", OS: "linux", Arch: "amd64", Kind: "archive"}, + {Filename: "go1.3.1.linux-amd64.tar.gz", Version: "go1.3.1", OS: "linux", Arch: "amd64", Kind: "archive"}, + {Filename: "go1.3.darwin-amd64.tar.gz", Version: "go1.3", OS: "darwin", Arch: "amd64", Kind: "archive"}, + {Filename: "go1.3.darwin-amd64.pkg", Version: "go1.3", OS: "darwin", Arch: "amd64", Kind: "installer"}, + {Filename: "go1.3.darwin-386.tar.gz", Version: "go1.3", OS: "darwin", Arch: "386", Kind: "archive"}, + {Filename: "go1.3beta1.linux-amd64.tar.gz", Version: "go1.3beta1", OS: "linux", Arch: "amd64", Kind: "archive"}, + {Filename: "go1.3beta2.linux-amd64.tar.gz", Version: "go1.3beta2", OS: "linux", Arch: "amd64", Kind: "archive"}, + {Filename: "go1.3rc1.linux-amd64.tar.gz", Version: "go1.3rc1", OS: "linux", Arch: "amd64", Kind: "archive"}, + {Filename: "go1.2.linux-amd64.tar.gz", Version: "go1.2", OS: "linux", Arch: "amd64", Kind: "archive"}, + {Filename: "go1.2.2.linux-amd64.tar.gz", Version: "go1.2.2", OS: "linux", Arch: "amd64", Kind: "archive"}, + } + sort.Sort(fileOrder(fs)) + var s []string + for _, f := range fs { + s = append(s, f.Filename) + } + got := strings.Join(s, "\n") + want := strings.Join([]string{ + "go1.3.1.src.tar.gz", + "go1.3.1.linux-amd64.tar.gz", + "go1.3.src.tar.gz", + "go1.3.darwin-386.tar.gz", + "go1.3.darwin-amd64.tar.gz", + "go1.3.darwin-amd64.pkg", + "go1.3.linux-amd64.tar.gz", + "go1.2.2.linux-amd64.tar.gz", + "go1.2.linux-amd64.tar.gz", + "go1.3rc1.linux-amd64.tar.gz", + "go1.3beta2.linux-amd64.tar.gz", + "go1.3beta1.linux-amd64.tar.gz", + }, "\n") + if got != want { + t.Errorf("sort order is\n%s\nwant:\n%s", got, want) + } +} diff --git a/godoc/dl/tmpl.go b/godoc/dl/tmpl.go new file mode 100644 index 0000000000..e1727c9f52 --- /dev/null +++ b/godoc/dl/tmpl.go @@ -0,0 +1,267 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package dl + +// TODO(adg): refactor this to use the tools/godoc/static template. + +const templateHTML = ` +{{define "root"}} + + + + + Downloads - The Go Programming Language + + + + + + + + +
+
+ +

Downloads

+ +

+After downloading a binary release suitable for your system, +please follow the installation instructions. +

+ +

+If you are building from source, +follow the source installation instructions. +

+ +

+See the release history for more +information about Go releases. +

+ +{{with .Featured}} + +{{range .}} +{{template "download" .}} +{{end}} +{{end}} + +
+ +{{with .Stable}} +

Stable versions

+{{template "releases" .}} +{{end}} + +{{with .Unstable}} +

Unstable version

+{{template "releases" .}} +{{end}} + +

Older versions

+ +

+Older releases of Go are available at Google Code. +

+ + + + + + +
+
+ + + + + + +{{end}} + +{{define "releases"}} +{{range .}} +
+ +
+

{{.Version}} ▾

+ {{if .Stable}}{{else}} +

This is an unstable version of Go. Use with caution.

+ {{end}} + {{template "files" .Files}} +
+
+{{end}} +{{end}} + +{{define "files"}} + + + + + + + + + + + +{{range .}} + + + + + + + + +{{else}} + + + +{{end}} +
File nameKindOSArchSizeSHA1 Checksum
{{.Filename}}{{pretty .Kind}}{{.PrettyOS}}{{pretty .Arch}}{{.PrettySize}}{{.Checksum}}
No downloads available.
+{{end}} + +{{define "download"}} + +
{{.Platform}}
+{{with .Requirements}}
{{.}}
{{end}} +
+ {{.Filename}} + {{if .Size}}({{.PrettySize}}){{end}} +
+
SHA1: {{.Checksum}}
+ +{{end}} +` diff --git a/godoc/proxy/appengine.go b/godoc/proxy/appengine.go new file mode 100644 index 0000000000..5a13027acc --- /dev/null +++ b/godoc/proxy/appengine.go @@ -0,0 +1,13 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package proxy + +import "google.golang.org/appengine" + +func init() { + onAppengine = !appengine.IsDevAppServer() +} diff --git a/godoc/proxy/proxy.go b/godoc/proxy/proxy.go new file mode 100644 index 0000000000..74abacbd48 --- /dev/null +++ b/godoc/proxy/proxy.go @@ -0,0 +1,169 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package proxy proxies requests to the sandbox compiler service and the +// playground share handler. +// It is designed to run only on the instance of godoc that serves golang.org. +package proxy + +import ( + "bytes" + "crypto/sha1" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httputil" + "net/url" + "time" + + "golang.org/x/net/context" + + "google.golang.org/appengine" + "google.golang.org/appengine/log" + "google.golang.org/appengine/memcache" + "google.golang.org/appengine/urlfetch" +) + +type Request struct { + Body string +} + +type Response struct { + Errors string + Events []Event +} + +type Event struct { + Message string + Kind string // "stdout" or "stderr" + Delay time.Duration // time to wait before printing Message +} + +const ( + // We need to use HTTP here for "reasons", but the traffic isn't + // sensitive and it only travels across Google's internal network + // so we should be OK. + sandboxURL = "http://sandbox.golang.org/compile" + playgroundURL = "http://play.golang.org" +) + +const expires = 7 * 24 * time.Hour // 1 week +var cacheControlHeader = fmt.Sprintf("public, max-age=%d", int(expires.Seconds())) + +func RegisterHandlers(mux *http.ServeMux) { + mux.HandleFunc("/compile", compile) + mux.HandleFunc("/share", share) +} + +func compile(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "I only answer to POST requests.", http.StatusMethodNotAllowed) + return + } + + c := appengine.NewContext(r) + + body := r.FormValue("body") + res := &Response{} + key := cacheKey(body) + if _, err := memcache.Gob.Get(c, key, res); err != nil { + if err != memcache.ErrCacheMiss { + log.Errorf(c, "getting response cache: %v", err) + } + + req := &Request{Body: body} + if err := makeSandboxRequest(c, req, res); err != nil { + log.Errorf(c, "compile error: %v", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + item := &memcache.Item{Key: key, Object: res} + if err := memcache.Gob.Set(c, item); err != nil { + log.Errorf(c, "setting response cache: %v", err) + } + } + + expiresTime := time.Now().Add(expires).UTC() + w.Header().Set("Expires", expiresTime.Format(time.RFC1123)) + w.Header().Set("Cache-Control", cacheControlHeader) + + var out interface{} + switch r.FormValue("version") { + case "2": + out = res + default: // "1" + out = struct { + CompileErrors string `json:"compile_errors"` + Output string `json:"output"` + }{res.Errors, flatten(res.Events)} + } + if err := json.NewEncoder(w).Encode(out); err != nil { + log.Errorf(c, "encoding response: %v", err) + } +} + +// makeSandboxRequest sends the given Request to the sandbox +// and stores the response in the given Response. +func makeSandboxRequest(c context.Context, req *Request, res *Response) error { + reqJ, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("marshalling request: %v", err) + } + r, err := urlfetch.Client(c).Post(sandboxURL, "application/json", bytes.NewReader(reqJ)) + if err != nil { + return fmt.Errorf("making request: %v", err) + } + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + b, _ := ioutil.ReadAll(r.Body) + return fmt.Errorf("bad status: %v body:\n%s", r.Status, b) + } + err = json.NewDecoder(r.Body).Decode(res) + if err != nil { + return fmt.Errorf("unmarshalling response: %v", err) + } + return nil +} + +// flatten takes a sequence of Events and returns their contents, concatenated. +func flatten(seq []Event) string { + var buf bytes.Buffer + for _, e := range seq { + buf.WriteString(e.Message) + } + return buf.String() +} + +func cacheKey(body string) string { + h := sha1.New() + io.WriteString(h, body) + return fmt.Sprintf("prog-%x", h.Sum(nil)) +} + +func share(w http.ResponseWriter, r *http.Request) { + if !allowShare(r) { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } + target, _ := url.Parse(playgroundURL) + p := httputil.NewSingleHostReverseProxy(target) + p.Transport = &urlfetch.Transport{Context: appengine.NewContext(r)} + p.ServeHTTP(w, r) +} + +var onAppengine = false // will be overriden by appengine.go and appenginevm.go + +func allowShare(r *http.Request) bool { + if !onAppengine { + return true + } + switch r.Header.Get("X-AppEngine-Country") { + case "", "ZZ", "HK", "CN", "RC": + return false + } + return true +} diff --git a/godoc/short/short.go b/godoc/short/short.go new file mode 100644 index 0000000000..da72b0d6bf --- /dev/null +++ b/godoc/short/short.go @@ -0,0 +1,171 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// Package short implements a simple URL shortener, serving an administrative +// interface at /s and shortened urls from /s/key. +// It is designed to run only on the instance of godoc that serves golang.org. +package short + +// TODO(adg): collect statistics on URL visits + +import ( + "errors" + "fmt" + "html/template" + "net/http" + "net/url" + "regexp" + + "golang.org/x/net/context" + + "google.golang.org/appengine" + "google.golang.org/appengine/datastore" + "google.golang.org/appengine/log" + "google.golang.org/appengine/memcache" + "google.golang.org/appengine/user" +) + +const ( + prefix = "/s" + kind = "Link" + baseURL = "https://golang.org" + prefix +) + +// Link represents a short link. +type Link struct { + Key, Target string +} + +var validKey = regexp.MustCompile(`^[a-zA-Z0-9-_.]+$`) + +func RegisterHandlers(mux *http.ServeMux) { + mux.HandleFunc(prefix, adminHandler) + mux.HandleFunc(prefix+"/", linkHandler) +} + +// linkHandler services requests to short URLs. +// http://golang.org/s/key +// It consults memcache and datastore for the Link for key. +// It then sends a redirects or an error message. +func linkHandler(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + key := r.URL.Path[len(prefix)+1:] + if !validKey.MatchString(key) { + http.Error(w, "not found", http.StatusNotFound) + return + } + + var link Link + _, err := memcache.JSON.Get(c, cacheKey(key), &link) + if err != nil { + k := datastore.NewKey(c, kind, key, 0, nil) + err = datastore.Get(c, k, &link) + switch err { + case datastore.ErrNoSuchEntity: + http.Error(w, "not found", http.StatusNotFound) + return + default: // != nil + log.Errorf(c, "%q: %v", key, err) + http.Error(w, "internal server error", http.StatusInternalServerError) + return + case nil: + item := &memcache.Item{ + Key: cacheKey(key), + Object: &link, + } + if err := memcache.JSON.Set(c, item); err != nil { + log.Warningf(c, "%q: %v", key, err) + } + } + } + + http.Redirect(w, r, link.Target, http.StatusFound) +} + +var adminTemplate = template.Must(template.New("admin").Parse(templateHTML)) + +// adminHandler serves an administrative interface. +func adminHandler(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + if !user.IsAdmin(c) { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + + var newLink *Link + var doErr error + if r.Method == "POST" { + key := r.FormValue("key") + switch r.FormValue("do") { + case "Add": + newLink = &Link{key, r.FormValue("target")} + doErr = putLink(c, newLink) + case "Delete": + k := datastore.NewKey(c, kind, key, 0, nil) + doErr = datastore.Delete(c, k) + default: + http.Error(w, "unknown action", http.StatusBadRequest) + } + err := memcache.Delete(c, cacheKey(key)) + if err != nil && err != memcache.ErrCacheMiss { + log.Warningf(c, "%q: %v", key, err) + } + } + + var links []*Link + _, err := datastore.NewQuery(kind).Order("Key").GetAll(c, &links) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + log.Errorf(c, "%v", err) + return + } + + // Put the new link in the list if it's not there already. + // (Eventual consistency means that it might not show up + // immediately, which might be confusing for the user.) + if newLink != nil && doErr == nil { + found := false + for i := range links { + if links[i].Key == newLink.Key { + found = true + break + } + } + if !found { + links = append([]*Link{newLink}, links...) + } + newLink = nil + } + + var data = struct { + BaseURL string + Prefix string + Links []*Link + New *Link + Error error + }{baseURL, prefix, links, newLink, doErr} + if err := adminTemplate.Execute(w, &data); err != nil { + log.Criticalf(c, "adminTemplate: %v", err) + } +} + +// putLink validates the provided link and puts it into the datastore. +func putLink(c context.Context, link *Link) error { + if !validKey.MatchString(link.Key) { + return errors.New("invalid key; must match " + validKey.String()) + } + if _, err := url.Parse(link.Target); err != nil { + return fmt.Errorf("bad target: %v", err) + } + k := datastore.NewKey(c, kind, link.Key, 0, nil) + _, err := datastore.Put(c, k, link) + return err +} + +// cacheKey returns a short URL key as a memcache key. +func cacheKey(key string) string { + return "link-" + key +} diff --git a/godoc/short/tmpl.go b/godoc/short/tmpl.go new file mode 100644 index 0000000000..66f5401eb8 --- /dev/null +++ b/godoc/short/tmpl.go @@ -0,0 +1,119 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package short + +const templateHTML = ` + + + +golang.org URL shortener + + + + + + +{{with .Error}} + + + + + + +{{end}} + + + + + + + + + + + + + + +{{with .Links}} + + + + + +{{range .}} + + + + + +{{end}} +{{end}} + +
Error
{{.}}
KeyTarget
+
Short Link  
+
+ + +
+
+ + + + + + +`