qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-18 09:04:49 -07:00
Code Releases Activity

godoc: migrate to App Engine flexible

Browse Source 
See bug for more details on exactly what was migrated.

Notably:
* No more Google-internal deployment scripts; see README.godoc-app and
  the Makefile for details.
* Build tag "golangorg" is used for the godoc configuration used for
  golang.org.
* Use of App Engine libraries replaced with GCP client libraries.
* Redis is used to replace App Engine memcache.
* Google analytics is controlled by an environment variable.
* Regression tests have been migrated from Google-internal.
* hg -> git hash map is moved from Google-internal.

Updates golang/go#27205.

Change-Id: Ia0a983f239c50eda8be2363494c8b784f60c2c6d
Reviewed-on: https://go-review.googlesource.com/133355
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
This commit is contained in:
Chris Broadfoot 2018-09-04 09:55:45 -07:00
parent b71db7f417
commit ee6b03148c
27 changed files with 780 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
cmd/godoc/Dockerfile.prod Normal file
View File

@ -0,0 +1,42 @@
# Builder
#########
FROM golang:1.11 AS build
RUN apt-get update && apt-get install -y \
zip # required for generate-index.bash
ENV GODOC_REF release-branch.go1.11
RUN go get -v -d \
golang.org/x/net/context \
google.golang.org/appengine \
cloud.google.com/go/datastore \
golang.org/x/build \
github.com/gomodule/redigo/redis
COPY . /go/src/golang.org/x/tools
WORKDIR /go/src/golang.org/x/tools/cmd/godoc
RUN git clone --single-branch --depth=1 -b $GODOC_REF https://go.googlesource.com/go /docset
RUN GODOC_DOCSET=/docset ./generate-index.bash
RUN go build -o /godoc -tags=golangorg golang.org/x/tools/cmd/godoc
# Final image
#############
FROM gcr.io/distroless/base
WORKDIR /app
COPY --from=build /godoc /app/
COPY --from=build /go/src/golang.org/x/tools/cmd/godoc/hg-git-mapping.bin /app/
COPY --from=build /docset /goroot
ENV GOROOT /goroot
COPY --from=build /go/src/golang.org/x/tools/cmd/godoc/index.split.* /app/
ENV GODOC_INDEX_GLOB index.split.*
CMD ["/app/godoc"]

24
cmd/godoc/Makefile Normal file
View File

@ -0,0 +1,24 @@
# Copyright 2018 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
.PHONY: usage
usage:
echo "See Makefile"
exit 1
docker-prod: Dockerfile.prod
cd ../..; docker build -f cmd/godoc/Dockerfile.prod --tag=gcr.io/golang-org/godoc:$(VERSION) .
push-prod: docker-prod
docker push gcr.io/golang-org/godoc:$(VERSION)
deploy-prod: push-prod
gcloud -q app deploy app.prod.yaml --project golang-org --no-promote --image-url gcr.io/golang-org/godoc:$(VERSION)
get-latest-url:
@gcloud app versions list -s default --project golang-org --sort-by=~version.createTime --format='value(version.versionUrl)' --limit 1 | cut -f1
regtest:
./regtest.bash $(shell make get-latest-url)

67
cmd/godoc/README.godoc-app
View File

@ -7,31 +7,78 @@ Prerequisites
* Google Cloud SDK
https://cloud.google.com/sdk/
* Redis
* Go sources under $GOROOT
* Godoc sources inside $GOPATH
(go get -d golang.org/x/tools/cmd/godoc)
Running in dev_appserver.py
---------------------------
Running locally, in production mode
-----------------------------------
Use dev_appserver.py to run the server in development mode:
Build the app:
dev_appserver.py app.dev.yaml
go build -tags golangorg
To run the server with generated zip file and search index:
Run the app:
./generate-index.bash
dev_appserver.py app.prod.yaml
./godoc
godoc should come up at http://localhost:8080
Use the --host and --port flags to listen on a different address.
To clean up the index files, use git:
Use the PORT environment variable to change the port:
git clean -xn # n is dry run, replace with f
PORT=8081 ./godoc
Running locally, in production mode, using Docker
-------------------------------------------------
Build the app's Docker container:
VERSION=$(git rev-parse HEAD) make docker-prod
Make sure redis is running on port 6379:
$ echo PING | nc localhost 6379
+PONG
^C
Run the datastore emulator:
gcloud beta emulators datastore start --project golang-org
In another terminal window, run the container:
$(gcloud beta emulators datastore env-init)
docker run --rm \
--net host \
--env GODOC_REDIS_ADDR=localhost:6379 \
--env DATASTORE_EMULATOR_HOST=$DATASTORE_EMULATOR_HOST \
--env DATASTORE_PROJECT_ID=$DATASTORE_PROJECT_ID \
gcr.io/golang-org/godoc
godoc should come up at http://localhost:8080
Deploying to golang.org
-----------------------
Build the image, push it to gcr.io, and deploy to Flex:
VERSION=$(git rev-parse HEAD) make deploy-prod
Run regression tests:
make regtest
Go to the console to migrate traffic to the newly deployed version:
https://console.cloud.google.com/appengine/versions?project=golang-org&serviceId=default&versionssize=50
Shut down any very old versions (keep at least one to roll back to, just in case).
Troubleshooting
---------------

30
cmd/godoc/app.prod.yaml
View File

@ -1,18 +1,16 @@
runtime: go
api_version: go1
instance_class: F4_1G
handlers:
- url: /s
script: _go_app
login: admin
- url: /dl/init
script: _go_app
login: admin
- url: /.*
script: _go_app
runtime: custom
env: flex
env_variables:
GODOC_ZIP: godoc.zip
GODOC_ZIP_PREFIX: goroot
GODOC_INDEX_GLOB: 'index.split.*'
GODOC_PROD: true
# GODOC_ENFORCE_HOSTS: true # TODO(cbro): modify host filter to allow version-specific URLs (see issue 27205).
GODOC_REDIS_ADDR: 10.0.0.4:6379 # instance "gophercache"
GODOC_ANALYTICS: UA-11222381-2
DATASTORE_PROJECT_ID: golang-org
network:
name: golang
resources:
cpu: 4
memory_gb: 7.50

63
cmd/godoc/appinit.go
View File

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build appengine
// +build golangorg
package main
@ -11,26 +11,34 @@ package main
import (
"archive/zip"
"context"
"io"
"log"
"net/http"
"os"
"path"
"regexp"
"runtime"
"strings"
"golang.org/x/tools/godoc"
"golang.org/x/tools/godoc/dl"
"golang.org/x/tools/godoc/proxy"
"golang.org/x/tools/godoc/redirect"
"golang.org/x/tools/godoc/short"
"golang.org/x/tools/godoc/static"
"golang.org/x/tools/godoc/vfs"
"golang.org/x/tools/godoc/vfs/gatefs"
"golang.org/x/tools/godoc/vfs/mapfs"
"golang.org/x/tools/godoc/vfs/zipfs"
"google.golang.org/appengine"
"cloud.google.com/go/datastore"
"golang.org/x/tools/internal/memcache"
)
func init() {
func main() {
log.SetFlags(log.Lshortfile | log.LstdFlags)
var (
// .zip filename
zipFilename = os.Getenv("GODOC_ZIP")
@ -43,7 +51,6 @@ func init() {
indexFilenames = os.Getenv("GODOC_INDEX_GLOB")
)
enforceHosts = !appengine.IsDevAppServer()
playEnabled = true
log.Println("initializing godoc ...")
@ -84,17 +91,61 @@ func init() {
pres.ShowExamples = true
pres.DeclLinks = true
pres.NotesRx = regexp.MustCompile("BUG")
pres.GoogleAnalytics = os.Getenv("GODOC_ANALYTICS")
readTemplates(pres, true)
datastoreClient, memcacheClient := getClients()
// NOTE(cbro): registerHandlers registers itself against DefaultServeMux.
// The mux returned has host enforcement, so it's important to register
// against this mux and not DefaultServeMux.
mux := registerHandlers(pres)
dl.RegisterHandlers(mux)
short.RegisterHandlers(mux)
dl.RegisterHandlers(mux, datastoreClient, memcacheClient)
short.RegisterHandlers(mux, datastoreClient, memcacheClient)
// Register /compile and /share handlers against the default serve mux
// so that other app modules can make plain HTTP requests to those
// hosts. (For reasons, HTTPS communication between modules is broken.)
proxy.RegisterHandlers(http.DefaultServeMux)
http.HandleFunc("/_ah/health", func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "ok")
})
http.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "User-agent: *\nDisallow: /search\n")
})
if err := redirect.LoadChangeMap("hg-git-mapping.bin"); err != nil {
log.Fatalf("LoadChangeMap: %v", err)
}
log.Println("godoc initialization complete")
// TODO(cbro): add instrumentation via opencensus.
port := "8080"
if p := os.Getenv("PORT"); p != "" { // PORT is set by GAE flex.
port = p
}
log.Fatal(http.ListenAndServe(":"+port, nil))
}
func getClients() (*datastore.Client, *memcache.Client) {
ctx := context.Background()
datastoreClient, err := datastore.NewClient(ctx, "")
if err != nil {
if strings.Contains(err.Error(), "missing project") {
log.Fatalf("Missing datastore project. Set the DATASTORE_PROJECT_ID env variable. Use `gcloud beta emulators datastore` to start a local datastore.")
}
log.Fatalf("datastore.NewClient: %v.", err)
}
redisAddr := os.Getenv("GODOC_REDIS_ADDR")
if redisAddr == "" {
log.Fatalf("Missing redis server for godoc in production mode. set GODOC_REDIS_ADDR environment variable.")
}
memcacheClient := memcache.New(redisAddr)
return datastoreClient, memcacheClient
}

2
cmd/godoc/dl.go
View File

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build !golangorg
package main

14
cmd/godoc/generate-index.bash
View File

@ -25,24 +25,24 @@ install() {
}
getArgs() {
if [ ! -v GOROOT ]; then
GOROOT="$(go env GOROOT)"
echo "GOROOT not set explicitly, using go env value instead"
if [ ! -v GODOC_DOCSET ]; then
GODOC_DOCSET="$(go env GOROOT)"
echo "GODOC_DOCSET not set explicitly, using GOROOT instead"
fi
# safety checks
if [ ! -d "$GOROOT" ]; then
error "$GOROOT is not a directory"
if [ ! -d "$GODOC_DOCSET" ]; then
error "$GODOC_DOCSET is not a directory"
fi
# reporting
echo "GOROOT = $GOROOT"
echo "GODOC_DOCSET = $GODOC_DOCSET"
}
makeZipfile() {
echo "*** make $ZIPFILE"
rm -f $ZIPFILE goroot
ln -s "$GOROOT" goroot
ln -s "$GODOC_DOCSET" goroot
zip -q -r $ZIPFILE goroot/* # glob to ignore dotfiles (like .git)
rm goroot
}

5
cmd/godoc/handlers.go
View File

@ -21,6 +21,7 @@ import (
"text/template"
"golang.org/x/tools/godoc"
"golang.org/x/tools/godoc/env"
"golang.org/x/tools/godoc/redirect"
"golang.org/x/tools/godoc/vfs"
)
@ -30,8 +31,6 @@ var (
fs = vfs.NameSpace{}
)
var enforceHosts = false // set true in production on app engine
// hostEnforcerHandler redirects requests to "http://foo.golang.org/bar"
// to "https://golang.org/bar".
// It permits requests to the host "godoc-test.golang.org" for testing and
@ -41,7 +40,7 @@ type hostEnforcerHandler struct {
}
func (h hostEnforcerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if !enforceHosts {
if !env.EnforceHosts() {
h.h.ServeHTTP(w, r)
return
}

BIN
cmd/godoc/hg-git-mapping.bin Normal file
View File

Binary file not shown.

2
cmd/godoc/main.go
View File

@ -23,7 +23,7 @@
// godoc crypto/block Cipher NewCMAC
// - prints doc for Cipher and NewCMAC in package crypto/block
// +build !appengine
// +build !golangorg
package main

2
cmd/godoc/play.go
View File

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build !golangorg
package main

110
cmd/godoc/regtest.bash Executable file
View File

@ -0,0 +1,110 @@
#!/usr/bin/env bash
# Copyright 2018 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
# Regression tests for golang.org.
# Usage: ./regtest.bash https://golang.org/
#TODO: turn this into a Go program. maybe behind a build tag and "go run regtest.go <url>"
set -e
addr="$(echo $1 | sed -e 's/\/$//')"
if [ -z "$addr" ]; then
echo "usage: $0 <addr>" 1>&2
echo "example: $0 https://20180928t023837-dot-golang-org.appspot.com/" 1>&2
exit 1
fi
set -u
# fetch url, check the response with a regexp.
fetch() {
curl -s "${addr}$1" | grep "$2" > /dev/null
}
fatal() {
log "$1"
exit 1
}
log() {
echo "$1" 1>&2
}
logn() {
echo -n "$1" 1>&2
}
log "Checking FAQ..."
fetch /doc/faq 'What is the purpose of the project' || {
fatal "FAQ did not match."
}
log "Checking package listing..."
fetch /pkg/ 'Package tar' || {
fatal "package listing page did not match."
}
log "Checking os package..."
fetch /pkg/os/ 'func Open' || {
fatal "os package page did not match."
}
log "Checking robots.txt..."
fetch /robots.txt 'Disallow: /search' || {
fatal "robots.txt did not match."
}
log "Checking /change/ redirect..."
fetch /change/75944e2e3a63 'bdb10cf' || {
fatal "/change/ direct did not match."
}
log "Checking /dl/ page has data..."
fetch /dl/ 'go1.11.windows-amd64.msi' || {
fatal "/dl/ did not match."
}
log "Checking /dl/?mode=json page has data..."
fetch /dl/?mode=json 'go1.11.windows-amd64.msi' || {
fatal "/dl/?mode=json did not match."
}
log "Checking shortlinks (/s/go2design)..."
fetch /s/go2design 'proposal.*Found' || {
fatal "/s/go2design did not match."
}
log "Checking analytics on pages..."
ga_id="UA-11222381-2"
fetch / $ga_id || fatal "/ missing GA."
fetch /dl/ $ga_id || fatal "/dl/ missing GA."
fetch /project/ $ga_id || fatal "/project missing GA."
fetch /pkg/context/ $ga_id || fatal "/pkg/context missing GA."
log "Checking search..."
fetch /search?q=IsDir 'src/os/types.go' || {
fatal "search result did not match."
}
log "Checking compile service..."
compile="curl -s ${addr}/compile"
p="package main; func main() { print(6*7); }"
$compile --data-urlencode "body=$p" | tee /tmp/compile.out | grep '^{"compile_errors":"","output":"42"}$' > /dev/null || {
cat /tmp/compile.out
fatal "compile service output did not match."
}
$compile --data-urlencode "body=//empty" | tee /tmp/compile.out | grep "expected 'package', found 'EOF'" > /dev/null || {
cat /tmp/compile.out
fatal "compile service error output did not match."
}
# Check API version 2
d="version=2&body=package+main%3Bimport+(%22fmt%22%3B%22time%22)%3Bfunc+main()%7Bfmt.Print(%22A%22)%3Btime.Sleep(time.Second)%3Bfmt.Print(%22B%22)%7D"
$compile --data "$d" | grep '^{"Errors":"","Events":\[{"Message":"A","Kind":"stdout","Delay":0},{"Message":"B","Kind":"stdout","Delay":1000000000}\]}$' > /dev/null || {
fatal "compile service v2 output did not match."
}
log "All OK"

2
cmd/godoc/remotesearch.go
View File

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build !golangorg
package main

13
godoc/appengine.go
View File

@ -1,13 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build appengine
package godoc
import "google.golang.org/appengine"
func init() {
onAppengine = !appengine.IsDevAppServer()
}

121
godoc/dl/dl.go
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
// +build appengine
// Package dl implements a simple downloads frontend server.
//
// It accepts HTTP POST requests to create a new download metadata entity, and
@ -19,6 +17,7 @@ import (
"html"
"html/template"
"io"
"log"
"net/http"
"regexp"
"sort"
@ -27,11 +26,10 @@ import (
"sync"
"time"
"cloud.google.com/go/datastore"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
"google.golang.org/appengine/memcache"
"golang.org/x/tools/godoc/env"
"golang.org/x/tools/internal/memcache"
)
const (
@ -40,11 +38,21 @@ const (
cacheDuration = time.Hour
)
func RegisterHandlers(mux *http.ServeMux) {
mux.HandleFunc("/dl", getHandler)
mux.HandleFunc("/dl/", getHandler) // also serves listHandler
mux.HandleFunc("/dl/upload", uploadHandler)
mux.HandleFunc("/dl/init", initHandler)
type server struct {
datastore *datastore.Client
memcache *memcache.CodecClient
}
func RegisterHandlers(mux *http.ServeMux, dc *datastore.Client, mc *memcache.Client) {
s := server{dc, mc.WithCodec(memcache.Gob)}
mux.HandleFunc("/dl", s.getHandler)
mux.HandleFunc("/dl/", s.getHandler) // also serves listHandler
mux.HandleFunc("/dl/upload", s.uploadHandler)
// NOTE(cbro): this only needs to be run once per project,
// and should be behind an admin login.
// TODO(cbro): move into a locally-run program? or remove?
// mux.HandleFunc("/dl/init", initHandler)
}
// File represents a file on the golang.org downloads page.
@ -191,26 +199,25 @@ var (
templateFuncs = template.FuncMap{"pretty": pretty}
)
func listHandler(w http.ResponseWriter, r *http.Request) {
func (h server) listHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var (
c = appengine.NewContext(r)
d listTemplateData
)
if _, err := memcache.Gob.Get(c, cacheKey, &d); err != nil {
if err == memcache.ErrCacheMiss {
log.Debugf(c, "cache miss")
} else {
log.Errorf(c, "cache get error: %v", err)
ctx := r.Context()
var d listTemplateData
if err := h.memcache.Get(ctx, cacheKey, &d); err != nil {
if err != memcache.ErrCacheMiss {
log.Printf("ERROR cache get error: %v", err)
// NOTE(cbro): continue to hit datastore if the memcache is down.
}
var fs []File
_, err := datastore.NewQuery("File").Ancestor(rootKey(c)).GetAll(c, &fs)
if err != nil {
log.Errorf(c, "error listing: %v", err)
q := datastore.NewQuery("File").Ancestor(rootKey)
if _, err := h.datastore.GetAll(ctx, q, &fs); err != nil {
log.Printf("ERROR error listing: %v", err)
http.Error(w, "Could not get download page. Try again in a few minutes.", 500)
return
}
d.Stable, d.Unstable, d.Archive = filesToReleases(fs)
@ -219,8 +226,8 @@ func listHandler(w http.ResponseWriter, r *http.Request) {
}
item := &memcache.Item{Key: cacheKey, Object: &d, Expiration: cacheDuration}
if err := memcache.Gob.Set(c, item); err != nil {
log.Errorf(c, "cache set error: %v", err)
if err := h.memcache.Set(ctx, item); err != nil {
log.Printf("ERROR cache set error: %v", err)
}
}
@ -229,13 +236,13 @@ func listHandler(w http.ResponseWriter, r *http.Request) {
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
if err := enc.Encode(d.Stable); err != nil {
log.Errorf(c, "failed rendering JSON for releases: %v", err)
log.Printf("ERROR rendering JSON for releases: %v", err)
}
return
}
if err := listTemplate.ExecuteTemplate(w, "root", d); err != nil {
log.Errorf(c, "error executing template: %v", err)
log.Printf("ERROR executing template: %v", err)
}
}
@ -383,12 +390,12 @@ func parseVersion(v string) (maj, min int, tail string) {
return
}
func uploadHandler(w http.ResponseWriter, r *http.Request) {
func (h server) uploadHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
c := appengine.NewContext(r)
ctx := r.Context()
// Authenticate using a user token (same as gomote).
user := r.FormValue("user")
@ -396,7 +403,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
http.Error(w, "bad user", http.StatusForbidden)
return
}
if r.FormValue("key") != userKey(c, user) {
if r.FormValue("key") != h.userKey(ctx, user) {
http.Error(w, "bad key", http.StatusForbidden)
return
}
@ -404,7 +411,7 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
var f File
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(&f); err != nil {
log.Errorf(c, "error decoding upload JSON: %v", err)
log.Printf("ERROR decoding upload JSON: %v", err)
http.Error(w, "Something broke", http.StatusInternalServerError)
return
}
@ -415,19 +422,19 @@ func uploadHandler(w http.ResponseWriter, r *http.Request) {
if f.Uploaded.IsZero() {
f.Uploaded = time.Now()
}
k := datastore.NewKey(c, "File", f.Filename, 0, rootKey(c))
if _, err := datastore.Put(c, k, &f); err != nil {
log.Errorf(c, "putting File entity: %v", err)
k := datastore.NameKey("File", f.Filename, rootKey)
if _, err := h.datastore.Put(ctx, k, &f); err != nil {
log.Printf("ERROR File entity: %v", err)
http.Error(w, "could not put File entity", http.StatusInternalServerError)
return
}
if err := memcache.Delete(c, cacheKey); err != nil {
log.Errorf(c, "cache delete error: %v", err)
if err := h.memcache.Delete(ctx, cacheKey); err != nil {
log.Printf("ERROR delete error: %v", err)
}
io.WriteString(w, "OK")
}
func getHandler(w http.ResponseWriter, r *http.Request) {
func (h server) getHandler(w http.ResponseWriter, r *http.Request) {
// For go get golang.org/dl/go1.x.y, we need to serve the
// same meta tags at /dl for cmd/go to validate against /dl/go1.x.y:
if r.URL.Path == "/dl" && (r.Method == "GET" || r.Method == "HEAD") && r.FormValue("go-get") == "1" {
@ -444,7 +451,7 @@ func getHandler(w http.ResponseWriter, r *http.Request) {
name := strings.TrimPrefix(r.URL.Path, "/dl/")
if name == "" {
listHandler(w, r)
h.listHandler(w, r)
return
}
if fileRe.MatchString(name) {
@ -486,10 +493,10 @@ func validUser(user string) bool {
return false
}
func userKey(c context.Context, user string) string {
h := hmac.New(md5.New, []byte(secret(c)))
h.Write([]byte("user-" + user))
return fmt.Sprintf("%x", h.Sum(nil))
func (h server) userKey(c context.Context, user string) string {
hash := hmac.New(md5.New, []byte(h.secret(c)))
hash.Write([]byte("user-" + user))
return fmt.Sprintf("%x", hash.Sum(nil))
}
var (
@ -497,18 +504,18 @@ var (
goGetRe = regexp.MustCompile(`^go[0-9a-z.]+\.[0-9a-z.-]+$`)
)
func initHandler(w http.ResponseWriter, r *http.Request) {
func (h server) initHandler(w http.ResponseWriter, r *http.Request) {
var fileRoot struct {
Root string
}
c := appengine.NewContext(r)
k := rootKey(c)
err := datastore.RunInTransaction(c, func(c context.Context) error {
err := datastore.Get(c, k, &fileRoot)
ctx := r.Context()
k := rootKey
_, err := h.datastore.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
err := tx.Get(k, &fileRoot)
if err != nil && err != datastore.ErrNoSuchEntity {
return err
}
_, err = datastore.Put(c, k, &fileRoot)
_, err = tx.Put(k, &fileRoot)
return err
}, nil)
if err != nil {
@ -519,9 +526,7 @@ func initHandler(w http.ResponseWriter, r *http.Request) {
}
// rootKey is the ancestor of all File entities.
func rootKey(c context.Context) *datastore.Key {
return datastore.NewKey(c, "FileRoot", "root", 0, nil)
}
var rootKey = datastore.NameKey("FileRoot", "root", nil)
// pretty returns a human-readable version of the given OS, Arch, or Kind.
func pretty(s string) string {
@ -559,11 +564,11 @@ type builderKey struct {
Secret string
}
func (k *builderKey) Key(c context.Context) *datastore.Key {
return datastore.NewKey(c, "BuilderKey", "root", 0, nil)
func (k *builderKey) Key() *datastore.Key {
return datastore.NameKey("BuilderKey", "root", nil)
}
func secret(c context.Context) string {
func (h server) secret(ctx context.Context) string {
// check with rlock
theKey.RLock()
k := theKey.Secret
@ -580,18 +585,18 @@ func secret(c context.Context) string {
}
// fill
if err := datastore.Get(c, theKey.Key(c), &theKey.builderKey); err != nil {
if err := h.datastore.Get(ctx, theKey.Key(), &theKey.builderKey); err != nil {
if err == datastore.ErrNoSuchEntity {
// If the key is not stored in datastore, write it.
// This only happens at the beginning of a new deployment.
// The code is left here for SDK use and in case a fresh
// deployment is ever needed. "gophers rule" is not the
// real key.
if !appengine.IsDevAppServer() {
if env.IsProd() {
panic("lost key from datastore")
}
theKey.Secret = "gophers rule"
datastore.Put(c, theKey.Key(c), &theKey.builderKey)
h.datastore.Put(ctx, theKey.Key(), &theKey.builderKey)
return theKey.Secret
}
panic("cannot load builder key: " + err.Error())

2
godoc/dl/dl_test.go
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
// +build appengine
package dl
import (

2
godoc/dl/tmpl.go
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
// +build appengine
package dl
// TODO(adg): refactor this to use the tools/godoc/static template.

41
godoc/env/env.go vendored Normal file
View File

@ -0,0 +1,41 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package env provides environment information for the godoc server running on
// golang.org.
package env
import (
"log"
"os"
"strconv"
)
var (
isProd = boolEnv("GODOC_PROD")
enforceHosts = boolEnv("GODOC_ENFORCE_HOSTS")
)
// IsProd reports whether the server is running in its production configuration
// on golang.org.
func IsProd() bool {
return isProd
}
// EnforceHosts reports whether host filtering should be enforced.
func EnforceHosts() bool {
return enforceHosts
}
func boolEnv(key string) bool {
v := os.Getenv(key)
if v == "" {
return false
}
b, err := strconv.ParseBool(v)
if err != nil {
log.Fatalf("environment variable %s (%q) must be a boolean", key, v)
}
return b
}

25
godoc/page.go
View File

@ -10,6 +10,8 @@ import (
"path/filepath"
"runtime"
"strings"
"golang.org/x/tools/godoc/env"
)
// Page describes the contents of the top-level godoc webpage.
@ -22,10 +24,11 @@ type Page struct {
Body []byte
GoogleCN bool // page is being served from golang.google.cn
// filled in by servePage
SearchBox bool
Playground bool
Version string
// filled in by ServePage
SearchBox bool
Playground bool
Version string
GoogleAnalytics string
}
func (p *Presentation) ServePage(w http.ResponseWriter, page Page) {
@ -35,6 +38,7 @@ func (p *Presentation) ServePage(w http.ResponseWriter, page Page) {
page.SearchBox = p.Corpus.IndexEnabled
page.Playground = p.ShowPlayground
page.Version = runtime.Version()
page.GoogleAnalytics = p.GoogleAnalytics
applyTemplateToResponseWriter(w, p.GodocHTML, page)
}
@ -49,20 +53,19 @@ func (p *Presentation) ServeError(w http.ResponseWriter, r *http.Request, relpat
}
}
p.ServePage(w, Page{
Title: "File " + relpath,
Subtitle: relpath,
Body: applyTemplate(p.ErrorHTML, "errorHTML", err),
GoogleCN: googleCN(r),
Title: "File " + relpath,
Subtitle: relpath,
Body: applyTemplate(p.ErrorHTML, "errorHTML", err),
GoogleCN: googleCN(r),
GoogleAnalytics: p.GoogleAnalytics,
})
}
var onAppengine = false // overridden in appengine.go when on app engine
func googleCN(r *http.Request) bool {
if r.FormValue("googlecn") != "" {
return true
}
if !onAppengine {
if !env.IsProd() {
return false
}
if strings.HasSuffix(r.Host, ".cn") {

4
godoc/pres.go
View File

@ -92,6 +92,10 @@ type Presentation struct {
// body for displaying search results.
SearchResults []SearchResultFunc
// GoogleAnalytics optionally adds Google Analytics via the provided
// tracking ID to each page.
GoogleAnalytics string
initFuncMapOnce sync.Once
funcMap template.FuncMap
templateFuncs template.FuncMap

57
godoc/proxy/proxy.go
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build appengine
// Package proxy proxies requests to the playground's compile and share handlers.
// It is designed to run only on the instance of godoc that serves golang.org.
package proxy
@ -13,6 +11,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httputil"
"net/url"
@ -20,12 +19,18 @@ import (
"time"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/log"
"google.golang.org/appengine/urlfetch"
"golang.org/x/tools/godoc/env"
)
const playgroundURL = "https://play.golang.org"
var proxy *httputil.ReverseProxy
func init() {
target, _ := url.Parse(playgroundURL)
proxy = httputil.NewSingleHostReverseProxy(target)
}
type Request struct {
Body string
}
@ -41,8 +46,6 @@ type Event struct {
Delay time.Duration // time to wait before printing Message
}
const playgroundURL = "https://play.golang.org"
const expires = 7 * 24 * time.Hour // 1 week
var cacheControlHeader = fmt.Sprintf("public, max-age=%d", int(expires.Seconds()))
@ -57,21 +60,17 @@ func compile(w http.ResponseWriter, r *http.Request) {
return
}
ctx := appengine.NewContext(r)
ctx := r.Context()
body := r.FormValue("body")
res := &Response{}
req := &Request{Body: body}
if err := makeCompileRequest(ctx, req, res); err != nil {
log.Errorf(ctx, "compile error: %v", err)
log.Printf("ERROR compile error: %v", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
expiresTime := time.Now().Add(expires).UTC()
w.Header().Set("Expires", expiresTime.Format(time.RFC1123))
w.Header().Set("Cache-Control", cacheControlHeader)
var out interface{}
switch r.FormValue("version") {
case "2":
@ -82,9 +81,17 @@ func compile(w http.ResponseWriter, r *http.Request) {
Output string `json:"output"`
}{res.Errors, flatten(res.Events)}
}
if err := json.NewEncoder(w).Encode(out); err != nil {
log.Errorf(ctx, "encoding response: %v", err)
b, err := json.Marshal(out)
if err != nil {
log.Printf("ERROR encoding response: %v", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
expiresTime := time.Now().Add(expires).UTC()
w.Header().Set("Expires", expiresTime.Format(time.RFC1123))
w.Header().Set("Cache-Control", cacheControlHeader)
w.Write(b)
}
// makePlaygroundRequest sends the given Request to the playground compile
@ -94,17 +101,22 @@ func makeCompileRequest(ctx context.Context, req *Request, res *Response) error
if err != nil {
return fmt.Errorf("marshalling request: %v", err)
}
r, err := urlfetch.Client(ctx).Post(playgroundURL+"/compile", "application/json", bytes.NewReader(reqJ))
hReq, _ := http.NewRequest("POST", playgroundURL+"/compile", bytes.NewReader(reqJ))
hReq.Header.Set("Content-Type", "application/json")
hReq = hReq.WithContext(ctx)
r, err := http.DefaultClient.Do(hReq)
if err != nil {
return fmt.Errorf("making request: %v", err)
}
defer r.Body.Close()
if r.StatusCode != http.StatusOK {
b, _ := ioutil.ReadAll(r.Body)
return fmt.Errorf("bad status: %v body:\n%s", r.Status, b)
}
err = json.NewDecoder(r.Body).Decode(res)
if err != nil {
if err := json.NewDecoder(r.Body).Decode(res); err != nil {
return fmt.Errorf("unmarshalling response: %v", err)
}
return nil
@ -124,17 +136,14 @@ func share(w http.ResponseWriter, r *http.Request) {
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
return
}
target, _ := url.Parse(playgroundURL)
p := httputil.NewSingleHostReverseProxy(target)
p.Transport = &urlfetch.Transport{Context: appengine.NewContext(r)}
p.ServeHTTP(w, r)
proxy.ServeHTTP(w, r)
}
func googleCN(r *http.Request) bool {
if r.FormValue("googlecn") != "" {
return true
}
if appengine.IsDevAppServer() {
if !env.IsProd() {
return false
}
if strings.HasSuffix(r.Host, ".cn") {

79
godoc/short/short.go
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
// +build appengine
// Package short implements a simple URL shortener, serving an administrative
// interface at /s and shortened urls from /s/key.
// It is designed to run only on the instance of godoc that serves golang.org.
@ -15,16 +13,15 @@ import (
"errors"
"fmt"
"html/template"
"io"
"log"
"net/http"
"net/url"
"regexp"
"cloud.google.com/go/datastore"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
"google.golang.org/appengine/memcache"
"golang.org/x/tools/internal/memcache"
"google.golang.org/appengine/user"
)
@ -41,17 +38,32 @@ type Link struct {
var validKey = regexp.MustCompile(`^[a-zA-Z0-9-_.]+$`)
func RegisterHandlers(mux *http.ServeMux) {
mux.HandleFunc(prefix, adminHandler)
mux.HandleFunc(prefix+"/", linkHandler)
type server struct {
datastore *datastore.Client
memcache *memcache.CodecClient
}
func RegisterHandlers(mux *http.ServeMux, dc *datastore.Client, mc *memcache.Client) {
s := server{dc, mc.WithCodec(memcache.JSON)}
mux.HandleFunc(prefix+"/", s.linkHandler)
// TODO(cbro): move storage of the links to a text file in Gerrit.
// Disable the admin handler until that happens, since GAE Flex doesn't support
// the "google.golang.org/appengine/user" package.
// See golang.org/issue/27205#issuecomment-418673218
// mux.HandleFunc(prefix, adminHandler)
mux.HandleFunc(prefix, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
io.WriteString(w, "Link creation temporarily unavailable. See golang.org/issue/27205.")
})
}
// linkHandler services requests to short URLs.
// http://golang.org/s/key
// It consults memcache and datastore for the Link for key.
// It then sends a redirects or an error message.
func linkHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
func (h server) linkHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
key := r.URL.Path[len(prefix)+1:]
if !validKey.MatchString(key) {
@ -60,16 +72,15 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
}
var link Link
_, err := memcache.JSON.Get(c, cacheKey(key), &link)
if err != nil {
k := datastore.NewKey(c, kind, key, 0, nil)
err = datastore.Get(c, k, &link)
if err := h.memcache.Get(ctx, cacheKey(key), &link); err != nil {
k := datastore.NameKey(kind, key, nil)
err = h.datastore.Get(ctx, k, &link)
switch err {
case datastore.ErrNoSuchEntity:
http.Error(w, "not found", http.StatusNotFound)
return
default: // != nil
log.Errorf(c, "%q: %v", key, err)
log.Printf("ERROR %q: %v", key, err)
http.Error(w, "internal server error", http.StatusInternalServerError)
return
case nil:
@ -77,8 +88,8 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
Key: cacheKey(key),
Object: &link,
}
if err := memcache.JSON.Set(c, item); err != nil {
log.Warningf(c, "%q: %v", key, err)
if err := h.memcache.Set(ctx, item); err != nil {
log.Printf("WARNING %q: %v", key, err)
}
}
}
@ -89,10 +100,10 @@ func linkHandler(w http.ResponseWriter, r *http.Request) {
var adminTemplate = template.Must(template.New("admin").Parse(templateHTML))
// adminHandler serves an administrative interface.
func adminHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
func (h server) adminHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if !user.IsAdmin(c) {
if !user.IsAdmin(ctx) {
http.Error(w, "forbidden", http.StatusForbidden)
return
}
@ -104,24 +115,24 @@ func adminHandler(w http.ResponseWriter, r *http.Request) {
switch r.FormValue("do") {
case "Add":
newLink = &Link{key, r.FormValue("target")}
doErr = putLink(c, newLink)
doErr = h.putLink(ctx, newLink)
case "Delete":
k := datastore.NewKey(c, kind, key, 0, nil)
doErr = datastore.Delete(c, k)
k := datastore.NameKey(kind, key, nil)
doErr = h.datastore.Delete(ctx, k)
default:
http.Error(w, "unknown action", http.StatusBadRequest)
}
err := memcache.Delete(c, cacheKey(key))
err := h.memcache.Delete(ctx, cacheKey(key))
if err != nil && err != memcache.ErrCacheMiss {
log.Warningf(c, "%q: %v", key, err)
log.Printf("WARNING %q: %v", key, err)
}
}
var links []*Link
_, err := datastore.NewQuery(kind).Order("Key").GetAll(c, &links)
if err != nil {
q := datastore.NewQuery(kind).Order("Key")
if _, err := h.datastore.GetAll(ctx, q, &links); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "%v", err)
log.Printf("ERROR %v", err)
return
}
@ -150,20 +161,20 @@ func adminHandler(w http.ResponseWriter, r *http.Request) {
Error error
}{baseURL, prefix, links, newLink, doErr}
if err := adminTemplate.Execute(w, &data); err != nil {
log.Criticalf(c, "adminTemplate: %v", err)
log.Printf("ERROR adminTemplate: %v", err)
}
}
// putLink validates the provided link and puts it into the datastore.
func putLink(c context.Context, link *Link) error {
func (h server) putLink(ctx context.Context, link *Link) error {
if !validKey.MatchString(link.Key) {
return errors.New("invalid key; must match " + validKey.String())
}
if _, err := url.Parse(link.Target); err != nil {
return fmt.Errorf("bad target: %v", err)
}
k := datastore.NewKey(c, kind, link.Key, 0, nil)
_, err := datastore.Put(c, k, link)
k := datastore.NameKey(kind, link.Key, nil)
_, err := h.datastore.Put(ctx, k, link)
return err
}

2
godoc/short/tmpl.go
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
// +build appengine
package short
const templateHTML = `

22
godoc/static/godoc.html
View File

@ -15,6 +15,19 @@
{{end}}
<link rel="stylesheet" href="/lib/godoc/jquery.treeview.css">
<script>window.initFuncs = [];</script>
{{with .GoogleAnalytics}}
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(["_setAccount", "{{.}}"]);
window.trackPageview = function() {
_gaq.push(["_trackPageview", location.pathname+location.hash]);
};
window.trackPageview();
window.trackEvent = function(category, action, opt_label, opt_value, opt_noninteraction) {
_gaq.push(["_trackEvent", category, action, opt_label, opt_value, opt_noninteraction]);
};
</script>
{{end}}
<script src="/lib/godoc/jquery.js" defer></script>
<script src="/lib/godoc/jquery.treeview.js" defer></script>
<script src="/lib/godoc/jquery.treeview.edit.js" defer></script>
@ -112,6 +125,15 @@ and code is licensed under a <a href="/LICENSE">BSD license</a>.<br>
</div><!-- .container -->
</div><!-- #page -->
{{if .GoogleAnalytics}}
<script type="text/javascript">
(function() {
var ga = document.createElement("script"); ga.type = "text/javascript"; ga.async = true;
ga.src = ("https:" == document.location.protocol ? "https://ssl" : "http://www") + ".google-analytics.com/ga.js";
var s = document.getElementsByTagName("script")[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
{{end}}
</body>
</html>

2
godoc/static/static.go
View File

File diff suppressed because one or more lines are too long

157
internal/memcache/memcache.go Normal file
View File

@ -0,0 +1,157 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package memcache provides a minimally compatible interface for
// google.golang.org/appengine/memcache
// and stores the data in Redis (e.g., via Cloud Memorystore).
package memcache
import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"errors"
"time"
"github.com/gomodule/redigo/redis"
)
var ErrCacheMiss = errors.New("memcache: cache miss")
func New(addr string) *Client {
const maxConns = 20
pool := redis.NewPool(func() (redis.Conn, error) {
return redis.Dial("tcp", addr)
}, maxConns)
return &Client{
pool: pool,
}
}
type Client struct {
pool *redis.Pool
}
type CodecClient struct {
client *Client
codec Codec
}
type Item struct {
Key string
Value []byte
Object interface{} // Used with Codec.
Expiration time.Duration // Read-only.
}
func (c *Client) WithCodec(codec Codec) *CodecClient {
return &CodecClient{
c, codec,
}
}
func (c *Client) Delete(ctx context.Context, key string) error {
conn, err := c.pool.GetContext(ctx)
if err != nil {
return err
}
defer conn.Close()
_, err = conn.Do("DEL", key)
return err
}
func (c *CodecClient) Delete(ctx context.Context, key string) error {
return c.client.Delete(ctx, key)
}
func (c *Client) Set(ctx context.Context, item *Item) error {
if item.Value == nil {
return errors.New("nil item value")
}
return c.set(ctx, item.Key, item.Value, item.Expiration)
}
func (c *CodecClient) Set(ctx context.Context, item *Item) error {
if item.Object == nil {
return errors.New("nil object value")
}
b, err := c.codec.Marshal(item.Object)
if err != nil {
return err
}
return c.client.set(ctx, item.Key, b, item.Expiration)
}
func (c *Client) set(ctx context.Context, key string, value []byte, expiration time.Duration) error {
conn, err := c.pool.GetContext(ctx)
if err != nil {
return err
}
defer conn.Close()
if expiration == 0 {
_, err := conn.Do("SET", key, value)
return err
}
// NOTE(cbro): redis does not support expiry in units more granular than a second.
exp := int64(expiration.Seconds())
if exp == 0 {
// Redis doesn't allow a zero expiration, delete the key instead.
_, err := conn.Do("DEL", key)
return err
}
_, err = conn.Do("SETEX", key, exp, value)
return err
}
// Get gets the item.
func (c *Client) Get(ctx context.Context, key string) ([]byte, error) {
conn, err := c.pool.GetContext(ctx)
if err != nil {
return nil, err
}
defer conn.Close()
b, err := redis.Bytes(conn.Do("GET", key))
if err == redis.ErrNil {
err = ErrCacheMiss
}
return b, err
}
func (c *CodecClient) Get(ctx context.Context, key string, v interface{}) error {
b, err := c.client.Get(ctx, key)
if err != nil {
return err
}
return c.codec.Unmarshal(b, v)
}
var (
Gob = Codec{gobMarshal, gobUnmarshal}
JSON = Codec{json.Marshal, json.Unmarshal}
)
type Codec struct {
Marshal func(interface{}) ([]byte, error)
Unmarshal func([]byte, interface{}) error
}
func gobMarshal(v interface{}) ([]byte, error) {
var buf bytes.Buffer
if err := gob.NewEncoder(&buf).Encode(v); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func gobUnmarshal(data []byte, v interface{}) error {
return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v)
}

83
internal/memcache/memcache_test.go Normal file
View File

@ -0,0 +1,83 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package memcache
import (
"context"
"os"
"testing"
"time"
)
func getClient(t *testing.T) *Client {
t.Helper()
addr := os.Getenv("GOLANG_REDIS_ADDR")
if addr == "" {
t.Skip("skipping because GOLANG_REDIS_ADDR is unset")
}
return New(addr)
}
func TestCacheMiss(t *testing.T) {
c := getClient(t)
ctx := context.Background()
if _, err := c.Get(ctx, "doesnotexist"); err != ErrCacheMiss {
t.Errorf("got %v; want ErrCacheMiss", err)
}
}
func TestExpiry(t *testing.T) {
c := getClient(t).WithCodec(Gob)
ctx := context.Background()
key := "testexpiry"
firstTime := time.Now()
err := c.Set(ctx, &Item{
Key: key,
Object: firstTime,
Expiration: 3500 * time.Millisecond, // NOTE: check that non-rounded expiries work.
})
if err != nil {
t.Fatalf("Set: %v", err)
}
var newTime time.Time
if err := c.Get(ctx, key, &newTime); err != nil {
t.Fatalf("Get: %v", err)
}
if !firstTime.Equal(newTime) {
t.Errorf("Get: got value %v, want %v", newTime, firstTime)
}
time.Sleep(4 * time.Second)
if err := c.Get(ctx, key, &newTime); err != ErrCacheMiss {
t.Errorf("Get: got %v, want ErrCacheMiss", err)
}
}
func TestShortExpiry(t *testing.T) {
c := getClient(t).WithCodec(Gob)
ctx := context.Background()
key := "testshortexpiry"
err := c.Set(ctx, &Item{
Key: key,
Value: []byte("ok"),
Expiration: time.Millisecond,
})
if err != nil {
t.Fatalf("Set: %v", err)
}
if err := c.Get(ctx, key, nil); err != ErrCacheMiss {
t.Errorf("GetBytes: got %v, want ErrCacheMiss", err)
}
}