diff --git a/cmd/tip/Dockerfile b/cmd/tip/Dockerfile deleted file mode 100644 index 5fe8b39d27..0000000000 --- a/cmd/tip/Dockerfile +++ /dev/null @@ -1,176 +0,0 @@ -FROM golang:1.11 - -RUN apt-get update && apt-get install --no-install-recommends -y -q build-essential git - -# For implicit GOCACHE (Issues 29243 and 29251), set HOME: -RUN mkdir -p /home/gopher -ENV HOME /home/gopher - -# golang puts its go install here (weird but true) -ENV GOROOT_BOOTSTRAP /usr/local/go - -# BEGIN deps (run `make update-deps` to update) - -# Repo cloud.google.com/go at b5eca92 (2018-10-23) -ENV REV=b5eca92245a08e245bc29c4880c9779ea4aeaa9a -RUN go get -d cloud.google.com/go/compute/metadata `#and 7 other pkgs` &&\ - (cd /go/src/cloud.google.com/go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo github.com/golang/protobuf at b4deda0 (2018-04-30) -ENV REV=b4deda0973fb4c70b50d226b1af49f3da59f5265 -RUN go get -d github.com/golang/protobuf/proto `#and 6 other pkgs` &&\ - (cd /go/src/github.com/golang/protobuf && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo github.com/googleapis/gax-go at 317e000 (2017-09-15) -ENV REV=317e0006254c44a0ac427cc52a0e083ff0b9622f -RUN go get -d github.com/googleapis/gax-go &&\ - (cd /go/src/github.com/googleapis/gax-go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo go.opencensus.io at ebd8d31 (2018-05-16) -ENV REV=ebd8d31470fedf6c27d0e3056653ddff642509b8 -RUN go get -d go.opencensus.io/exporter/stackdriver/propagation `#and 12 other pkgs` &&\ - (cd /go/src/go.opencensus.io && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo golang.org/x/build at 7b78c20 (2018-12-13) -ENV REV=7b78c2042368d5c56ee9dbd92ab5fa988c763944 -RUN go get -d golang.org/x/build/autocertcache &&\ - (cd /go/src/golang.org/x/build && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo golang.org/x/crypto at e4dc69e (2018-11-06) -ENV REV=e4dc69e5b2fd71dcaf8bd5d054eb936deb78d1fa -RUN go get -d golang.org/x/crypto/acme `#and 2 other pkgs` &&\ - (cd /go/src/golang.org/x/crypto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo golang.org/x/net at 891ebc4 (2018-12-13) -ENV REV=891ebc4b82d6e74f468c533b06f983c7be918a96 -RUN go get -d golang.org/x/net/context `#and 8 other pkgs` &&\ - (cd /go/src/golang.org/x/net && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo golang.org/x/oauth2 at f42d051 (2018-11-06) -ENV REV=f42d05182288abf10faef86d16c0d07b8d40ea2d -RUN go get -d golang.org/x/oauth2 `#and 5 other pkgs` &&\ - (cd /go/src/golang.org/x/oauth2 && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo golang.org/x/sys at 4d1cda0 (2018-12-13) -ENV REV=4d1cda033e0619309c606fc686de3adcf599539e -RUN go get -d golang.org/x/sys/unix &&\ - (cd /go/src/golang.org/x/sys && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo golang.org/x/text at 6f44c5a (2018-10-30) -ENV REV=6f44c5a2ea40ee3593d98cdcc905cc1fdaa660e2 -RUN go get -d golang.org/x/text/secure/bidirule `#and 4 other pkgs` &&\ - (cd /go/src/golang.org/x/text && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo google.golang.org/api at 20530fd (2018-05-06) -ENV REV=20530fd5d65ad2caee87891f9896d7547cb400c9 -RUN go get -d google.golang.org/api/gensupport `#and 9 other pkgs` &&\ - (cd /go/src/google.golang.org/api && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo google.golang.org/genproto at 86e600f (2018-04-27) -ENV REV=86e600f69ee4704c6efbf6a2a40a5c10700e76c2 -RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 4 other pkgs` &&\ - (cd /go/src/google.golang.org/genproto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Repo google.golang.org/grpc at 07ef407 (2018-08-06) -ENV REV=07ef407d991f1004e6c3367c8f452ed9a02f17ff -RUN go get -d google.golang.org/grpc `#and 26 other pkgs` &&\ - (cd /go/src/google.golang.org/grpc && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV) - -# Optimization to speed up iterative development, not necessary for correctness: -RUN go install cloud.google.com/go/compute/metadata \ - cloud.google.com/go/iam \ - cloud.google.com/go/internal \ - cloud.google.com/go/internal/optional \ - cloud.google.com/go/internal/trace \ - cloud.google.com/go/internal/version \ - cloud.google.com/go/storage \ - github.com/golang/protobuf/proto \ - github.com/golang/protobuf/protoc-gen-go/descriptor \ - github.com/golang/protobuf/ptypes \ - github.com/golang/protobuf/ptypes/any \ - github.com/golang/protobuf/ptypes/duration \ - github.com/golang/protobuf/ptypes/timestamp \ - github.com/googleapis/gax-go \ - go.opencensus.io/exporter/stackdriver/propagation \ - go.opencensus.io/internal \ - go.opencensus.io/internal/tagencoding \ - go.opencensus.io/plugin/ochttp \ - go.opencensus.io/plugin/ochttp/propagation/b3 \ - go.opencensus.io/stats \ - go.opencensus.io/stats/internal \ - go.opencensus.io/stats/view \ - go.opencensus.io/tag \ - go.opencensus.io/trace \ - go.opencensus.io/trace/internal \ - go.opencensus.io/trace/propagation \ - golang.org/x/build/autocertcache \ - golang.org/x/crypto/acme \ - golang.org/x/crypto/acme/autocert \ - golang.org/x/net/context \ - golang.org/x/net/context/ctxhttp \ - golang.org/x/net/http/httpguts \ - golang.org/x/net/http2 \ - golang.org/x/net/http2/hpack \ - golang.org/x/net/idna \ - golang.org/x/net/internal/timeseries \ - golang.org/x/net/trace \ - golang.org/x/oauth2 \ - golang.org/x/oauth2/google \ - golang.org/x/oauth2/internal \ - golang.org/x/oauth2/jws \ - golang.org/x/oauth2/jwt \ - golang.org/x/sys/unix \ - golang.org/x/text/secure/bidirule \ - golang.org/x/text/transform \ - golang.org/x/text/unicode/bidi \ - golang.org/x/text/unicode/norm \ - google.golang.org/api/gensupport \ - google.golang.org/api/googleapi \ - google.golang.org/api/googleapi/internal/uritemplates \ - google.golang.org/api/googleapi/transport \ - google.golang.org/api/internal \ - google.golang.org/api/iterator \ - google.golang.org/api/option \ - google.golang.org/api/storage/v1 \ - google.golang.org/api/transport/http \ - google.golang.org/genproto/googleapis/api/annotations \ - google.golang.org/genproto/googleapis/iam/v1 \ - google.golang.org/genproto/googleapis/rpc/code \ - google.golang.org/genproto/googleapis/rpc/status \ - google.golang.org/grpc \ - google.golang.org/grpc/balancer \ - google.golang.org/grpc/balancer/base \ - google.golang.org/grpc/balancer/roundrobin \ - google.golang.org/grpc/codes \ - google.golang.org/grpc/connectivity \ - google.golang.org/grpc/credentials \ - google.golang.org/grpc/encoding \ - google.golang.org/grpc/encoding/proto \ - google.golang.org/grpc/grpclog \ - google.golang.org/grpc/internal \ - google.golang.org/grpc/internal/backoff \ - google.golang.org/grpc/internal/channelz \ - google.golang.org/grpc/internal/envconfig \ - google.golang.org/grpc/internal/grpcrand \ - google.golang.org/grpc/internal/transport \ - google.golang.org/grpc/keepalive \ - google.golang.org/grpc/metadata \ - google.golang.org/grpc/naming \ - google.golang.org/grpc/peer \ - google.golang.org/grpc/resolver \ - google.golang.org/grpc/resolver/dns \ - google.golang.org/grpc/resolver/passthrough \ - google.golang.org/grpc/stats \ - google.golang.org/grpc/status \ - google.golang.org/grpc/tap -# END deps. - -# golang sets GOPATH=/go -ADD . /go/src/tip -WORKDIR /go/src/tip -RUN go install --tags=autocert -ENTRYPOINT ["/go/bin/tip"] - -# We listen on 8080 (for historical reasons). The service.yaml maps public port 80 to 8080. -# We also listen on 443 for LetsEncrypt TLS. -EXPOSE 8080 443 diff --git a/cmd/tip/Makefile b/cmd/tip/Makefile deleted file mode 100644 index 8054f17d0a..0000000000 --- a/cmd/tip/Makefile +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2017 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -VERSION ?= $(shell git rev-parse --short HEAD) -MUTABLE_VERSION ?= latest - -IMAGE_STAGING := gcr.io/go-dashboard-dev/tip -IMAGE_PROD := gcr.io/symbolic-datum-552/tip - -.PHONY: usage - -usage: - echo "See Makefile" - exit 1 - -update-deps: - go install golang.org/x/build/cmd/gitlock - gitlock --update=Dockerfile --ignore=NONE --tags=autocert golang.org/x/tools/cmd/tip - -docker-image: Dockerfile *.go - docker build --force-rm -f Dockerfile --tag=$(IMAGE_PROD):$(VERSION) . - docker tag $(IMAGE_PROD):$(VERSION) $(IMAGE_PROD):$(MUTABLE_VERSION) - docker tag $(IMAGE_PROD):$(VERSION) $(IMAGE_STAGING):$(VERSION) - docker tag $(IMAGE_PROD):$(VERSION) $(IMAGE_STAGING):$(MUTABLE_VERSION) - -push-prod: docker-image - docker push $(IMAGE_PROD):$(MUTABLE_VERSION) - docker push $(IMAGE_PROD):$(VERSION) -push-staging: docker-image - docker push $(IMAGE_STAGING):$(MUTABLE_VERSION) - docker push $(IMAGE_STAGING):$(VERSION) - -deploy-prod: push-prod - go install golang.org/x/build/cmd/xb - xb --prod kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_PROD):$(VERSION) -deploy-staging: push-staging - go install golang.org/x/build/cmd/xb - xb --staging kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_STAGING):$(VERSION) - diff --git a/cmd/tip/README b/cmd/tip/README deleted file mode 100644 index b5f42a9926..0000000000 --- a/cmd/tip/README +++ /dev/null @@ -1,28 +0,0 @@ -============================================================ -Old instructions, only valid for talks.golang.org: -============================================================ - -1. Deploy the app. - - To deploy tip.golang.org: - (See Kubernetes instruction below.) - - To deploy talks.golang.org: - $ gcloud --project golang-org app deploy --no-promote talks.yaml - -2. Wait until the deployed version is serving requests. - -3. Go to the developer console and upgrade the default version. - https://console.developers.google.com/appengine/versions?project=golang-org&moduleId=tip - -4. Clean up any old versions (they continue to use at least one instance). - -============================================================ -New Kubernetes instructions, for tip.golang.org: -============================================================ - -Kubernetes instructions: - - * make deploy-prod - -Also: move talks.golang.org to GKE too? diff --git a/cmd/tip/cert.go b/cmd/tip/cert.go deleted file mode 100644 index 991cb38c13..0000000000 --- a/cmd/tip/cert.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -// +build autocert - -// This file contains autocert and cloud.google.com/go/storage -// dependencies we want to hide by default from the Go build system, -// which currently doesn't know how to fetch non-golang.org/x/* -// dependencies. The Dockerfile builds the production binary -// with this code using --tags=autocert. - -package main - -import ( - "context" - "crypto/tls" - "log" - "net/http" - "strings" - - "cloud.google.com/go/storage" - "golang.org/x/build/autocertcache" - "golang.org/x/crypto/acme/autocert" -) - -func init() { - runHTTPS = runHTTPSAutocert - certInit = certInitAutocert - wrapHTTPMux = wrapHTTPMuxAutocert -} - -var autocertManager *autocert.Manager - -func certInitAutocert() { - var cache autocert.Cache - if b := *autoCertCacheBucket; b != "" { - sc, err := storage.NewClient(context.Background()) - if err != nil { - log.Fatalf("storage.NewClient: %v", err) - } - cache = autocertcache.NewGoogleCloudStorageCache(sc, b) - } - autocertManager = &autocert.Manager{ - Prompt: autocert.AcceptTOS, - HostPolicy: autocert.HostWhitelist(strings.Split(*autoCertDomain, ",")...), - Cache: cache, - } -} - -func runHTTPSAutocert(h http.Handler) error { - s := &http.Server{ - Addr: ":https", - Handler: h, - TLSConfig: &tls.Config{ - GetCertificate: autocertManager.GetCertificate, - }, - } - return s.ListenAndServeTLS("", "") -} - -func wrapHTTPMuxAutocert(h http.Handler) http.Handler { - return autocertManager.HTTPHandler(h) -} diff --git a/cmd/tip/godoc.go b/cmd/tip/godoc.go deleted file mode 100644 index 13b891253d..0000000000 --- a/cmd/tip/godoc.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "errors" - "fmt" - "io" - "log" - "os" - "os/exec" - "path/filepath" -) - -type godocBuilder struct{} - -func prefix8(s string) string { - if len(s) < 8 { - return s - } - return s[:8] -} - -func (b godocBuilder) Signature(heads map[string]string) string { - // x/net is intentionally not a part of the signature, because - // at this time it does not contribute substantially to the deployed - // website, and so we don't want tip.golang.org redeployed whenever - // x/net changes. This will no longer matter when the Go website uses - // modules to pin its dependencies to specific versions. - return fmt.Sprintf("go=%v/tools=%v", prefix8(heads["go"]), prefix8(heads["tools"])) -} - -func (b godocBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) { - goDir := filepath.Join(dir, "go") - toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools") - netDir := filepath.Join(dir, "gopath/src/golang.org/x/net") - logger.Printf("checking out go repo ...") - if err := checkout(repoURL+"go", heads["go"], goDir); err != nil { - return nil, fmt.Errorf("checkout of go: %v", err) - } - logger.Printf("checking out tools repo ...") - if err := checkout(repoURL+"tools", heads["tools"], toolsDir); err != nil { - return nil, fmt.Errorf("checkout of tools: %v", err) - } - logger.Printf("checking out net repo ...") - if err := checkout(repoURL+"net", heads["net"], netDir); err != nil { - return nil, fmt.Errorf("checkout of net: %v", err) - } - - var logWriter io.Writer = toLoggerWriter{logger} - - make := exec.Command(filepath.Join(goDir, "src/make.bash")) - make.Dir = filepath.Join(goDir, "src") - make.Stdout = logWriter - make.Stderr = logWriter - logger.Printf("running make.bash in %s ...", make.Dir) - if err := make.Run(); err != nil { - return nil, fmt.Errorf("running make.bash: %v", err) - } - - logger.Printf("installing godoc ...") - goBin := filepath.Join(goDir, "bin/go") - goPath := filepath.Join(dir, "gopath") - install := exec.Command(goBin, "install", "golang.org/x/tools/cmd/godoc") - install.Stdout = logWriter - install.Stderr = logWriter - install.Env = append(os.Environ(), - "GOROOT="+goDir, - "GOPATH="+goPath, - "GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"), - ) - if err := install.Run(); err != nil { - return nil, fmt.Errorf("go install golang.org/x/tools/cmd/godoc: %v", err) - } - - logger.Printf("starting godoc ...") - godocBin := filepath.Join(goPath, "bin/godoc") - godoc := exec.Command(godocBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play") - godoc.Env = append(os.Environ(), "GOROOT="+goDir) - godoc.Stdout = logWriter - godoc.Stderr = logWriter - if err := godoc.Start(); err != nil { - return nil, fmt.Errorf("starting godoc: %v", err) - } - return godoc, nil -} - -var indexingMsg = []byte("Indexing in progress: result may be inaccurate") - -func (b godocBuilder) HealthCheck(hostport string) error { - body, err := getOK(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", hostport)) - if err != nil { - return err - } - if bytes.Contains(body, indexingMsg) { - return errors.New("still indexing") - } - return nil -} diff --git a/cmd/tip/talks.go b/cmd/tip/talks.go deleted file mode 100644 index cc1472e4d5..0000000000 --- a/cmd/tip/talks.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "errors" - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "runtime" -) - -type talksBuilder struct{} - -func (b talksBuilder) Signature(heads map[string]string) string { - return heads["talks"] -} - -const talksToolsRev = "8cab8a1319f0be9798e7fe78b15da75e5f94b2e9" - -func (b talksBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) { - // TODO: use logger - toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools") - if err := checkout(repoURL+"tools", talksToolsRev, toolsDir); err != nil { - return nil, err - } - talksDir := filepath.Join(dir, "gopath/src/golang.org/x/talks") - if err := checkout(repoURL+"talks", heads["talks"], talksDir); err != nil { - return nil, err - } - - goDir := os.Getenv("GOROOT_BOOTSTRAP") - if goDir == "" { - goDir = runtime.GOROOT() - } - goBin := filepath.Join(goDir, "bin/go") - goPath := filepath.Join(dir, "gopath") - presentPath := "golang.org/x/tools/cmd/present" - install := exec.Command(goBin, "install", "-tags=appenginevm", presentPath) - install.Env = append(os.Environ(), "GOROOT="+goDir, "GOPATH="+goPath) - if err := runErr(install); err != nil { - return nil, err - } - - talksBin := filepath.Join(goPath, "bin/present") - presentSrc := filepath.Join(goPath, "src", presentPath) - present := exec.Command(talksBin, "-http="+hostport, "-base="+presentSrc) - present.Dir = talksDir - // TODO(adg): log this somewhere useful - present.Stdout = os.Stdout - present.Stderr = os.Stderr - if err := present.Start(); err != nil { - return nil, err - } - return present, nil -} - -var talksMsg = []byte("Talks - The Go Programming Language") - -func (b talksBuilder) HealthCheck(hostport string) error { - body, err := getOK(fmt.Sprintf("http://%v/", hostport)) - if err != nil { - return err - } - if !bytes.Contains(body, talksMsg) { - return errors.New("couldn't match string") - } - return nil -} diff --git a/cmd/tip/talks.yaml b/cmd/tip/talks.yaml deleted file mode 100644 index 24391a753e..0000000000 --- a/cmd/tip/talks.yaml +++ /dev/null @@ -1,12 +0,0 @@ -service: talks -runtime: custom -env: flex - -manual_scaling: - instances: 1 - -env_variables: - TIP_BUILDER: 'talks' - -health_check: - enable_health_check: False diff --git a/cmd/tip/tip-prod.yaml b/cmd/tip/tip-prod.yaml deleted file mode 100644 index 8b30a973cd..0000000000 --- a/cmd/tip/tip-prod.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: tipgodoc-deployment -spec: - replicas: 1 - template: - metadata: - name: tipgodoc - labels: - app: tipgodoc - spec: - volumes: - - name: cache-volume - emptyDir: {} - containers: - - name: tipgodoc - image: gcr.io/symbolic-datum-552/tip:latest - imagePullPolicy: Always - command: ["/go/bin/tip", "--autocert=tip.golang.org,beta.golang.org", "--autocert-bucket=golang-tip-autocert"] - env: - - name: TMPDIR - value: /build - - name: TIP_BUILDER - value: godoc - volumeMounts: - - mountPath: /build - name: cache-volume - ports: - - containerPort: 8080 - - containerPort: 443 - resources: - requests: - cpu: "2" - memory: "4Gi" - limits: - cpu: "2" - memory: "8Gi" diff --git a/cmd/tip/tip-service.yaml b/cmd/tip/tip-service.yaml deleted file mode 100644 index eeb2a1cb53..0000000000 --- a/cmd/tip/tip-service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: tipgodoc -spec: - ports: - - port: 80 - targetPort: 8080 - name: http - - port: 443 - targetPort: 443 - name: https - selector: - app: tipgodoc - type: LoadBalancer - loadBalancerIP: 130.211.180.236 diff --git a/cmd/tip/tip.go b/cmd/tip/tip.go deleted file mode 100644 index 6109560602..0000000000 --- a/cmd/tip/tip.go +++ /dev/null @@ -1,441 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -// Command tip is the tip.golang.org server, -// serving the latest HEAD straight from the Git oven. -package main - -import ( - "bufio" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "net/http/httputil" - "net/url" - "os" - "os/exec" - "path/filepath" - "sync" - "time" -) - -const ( - repoURL = "https://go.googlesource.com/" - metaURL = "https://go.googlesource.com/?b=master&format=JSON" - startTimeout = 10 * time.Minute -) - -var startTime = time.Now() - -var ( - autoCertDomain = flag.String("autocert", "", "if non-empty, listen on port 443 and serve a LetsEncrypt cert for this hostname or hostnames (comma-separated)") - autoCertCacheBucket = flag.String("autocert-bucket", "", "if non-empty, the Google Cloud Storage bucket in which to store the LetsEncrypt cache") -) - -// Hooks that are set non-nil in cert.go if the "autocert" build tag -// is used. -var ( - certInit func() - runHTTPS func(http.Handler) error - wrapHTTPMux func(http.Handler) http.Handler -) - -func main() { - flag.Parse() - - const k = "TIP_BUILDER" - var b Builder - switch os.Getenv(k) { - case "godoc": - b = godocBuilder{} - case "talks": - b = talksBuilder{} - default: - log.Fatalf("Unknown %v value: %q", k, os.Getenv(k)) - } - - if certInit != nil { - certInit() - } - - p := &Proxy{builder: b} - go p.run() - mux := newServeMux(p) - - log.Printf("Starting up tip server for builder %q", os.Getenv(k)) - - errc := make(chan error, 1) - - go func() { - var httpMux http.Handler = mux - if wrapHTTPMux != nil { - httpMux = wrapHTTPMux(httpMux) - } - errc <- http.ListenAndServe(":8080", httpMux) - }() - if *autoCertDomain != "" { - if runHTTPS == nil { - errc <- errors.New("can't use --autocert without building binary with the autocert build tag") - } else { - go func() { - errc <- runHTTPS(mux) - }() - } - log.Printf("Listening on port 443 with LetsEncrypt support on domain %q", *autoCertDomain) - } - if err := <-errc; err != nil { - p.stop() - log.Fatal(err) - } -} - -// Proxy implements the tip.golang.org server: a reverse-proxy -// that builds and runs godoc instances showing the latest docs. -type Proxy struct { - builder Builder - - mu sync.Mutex // protects following fields - proxy http.Handler - cur string // signature of gorepo+toolsrepo - cmd *exec.Cmd // live godoc instance, or nil for none - side string - hostport string // host and port of the live instance - err error -} - -type Builder interface { - Signature(heads map[string]string) string - Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) - HealthCheck(hostport string) error -} - -func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/_tipstatus" { - p.serveStatus(w, r) - return - } - // Redirect the old beta.golang.org URL to tip.golang.org, - // just in case there are old links out there to - // beta.golang.org. (We used to run a "temporary" beta.golang.org - // GCE VM running godoc where "temporary" lasted two years. - // So it lasted so long, there are probably links to it out there.) - if r.Host == "beta.golang.org" { - u := *r.URL - u.Scheme = "https" - u.Host = "tip.golang.org" - http.Redirect(w, r, u.String(), http.StatusFound) - return - } - p.mu.Lock() - proxy := p.proxy - err := p.err - p.mu.Unlock() - if proxy == nil { - s := "starting up" - if err != nil { - s = err.Error() - } - http.Error(w, s, http.StatusInternalServerError) - return - } - proxy.ServeHTTP(w, r) -} - -func (p *Proxy) serveStatus(w http.ResponseWriter, r *http.Request) { - p.mu.Lock() - defer p.mu.Unlock() - fmt.Fprintf(w, "side=%v\ncurrent=%v\nerror=%v\nuptime=%v\n", p.side, p.cur, p.err, int(time.Since(startTime).Seconds())) -} - -func (p *Proxy) serveHealthCheck(w http.ResponseWriter, r *http.Request) { - p.mu.Lock() - defer p.mu.Unlock() - - // NOTE: (App Engine only; not GKE) Status 502, 503, 504 are - // the only status codes that signify an unhealthy app. So - // long as this handler returns one of those codes, this - // instance will not be sent any requests. - if p.proxy == nil { - log.Printf("Health check: not ready") - http.Error(w, "Not ready", http.StatusServiceUnavailable) - return - } - - if err := p.builder.HealthCheck(p.hostport); err != nil { - log.Printf("Health check failed: %v", err) - http.Error(w, "Health check failed", http.StatusServiceUnavailable) - return - } - io.WriteString(w, "ok") -} - -// run runs in its own goroutine. -func (p *Proxy) run() { - p.mu.Lock() - p.side = "a" - p.mu.Unlock() - for { - p.poll() - time.Sleep(30 * time.Second) - } -} - -func (p *Proxy) stop() { - p.mu.Lock() - defer p.mu.Unlock() - if p.cmd != nil { - p.cmd.Process.Kill() - } -} - -// poll runs from the run loop goroutine. -func (p *Proxy) poll() { - heads := gerritMetaMap() - if heads == nil { - return - } - - sig := p.builder.Signature(heads) - - p.mu.Lock() - changes := sig != p.cur - curSide := p.side - p.cur = sig - p.mu.Unlock() - - if !changes { - return - } - - newSide := "b" - if curSide == "b" { - newSide = "a" - } - - dir := filepath.Join(os.TempDir(), "tip", newSide) - if err := os.MkdirAll(dir, 0755); err != nil { - p.err = err - return - } - hostport := "localhost:8081" - if newSide == "b" { - hostport = "localhost:8082" - } - logger := log.New(os.Stderr, sig+": ", log.LstdFlags) - - cmd, err := p.builder.Init(logger, dir, hostport, heads) - if err != nil { - logger.Printf("Init failed: %v", err) - err = fmt.Errorf("builder.Init: %v", err) - } else { - go func() { - // TODO(adg,bradfitz): be smarter about dead processes - if err := cmd.Wait(); err != nil { - logger.Printf("process in %v exited: %v (%T)", dir, err, err) - if ee, ok := err.(*exec.ExitError); ok { - logger.Printf("ProcessState.Sys() = %v", ee.ProcessState.Sys()) - } - } - }() - err = waitReady(p.builder, hostport) - if err != nil { - cmd.Process.Kill() - err = fmt.Errorf("waitReady: %v", err) - } - } - - p.mu.Lock() - defer p.mu.Unlock() - if err != nil { - log.Println(err) - p.err = err - return - } - - u, err := url.Parse(fmt.Sprintf("http://%v/", hostport)) - if err != nil { - err = fmt.Errorf("parsing hostport: %v", err) - log.Println(err) - p.err = err - return - } - p.proxy = httputil.NewSingleHostReverseProxy(u) - p.side = newSide - p.hostport = hostport - if p.cmd != nil { - p.cmd.Process.Kill() - } - p.cmd = cmd -} - -func newServeMux(p *Proxy) http.Handler { - mux := http.NewServeMux() - mux.Handle("/", httpsOnlyHandler{p}) - mux.HandleFunc("/_ah/health", p.serveHealthCheck) - return mux -} - -func waitReady(b Builder, hostport string) error { - var err error - deadline := time.Now().Add(startTimeout) - for time.Now().Before(deadline) { - if err = b.HealthCheck(hostport); err == nil { - return nil - } - time.Sleep(time.Second) - } - return fmt.Errorf("timed out waiting for process at %v: %v", hostport, err) -} - -func runErr(cmd *exec.Cmd) error { - out, err := cmd.CombinedOutput() - if err != nil { - if len(out) == 0 { - return err - } - return fmt.Errorf("%s\n%v", out, err) - } - return nil -} - -func checkout(repo, hash, path string) error { - // Clone git repo if it doesn't exist. - if _, err := os.Stat(filepath.Join(path, ".git")); os.IsNotExist(err) { - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return fmt.Errorf("mkdir: %v", err) - } - if err := runErr(exec.Command("git", "clone", "--depth", "1", repo, path)); err != nil { - return fmt.Errorf("clone: %v", err) - } - } else if err != nil { - return fmt.Errorf("stat .git: %v", err) - } - - // Pull down changes and update to hash. - cmd := exec.Command("git", "fetch") - cmd.Dir = path - if err := runErr(cmd); err != nil { - return fmt.Errorf("fetch: %v", err) - } - cmd = exec.Command("git", "reset", "--hard", hash) - cmd.Dir = path - if err := runErr(cmd); err != nil { - return fmt.Errorf("reset: %v", err) - } - cmd = exec.Command("git", "clean", "-d", "-f", "-x") - cmd.Dir = path - if err := runErr(cmd); err != nil { - return fmt.Errorf("clean: %v", err) - } - return nil -} - -var timeoutClient = &http.Client{Timeout: 10 * time.Second} - -// gerritMetaMap returns the map from repo name (e.g. "go") to its -// latest master hash. -// The returned map is nil on any transient error. -func gerritMetaMap() map[string]string { - res, err := timeoutClient.Get(metaURL) - if err != nil { - log.Printf("Error getting Gerrit meta map: %v", err) - return nil - } - defer res.Body.Close() - defer io.Copy(ioutil.Discard, res.Body) // ensure EOF for keep-alive - if res.StatusCode != 200 { - return nil - } - var meta map[string]struct { - Branches map[string]string - } - br := bufio.NewReader(res.Body) - // For security reasons or something, this URL starts with ")]}'\n" before - // the JSON object. So ignore that. - // Shawn Pearce says it's guaranteed to always be just one line, ending in '\n'. - for { - b, err := br.ReadByte() - if err != nil { - return nil - } - if b == '\n' { - break - } - } - if err := json.NewDecoder(br).Decode(&meta); err != nil { - log.Printf("JSON decoding error from %v: %s", metaURL, err) - return nil - } - m := map[string]string{} - for repo, v := range meta { - if master, ok := v.Branches["master"]; ok { - m[repo] = master - } - } - return m -} - -func getOK(url string) (body []byte, err error) { - res, err := timeoutClient.Get(url) - if err != nil { - return nil, err - } - body, err = ioutil.ReadAll(res.Body) - res.Body.Close() - if err != nil { - return nil, err - } - if res.StatusCode != http.StatusOK { - return nil, errors.New(res.Status) - } - return body, nil -} - -// httpsOnlyHandler redirects requests to "http://example.com/foo?bar" to -// "https://example.com/foo?bar". It should be used when the server is listening -// for HTTP traffic behind a proxy that terminates TLS traffic, not when the Go -// server is terminating TLS directly. -type httpsOnlyHandler struct { - h http.Handler -} - -// isProxiedReq checks whether the server is running behind a proxy that may be -// terminating TLS. -func isProxiedReq(r *http.Request) bool { - if _, ok := r.Header["X-Appengine-Https"]; ok { - return true - } - if _, ok := r.Header["X-Forwarded-Proto"]; ok { - return true - } - return false -} - -func (h httpsOnlyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("X-Appengine-Https") == "off" || r.Header.Get("X-Forwarded-Proto") == "http" || - (!isProxiedReq(r) && r.TLS == nil) { - r.URL.Scheme = "https" - r.URL.Host = r.Host - http.Redirect(w, r, r.URL.String(), http.StatusFound) - return - } - if r.Header.Get("X-Appengine-Https") == "on" || r.Header.Get("X-Forwarded-Proto") == "https" || - (!isProxiedReq(r) && r.TLS != nil) { - // Only set this header when we're actually in production. - w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload") - } - h.h.ServeHTTP(w, r) -} - -type toLoggerWriter struct{ logger *log.Logger } - -func (w toLoggerWriter) Write(p []byte) (int, error) { - w.logger.Printf("%s", p) - return len(p), nil -} diff --git a/cmd/tip/tip_test.go b/cmd/tip/tip_test.go deleted file mode 100644 index 878954d571..0000000000 --- a/cmd/tip/tip_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package main - -import ( - "net/http/httptest" - "testing" -) - -func TestTipRedirects(t *testing.T) { - mux := newServeMux(&Proxy{builder: &godocBuilder{}}) - req := httptest.NewRequest("GET", "http://example.com/foo?bar=baz", nil) - req.Header.Set("X-Forwarded-Proto", "http") - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - if w.Code != 302 { - t.Errorf("expected Code to be 302, got %d", w.Code) - } - want := "https://example.com/foo?bar=baz" - if loc := w.Header().Get("Location"); loc != want { - t.Errorf("Location header: got %s, want %s", loc, want) - } -}