mirror of
https://github.com/golang/go
synced 2024-11-23 17:10:03 -07:00
misc/builder: support for uploading tarballs to googlecode
R=rsc CC=golang-dev https://golang.org/cl/1784042
This commit is contained in:
Andrew Gerrand
parent
b57ffae094
commit
3402c5e8fe
35
README
35
README
@ -9,3 +9,38 @@ and then visiting http://localhost:6060/doc/install.html.
|
||||
|
||||
Unless otherwise noted, the Go source files are distributed
|
||||
under the BSD-style license found in the LICENSE file.
|
||||
|
||||
--
|
||||
|
||||
Binary Distribution Notes
|
||||
|
||||
If you have just untarred a binary Go distribution, then there are some
|
||||
environment variables you'll need to set in your .profile (or equivalent):
|
||||
|
||||
GOOS should be set to your operating system (eg, linux),
|
||||
GOARCH should be your processor architecture (eg, amd64),
|
||||
GOROOT should be the directory you extracted the tarball to,
|
||||
GOBIN should point to $GOROOT/bin.
|
||||
|
||||
For example, if you downloaded the tarball
|
||||
|
||||
go.release.YYYY-MM-DD.linux-amd64.tar.gz
|
||||
|
||||
and extracted it to
|
||||
|
||||
/home/username/go
|
||||
|
||||
you would set the following variables:
|
||||
|
||||
export GOOS=linux
|
||||
export GOARCH=amd64
|
||||
export GOROOT=/home/username/go
|
||||
export GOBIN=$GOROOT/bin
|
||||
|
||||
See doc/install.html for more detail about these flags.
|
||||
|
||||
Additionally, $GOROOT should be in your $PATH:
|
||||
|
||||
export PATH=PATH:$GOROOT
|
||||
|
||||
|
||||
|
||||
@ -24,11 +24,19 @@ export GOARCH=XXX
|
||||
export GOOS=XXX
|
||||
export GOBIN=/gobuild/bin
|
||||
export PATH=$PATH:/gobuild/bin
|
||||
export BUILDER=XXX
|
||||
export BUILDER=$GOOS-$GOARCH
|
||||
export BUILDHOST=godashboard.appspot.com
|
||||
|
||||
* Write the key ~gobuild/.gobuildkey (you need to get it from someone who knows
|
||||
the key)
|
||||
* Write the key ~gobuild/.gobuildkey
|
||||
You need to get it from someone who knows the key.
|
||||
You may also use a filename of the form .gobuildkey-$BUILDER if you
|
||||
wish to run builders for multiple targets.
|
||||
|
||||
* Append your username and password googlecode.com credentials from
|
||||
https://code.google.com/hosting/settings
|
||||
to the buildkey file in the format "Username\nPassword\n".
|
||||
(This is for uploading tarballs to the project downloads section,
|
||||
and is an optional step.)
|
||||
|
||||
* sudo apt-get install bison gcc libc6-dev ed make
|
||||
* cd ~gobuild
|
||||
|
||||
@ -18,6 +18,8 @@ buildhost = ''
|
||||
buildport = -1
|
||||
buildkey = ''
|
||||
|
||||
upload_project = "go"
|
||||
|
||||
def main(args):
|
||||
global buildport, buildhost, buildkey
|
||||
|
||||
@ -35,14 +37,23 @@ def main(args):
|
||||
buildport = int(os.environ['BUILDPORT'])
|
||||
|
||||
try:
|
||||
buildkey = file('%s/.gobuildkey-%s' % (os.environ['HOME'], os.environ['BUILDER']), 'r').read().strip()
|
||||
buildkeyfile = file('%s/.gobuildkey-%s' % (os.environ['HOME'], os.environ['BUILDER']), 'r')
|
||||
buildkey = buildkeyfile.readline().strip()
|
||||
except IOError:
|
||||
try:
|
||||
buildkey = file('%s/.gobuildkey' % os.environ['HOME'], 'r').read().strip()
|
||||
buildkeyfile = file('%s/.gobuildkey' % os.environ['HOME'], 'r')
|
||||
buildkey = buildkeyfile.readline().strip()
|
||||
except IOError:
|
||||
print >>sys.stderr, "Need key in ~/.gobuildkey-%s or ~/.gobuildkey" % os.environ['BUILDER']
|
||||
return
|
||||
|
||||
# get upload credentials
|
||||
try:
|
||||
username = buildkeyfile.readline().strip()
|
||||
password = buildkeyfile.readline().strip()
|
||||
except:
|
||||
username, password = None, None
|
||||
|
||||
if args[1] == 'init':
|
||||
return doInit(args)
|
||||
elif args[1] == 'hwget':
|
||||
@ -55,6 +66,8 @@ def main(args):
|
||||
return doRecord(args)
|
||||
elif args[1] == 'benchmarks':
|
||||
return doBenchmarks(args)
|
||||
elif args[1] == 'upload':
|
||||
return doUpload(args, username, password)
|
||||
else:
|
||||
return usage(args[0])
|
||||
|
||||
@ -68,6 +81,7 @@ Commands:
|
||||
next <builder>: get the next revision number to by built by the given builder
|
||||
record <builder> <rev> <ok|log file>: record a build result
|
||||
benchmarks <builder> <rev> <log file>: record benchmark numbers
|
||||
upload <builder> <summary> <tar file>: upload tarball to googlecode
|
||||
''' % name)
|
||||
return 1
|
||||
|
||||
@ -165,6 +179,29 @@ def doBenchmarks(args):
|
||||
e.append(b)
|
||||
return command('benchmarks', {'node': c.node, 'builder': builder, 'benchmarkdata': binascii.b2a_base64(''.join(e))})
|
||||
|
||||
def doUpload(args, username, password):
|
||||
# fail gracefully if no username or password set
|
||||
if not username or not password:
|
||||
return
|
||||
|
||||
if len(args) != 5:
|
||||
return usage(args[0])
|
||||
builder = args[2]
|
||||
summary = args[3]
|
||||
filename = args[4]
|
||||
|
||||
from googlecode_upload import upload
|
||||
code, msg, url = upload(
|
||||
filename, # filename
|
||||
upload_project, # 'go'
|
||||
username,
|
||||
password,
|
||||
summary,
|
||||
builder.split('-'), # labels
|
||||
)
|
||||
if code != 201:
|
||||
raise Failed('Upload returned code %s msg "%s".' % (code, msg))
|
||||
|
||||
def encodeMultipartFormdata(fields, files):
|
||||
"""fields is a sequence of (name, value) elements for regular form fields.
|
||||
files is a sequence of (name, filename, value) elements for data to be uploaded as files"""
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2010 The Go Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style
|
||||
# license that can be found in the LICENSE file.
|
||||
@ -48,7 +48,7 @@ fi
|
||||
mkdir -p $GOROOT/bin
|
||||
|
||||
cd $GOROOT/..
|
||||
cp go/misc/dashboard/builder.sh go/misc/dashboard/buildcontrol.py .
|
||||
cp go/misc/dashboard/{builder.sh,buildcontrol.py,googlecode_upload.py} .
|
||||
chmod a+x builder.sh buildcontrol.py
|
||||
cd go
|
||||
../buildcontrol.py next $BUILDER
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2009 The Go Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style
|
||||
@ -31,6 +31,7 @@ fi
|
||||
|
||||
export PATH=$PATH:`pwd`/candidate/bin
|
||||
export GOBIN=`pwd`/candidate/bin
|
||||
export BAKED_GOROOT=/usr/local/go
|
||||
|
||||
while true ; do
|
||||
cd go || fatal "Cannot cd into 'go'"
|
||||
@ -72,6 +73,18 @@ while true ; do
|
||||
python ../../../buildcontrol.py benchmarks $BUILDER $rev ../../benchmarks || fatal "Cannot record benchmarks"
|
||||
cd .. || fatal "failed to cd out of pkg"
|
||||
fi
|
||||
# check if we're at a release (via the hg summary)
|
||||
# if so, package the tar.gz and upload to googlecode
|
||||
SUMMARY=$(hg log -l 1 | grep summary\: | awk '{print $2}')
|
||||
if [[ "x${SUMMARY:0:7}" == "xrelease" ]]; then
|
||||
echo "Uploading binary to googlecode"
|
||||
TARBALL="go.$SUMMARY.$BUILDER.tar.gz"
|
||||
./clean.bash --nopkg
|
||||
cd .. || fatal "Cannot cd up"
|
||||
tar czf ../$TARBALL . || fatal "Cannot create tarball"
|
||||
../buildcontrol.py upload $BUILDER $SUMMARY ../$TARBALL || fatal "Cannot upload tarball"
|
||||
cd src || fatal "Cannot cd src"
|
||||
fi
|
||||
fi
|
||||
cd ../.. || fatal "Cannot cd up"
|
||||
sleep 10
|
||||
|
||||
248
misc/dashboard/googlecode_upload.py
Normal file
248
misc/dashboard/googlecode_upload.py
Normal file
@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2006, 2007 Google Inc. All Rights Reserved.
|
||||
# Author: danderson@google.com (David Anderson)
|
||||
#
|
||||
# Script for uploading files to a Google Code project.
|
||||
#
|
||||
# This is intended to be both a useful script for people who want to
|
||||
# streamline project uploads and a reference implementation for
|
||||
# uploading files to Google Code projects.
|
||||
#
|
||||
# To upload a file to Google Code, you need to provide a path to the
|
||||
# file on your local machine, a small summary of what the file is, a
|
||||
# project name, and a valid account that is a member or owner of that
|
||||
# project. You can optionally provide a list of labels that apply to
|
||||
# the file. The file will be uploaded under the same name that it has
|
||||
# in your local filesystem (that is, the "basename" or last path
|
||||
# component). Run the script with '--help' to get the exact syntax
|
||||
# and available options.
|
||||
#
|
||||
# Note that the upload script requests that you enter your
|
||||
# googlecode.com password. This is NOT your Gmail account password!
|
||||
# This is the password you use on googlecode.com for committing to
|
||||
# Subversion and uploading files. You can find your password by going
|
||||
# to http://code.google.com/hosting/settings when logged in with your
|
||||
# Gmail account. If you have already committed to your project's
|
||||
# Subversion repository, the script will automatically retrieve your
|
||||
# credentials from there (unless disabled, see the output of '--help'
|
||||
# for details).
|
||||
#
|
||||
# If you are looking at this script as a reference for implementing
|
||||
# your own Google Code file uploader, then you should take a look at
|
||||
# the upload() function, which is the meat of the uploader. You
|
||||
# basically need to build a multipart/form-data POST request with the
|
||||
# right fields and send it to https://PROJECT.googlecode.com/files .
|
||||
# Authenticate the request using HTTP Basic authentication, as is
|
||||
# shown below.
|
||||
#
|
||||
# Licensed under the terms of the Apache Software License 2.0:
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Questions, comments, feature requests and patches are most welcome.
|
||||
# Please direct all of these to the Google Code users group:
|
||||
# http://groups.google.com/group/google-code-hosting
|
||||
|
||||
"""Google Code file uploader script.
|
||||
"""
|
||||
|
||||
__author__ = 'danderson@google.com (David Anderson)'
|
||||
|
||||
import httplib
|
||||
import os.path
|
||||
import optparse
|
||||
import getpass
|
||||
import base64
|
||||
import sys
|
||||
|
||||
|
||||
def upload(file, project_name, user_name, password, summary, labels=None):
|
||||
"""Upload a file to a Google Code project's file server.
|
||||
|
||||
Args:
|
||||
file: The local path to the file.
|
||||
project_name: The name of your project on Google Code.
|
||||
user_name: Your Google account name.
|
||||
password: The googlecode.com password for your account.
|
||||
Note that this is NOT your global Google Account password!
|
||||
summary: A small description for the file.
|
||||
labels: an optional list of label strings with which to tag the file.
|
||||
|
||||
Returns: a tuple:
|
||||
http_status: 201 if the upload succeeded, something else if an
|
||||
error occured.
|
||||
http_reason: The human-readable string associated with http_status
|
||||
file_url: If the upload succeeded, the URL of the file on Google
|
||||
Code, None otherwise.
|
||||
"""
|
||||
# The login is the user part of user@gmail.com. If the login provided
|
||||
# is in the full user@domain form, strip it down.
|
||||
if user_name.endswith('@gmail.com'):
|
||||
user_name = user_name[:user_name.index('@gmail.com')]
|
||||
|
||||
form_fields = [('summary', summary)]
|
||||
if labels is not None:
|
||||
form_fields.extend([('label', l.strip()) for l in labels])
|
||||
|
||||
content_type, body = encode_upload_request(form_fields, file)
|
||||
|
||||
upload_host = '%s.googlecode.com' % project_name
|
||||
upload_uri = '/files'
|
||||
auth_token = base64.b64encode('%s:%s'% (user_name, password))
|
||||
headers = {
|
||||
'Authorization': 'Basic %s' % auth_token,
|
||||
'User-Agent': 'Googlecode.com uploader v0.9.4',
|
||||
'Content-Type': content_type,
|
||||
}
|
||||
|
||||
server = httplib.HTTPSConnection(upload_host)
|
||||
server.request('POST', upload_uri, body, headers)
|
||||
resp = server.getresponse()
|
||||
server.close()
|
||||
|
||||
if resp.status == 201:
|
||||
location = resp.getheader('Location', None)
|
||||
else:
|
||||
location = None
|
||||
return resp.status, resp.reason, location
|
||||
|
||||
|
||||
def encode_upload_request(fields, file_path):
|
||||
"""Encode the given fields and file into a multipart form body.
|
||||
|
||||
fields is a sequence of (name, value) pairs. file is the path of
|
||||
the file to upload. The file will be uploaded to Google Code with
|
||||
the same file name.
|
||||
|
||||
Returns: (content_type, body) ready for httplib.HTTP instance
|
||||
"""
|
||||
BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
|
||||
CRLF = '\r\n'
|
||||
|
||||
body = []
|
||||
|
||||
# Add the metadata about the upload first
|
||||
for key, value in fields:
|
||||
body.extend(
|
||||
['--' + BOUNDARY,
|
||||
'Content-Disposition: form-data; name="%s"' % key,
|
||||
'',
|
||||
value,
|
||||
])
|
||||
|
||||
# Now add the file itself
|
||||
file_name = os.path.basename(file_path)
|
||||
f = open(file_path, 'rb')
|
||||
file_content = f.read()
|
||||
f.close()
|
||||
|
||||
body.extend(
|
||||
['--' + BOUNDARY,
|
||||
'Content-Disposition: form-data; name="filename"; filename="%s"'
|
||||
% file_name,
|
||||
# The upload server determines the mime-type, no need to set it.
|
||||
'Content-Type: application/octet-stream',
|
||||
'',
|
||||
file_content,
|
||||
])
|
||||
|
||||
# Finalize the form body
|
||||
body.extend(['--' + BOUNDARY + '--', ''])
|
||||
|
||||
return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body)
|
||||
|
||||
|
||||
def upload_find_auth(file_path, project_name, summary, labels=None,
|
||||
user_name=None, password=None, tries=3):
|
||||
"""Find credentials and upload a file to a Google Code project's file server.
|
||||
|
||||
file_path, project_name, summary, and labels are passed as-is to upload.
|
||||
|
||||
Args:
|
||||
file_path: The local path to the file.
|
||||
project_name: The name of your project on Google Code.
|
||||
summary: A small description for the file.
|
||||
labels: an optional list of label strings with which to tag the file.
|
||||
config_dir: Path to Subversion configuration directory, 'none', or None.
|
||||
user_name: Your Google account name.
|
||||
tries: How many attempts to make.
|
||||
"""
|
||||
|
||||
while tries > 0:
|
||||
if user_name is None:
|
||||
# Read username if not specified or loaded from svn config, or on
|
||||
# subsequent tries.
|
||||
sys.stdout.write('Please enter your googlecode.com username: ')
|
||||
sys.stdout.flush()
|
||||
user_name = sys.stdin.readline().rstrip()
|
||||
if password is None:
|
||||
# Read password if not loaded from svn config, or on subsequent tries.
|
||||
print 'Please enter your googlecode.com password.'
|
||||
print '** Note that this is NOT your Gmail account password! **'
|
||||
print 'It is the password you use to access Subversion repositories,'
|
||||
print 'and can be found here: http://code.google.com/hosting/settings'
|
||||
password = getpass.getpass()
|
||||
|
||||
status, reason, url = upload(file_path, project_name, user_name, password,
|
||||
summary, labels)
|
||||
# Returns 403 Forbidden instead of 401 Unauthorized for bad
|
||||
# credentials as of 2007-07-17.
|
||||
if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]:
|
||||
# Rest for another try.
|
||||
user_name = password = None
|
||||
tries = tries - 1
|
||||
else:
|
||||
# We're done.
|
||||
break
|
||||
|
||||
return status, reason, url
|
||||
|
||||
|
||||
def main():
|
||||
parser = optparse.OptionParser(usage='googlecode-upload.py -s SUMMARY '
|
||||
'-p PROJECT [options] FILE')
|
||||
parser.add_option('-s', '--summary', dest='summary',
|
||||
help='Short description of the file')
|
||||
parser.add_option('-p', '--project', dest='project',
|
||||
help='Google Code project name')
|
||||
parser.add_option('-u', '--user', dest='user',
|
||||
help='Your Google Code username')
|
||||
parser.add_option('-w', '--password', dest='password',
|
||||
help='Your Google Code password')
|
||||
parser.add_option('-l', '--labels', dest='labels',
|
||||
help='An optional list of comma-separated labels to attach '
|
||||
'to the file')
|
||||
|
||||
options, args = parser.parse_args()
|
||||
|
||||
if not options.summary:
|
||||
parser.error('File summary is missing.')
|
||||
elif not options.project:
|
||||
parser.error('Project name is missing.')
|
||||
elif len(args) < 1:
|
||||
parser.error('File to upload not provided.')
|
||||
elif len(args) > 1:
|
||||
parser.error('Only one file may be specified.')
|
||||
|
||||
file_path = args[0]
|
||||
|
||||
if options.labels:
|
||||
labels = options.labels.split(',')
|
||||
else:
|
||||
labels = None
|
||||
|
||||
status, reason, url = upload_find_auth(file_path, options.project,
|
||||
options.summary, labels,
|
||||
options.user, options.password)
|
||||
if url:
|
||||
print 'The file was uploaded successfully.'
|
||||
print 'URL: %s' % url
|
||||
return 0
|
||||
else:
|
||||
print 'An error occurred. Your file was not uploaded.'
|
||||
print 'Google Code upload server said: %s (%s)' % (reason, status)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -20,7 +20,9 @@ fi
|
||||
|
||||
GOBIN="${GOBIN:-$HOME/bin}"
|
||||
|
||||
rm -rf "$GOROOT"/pkg/${GOOS}_$GOARCH
|
||||
if [ "$1" != "--nopkg" ]; then
|
||||
rm -rf "$GOROOT"/pkg/${GOOS}_$GOARCH
|
||||
fi
|
||||
rm -f "$GOROOT"/lib/*.a
|
||||
for i in lib9 libbio libcgo libmach cmd pkg \
|
||||
../misc/cgo/gmp ../misc/cgo/stdio \
|
||||
|
||||
@ -104,6 +104,8 @@ HFILES=\
|
||||
"$(GOROOT)"/include/u.h\
|
||||
"$(GOROOT)"/include/libc.h\
|
||||
|
||||
BAKED_GOROOT?=$(GOROOT)
|
||||
|
||||
install: $(LIB)
|
||||
cp $(LIB) "$(GOROOT)/lib"
|
||||
|
||||
@ -117,7 +119,7 @@ $(LIB): $(OFILES)
|
||||
$(CC) -c $(CFLAGS) $<
|
||||
|
||||
goos.$O: goos.c
|
||||
$(CC) -c $(CFLAGS) -DGOOS='"$(GOOS)"' -DGOARCH='"$(GOARCH)"' -DGOROOT='"$(GOROOT)"' -DGOVERSION='"'"$$(../version.bash)"'"' $<
|
||||
$(CC) -c $(CFLAGS) -DGOOS='"$(GOOS)"' -DGOARCH='"$(GOARCH)"' -DGOROOT='"$(BAKED_GOROOT)"' -DGOVERSION='"'"$$(../version.bash)"'"' $<
|
||||
|
||||
clean:
|
||||
rm -f *.$O *.6 6.out $(LIB)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user