cmd/gofmt, go/format: refactor common pieces into internal/format

cmd/gofmt and go/format had 3 functions (parse, format and isSpace)
that had to be kept in-sync.

This CL extracts these 3 functions and refactors them into a new
internal/format package.
This CL is just code reorganization with no behavior nor semantic
change.

Change-Id: I593f24e9d3cadbbd9559a67e3b1d2ff190b4fd90
Reviewed-on: https://go-review.googlesource.com/6760
Reviewed-by: Robert Griesemer <gri@golang.org>

src/cmd/gofmt/gofmt.go

@@ -13,6 +13,7 @@ import (
 	"go/printer"
 	"go/scanner"
 	"go/token"
+	"internal/format"
 	"io"
 	"io/ioutil"
 	"os"
@@ -87,7 +88,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error
 		return err
 	}
 
-	file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, stdin)
+	file, sourceAdj, indentAdj, err := format.Parse(fileSet, filename, src, stdin)
 	if err != nil {
 		return err
 	}
@@ -106,7 +107,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error
 		simplify(file)
 	}
 
-	res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth})
+	res, err := format.Format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth})
 	if err != nil {
 		return err
 	}
@@ -234,154 +235,3 @@ func diff(b1, b2 []byte) (data []byte, err error) {
 	return
 
 }
-
-// ----------------------------------------------------------------------------
-// Support functions
-//
-// The functions parse, format, and isSpace below are identical to the
-// respective functions in src/go/format/format.go - keep them in sync!
-//
-// TODO(gri) Factor out this functionality, eventually.
-
-// parse parses src, which was read from the named file,
-// as a Go source file, declaration, or statement list.
-func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) (
-	file *ast.File,
-	sourceAdj func(src []byte, indent int) []byte,
-	indentAdj int,
-	err error,
-) {
-	// Try as whole source file.
-	file, err = parser.ParseFile(fset, filename, src, parserMode)
-	// If there's no error, return.  If the error is that the source file didn't begin with a
-	// package line and source fragments are ok, fall through to
-	// try as a source fragment.  Stop and return on any other error.
-	if err == nil || !fragmentOk || !strings.Contains(err.Error(), "expected 'package'") {
-		return
-	}
-
-	// If this is a declaration list, make it a source file
-	// by inserting a package clause.
-	// Insert using a ;, not a newline, so that the line numbers
-	// in psrc match the ones in src.
-	psrc := append([]byte("package p;"), src...)
-	file, err = parser.ParseFile(fset, filename, psrc, parserMode)
-	if err == nil {
-		sourceAdj = func(src []byte, indent int) []byte {
-			// Remove the package clause.
-			// Gofmt has turned the ; into a \n.
-			src = src[indent+len("package p\n"):]
-			return bytes.TrimSpace(src)
-		}
-		return
-	}
-	// If the error is that the source file didn't begin with a
-	// declaration, fall through to try as a statement list.
-	// Stop and return on any other error.
-	if !strings.Contains(err.Error(), "expected declaration") {
-		return
-	}
-
-	// If this is a statement list, make it a source file
-	// by inserting a package clause and turning the list
-	// into a function body.  This handles expressions too.
-	// Insert using a ;, not a newline, so that the line numbers
-	// in fsrc match the ones in src.
-	fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '}')
-	file, err = parser.ParseFile(fset, filename, fsrc, parserMode)
-	if err == nil {
-		sourceAdj = func(src []byte, indent int) []byte {
-			// Cap adjusted indent to zero.
-			if indent < 0 {
-				indent = 0
-			}
-			// Remove the wrapping.
-			// Gofmt has turned the ; into a \n\n.
-			// There will be two non-blank lines with indent, hence 2*indent.
-			src = src[2*indent+len("package p\n\nfunc _() {"):]
-			src = src[:len(src)-(indent+len("\n}\n"))]
-			return bytes.TrimSpace(src)
-		}
-		// Gofmt has also indented the function body one level.
-		// Adjust that with indentAdj.
-		indentAdj = -1
-	}
-
-	// Succeeded, or out of options.
-	return
-}
-
-// format formats the given package file originally obtained from src
-// and adjusts the result based on the original source via sourceAdj
-// and indentAdj.
-func format(
-	fset *token.FileSet,
-	file *ast.File,
-	sourceAdj func(src []byte, indent int) []byte,
-	indentAdj int,
-	src []byte,
-	cfg printer.Config,
-) ([]byte, error) {
-	if sourceAdj == nil {
-		// Complete source file.
-		var buf bytes.Buffer
-		err := cfg.Fprint(&buf, fset, file)
-		if err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	}
-
-	// Partial source file.
-	// Determine and prepend leading space.
-	i, j := 0, 0
-	for j < len(src) && isSpace(src[j]) {
-		if src[j] == '\n' {
-			i = j + 1 // byte offset of last line in leading space
-		}
-		j++
-	}
-	var res []byte
-	res = append(res, src[:i]...)
-
-	// Determine and prepend indentation of first code line.
-	// Spaces are ignored unless there are no tabs,
-	// in which case spaces count as one tab.
-	indent := 0
-	hasSpace := false
-	for _, b := range src[i:j] {
-		switch b {
-		case ' ':
-			hasSpace = true
-		case '\t':
-			indent++
-		}
-	}
-	if indent == 0 && hasSpace {
-		indent = 1
-	}
-	for i := 0; i < indent; i++ {
-		res = append(res, '\t')
-	}
-
-	// Format the source.
-	// Write it without any leading and trailing space.
-	cfg.Indent = indent + indentAdj
-	var buf bytes.Buffer
-	err := cfg.Fprint(&buf, fset, file)
-	if err != nil {
-		return nil, err
-	}
-	res = append(res, sourceAdj(buf.Bytes(), cfg.Indent)...)
-
-	// Determine and append trailing space.
-	i = len(src)
-	for i > 0 && isSpace(src[i-1]) {
-		i--
-	}
-	return append(res, src[i:]...), nil
-}
-
-func isSpace(b byte) bool {
-	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
-}

src/cmd/gofmt/long_test.go

@@ -15,6 +15,7 @@ import (
 	"go/ast"
 	"go/printer"
 	"go/token"
+	"internal/format"
 	"io"
 	"os"
 	"path/filepath"
@@ -32,7 +33,7 @@ var (
 )
 
 func gofmt(fset *token.FileSet, filename string, src *bytes.Buffer) error {
-	f, _, _, err := parse(fset, filename, src.Bytes(), false)
+	f, _, _, err := format.Parse(fset, filename, src.Bytes(), false)
 	if err != nil {
 		return err
 	}
@@ -60,7 +61,7 @@ func testFile(t *testing.T, b1, b2 *bytes.Buffer, filename string) {
 
 	// exclude files w/ syntax errors (typically test cases)
 	fset := token.NewFileSet()
-	if _, _, _, err = parse(fset, filename, b1.Bytes(), false); err != nil {
+	if _, _, _, err = format.Parse(fset, filename, b1.Bytes(), false); err != nil {
 		if *verbose {
 			fmt.Fprintf(os.Stderr, "ignoring %s\n", err)
 		}

src/go/format/format.go

@@ -12,8 +12,8 @@ import (
 	"go/parser"
 	"go/printer"
 	"go/token"
+	"internal/format"
 	"io"
-	"strings"
 )
 
 var config = printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}
@@ -82,7 +82,7 @@ func Node(dst io.Writer, fset *token.FileSet, node interface{}) error {
 //
 func Source(src []byte) ([]byte, error) {
 	fset := token.NewFileSet()
-	file, sourceAdj, indentAdj, err := parse(fset, "", src, true)
+	file, sourceAdj, indentAdj, err := format.Parse(fset, "", src, true)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +93,7 @@ func Source(src []byte) ([]byte, error) {
 		ast.SortImports(fset, file)
 	}
 
-	return format(fset, file, sourceAdj, indentAdj, src, config)
+	return format.Format(fset, file, sourceAdj, indentAdj, src, config)
 }
 
 func hasUnsortedImports(file *ast.File) bool {
@@ -113,154 +113,3 @@ func hasUnsortedImports(file *ast.File) bool {
 	}
 	return false
 }
-
-// ----------------------------------------------------------------------------
-// Support functions
-//
-// The functions parse, format, and isSpace below are identical to the
-// respective functions in cmd/gofmt/gofmt.go - keep them in sync!
-//
-// TODO(gri) Factor out this functionality, eventually.
-
-// parse parses src, which was read from the named file,
-// as a Go source file, declaration, or statement list.
-func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) (
-	file *ast.File,
-	sourceAdj func(src []byte, indent int) []byte,
-	indentAdj int,
-	err error,
-) {
-	// Try as whole source file.
-	file, err = parser.ParseFile(fset, filename, src, parserMode)
-	// If there's no error, return.  If the error is that the source file didn't begin with a
-	// package line and source fragments are ok, fall through to
-	// try as a source fragment.  Stop and return on any other error.
-	if err == nil || !fragmentOk || !strings.Contains(err.Error(), "expected 'package'") {
-		return
-	}
-
-	// If this is a declaration list, make it a source file
-	// by inserting a package clause.
-	// Insert using a ;, not a newline, so that the line numbers
-	// in psrc match the ones in src.
-	psrc := append([]byte("package p;"), src...)
-	file, err = parser.ParseFile(fset, filename, psrc, parserMode)
-	if err == nil {
-		sourceAdj = func(src []byte, indent int) []byte {
-			// Remove the package clause.
-			// Gofmt has turned the ; into a \n.
-			src = src[indent+len("package p\n"):]
-			return bytes.TrimSpace(src)
-		}
-		return
-	}
-	// If the error is that the source file didn't begin with a
-	// declaration, fall through to try as a statement list.
-	// Stop and return on any other error.
-	if !strings.Contains(err.Error(), "expected declaration") {
-		return
-	}
-
-	// If this is a statement list, make it a source file
-	// by inserting a package clause and turning the list
-	// into a function body.  This handles expressions too.
-	// Insert using a ;, not a newline, so that the line numbers
-	// in fsrc match the ones in src.
-	fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '}')
-	file, err = parser.ParseFile(fset, filename, fsrc, parserMode)
-	if err == nil {
-		sourceAdj = func(src []byte, indent int) []byte {
-			// Cap adjusted indent to zero.
-			if indent < 0 {
-				indent = 0
-			}
-			// Remove the wrapping.
-			// Gofmt has turned the ; into a \n\n.
-			// There will be two non-blank lines with indent, hence 2*indent.
-			src = src[2*indent+len("package p\n\nfunc _() {"):]
-			src = src[:len(src)-(indent+len("\n}\n"))]
-			return bytes.TrimSpace(src)
-		}
-		// Gofmt has also indented the function body one level.
-		// Adjust that with indentAdj.
-		indentAdj = -1
-	}
-
-	// Succeeded, or out of options.
-	return
-}
-
-// format formats the given package file originally obtained from src
-// and adjusts the result based on the original source via sourceAdj
-// and indentAdj.
-func format(
-	fset *token.FileSet,
-	file *ast.File,
-	sourceAdj func(src []byte, indent int) []byte,
-	indentAdj int,
-	src []byte,
-	cfg printer.Config,
-) ([]byte, error) {
-	if sourceAdj == nil {
-		// Complete source file.
-		var buf bytes.Buffer
-		err := cfg.Fprint(&buf, fset, file)
-		if err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	}
-
-	// Partial source file.
-	// Determine and prepend leading space.
-	i, j := 0, 0
-	for j < len(src) && isSpace(src[j]) {
-		if src[j] == '\n' {
-			i = j + 1 // byte offset of last line in leading space
-		}
-		j++
-	}
-	var res []byte
-	res = append(res, src[:i]...)
-
-	// Determine and prepend indentation of first code line.
-	// Spaces are ignored unless there are no tabs,
-	// in which case spaces count as one tab.
-	indent := 0
-	hasSpace := false
-	for _, b := range src[i:j] {
-		switch b {
-		case ' ':
-			hasSpace = true
-		case '\t':
-			indent++
-		}
-	}
-	if indent == 0 && hasSpace {
-		indent = 1
-	}
-	for i := 0; i < indent; i++ {
-		res = append(res, '\t')
-	}
-
-	// Format the source.
-	// Write it without any leading and trailing space.
-	cfg.Indent = indent + indentAdj
-	var buf bytes.Buffer
-	err := cfg.Fprint(&buf, fset, file)
-	if err != nil {
-		return nil, err
-	}
-	res = append(res, sourceAdj(buf.Bytes(), cfg.Indent)...)
-
-	// Determine and append trailing space.
-	i = len(src)
-	for i > 0 && isSpace(src[i-1]) {
-		i--
-	}
-	return append(res, src[i:]...), nil
-}
-
-func isSpace(b byte) bool {
-	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
-}

src/internal/format/format.go

@@ -0,0 +1,161 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package format
+
+import (
+	"bytes"
+	"go/ast"
+	"go/parser"
+	"go/printer"
+	"go/token"
+	"strings"
+)
+
+const parserMode = parser.ParseComments
+
+// Parse parses src, which was read from the named file,
+// as a Go source file, declaration, or statement list.
+func Parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) (
+	file *ast.File,
+	sourceAdj func(src []byte, indent int) []byte,
+	indentAdj int,
+	err error,
+) {
+	// Try as whole source file.
+	file, err = parser.ParseFile(fset, filename, src, parserMode)
+	// If there's no error, return.  If the error is that the source file didn't begin with a
+	// package line and source fragments are ok, fall through to
+	// try as a source fragment.  Stop and return on any other error.
+	if err == nil || !fragmentOk || !strings.Contains(err.Error(), "expected 'package'") {
+		return
+	}
+
+	// If this is a declaration list, make it a source file
+	// by inserting a package clause.
+	// Insert using a ;, not a newline, so that the line numbers
+	// in psrc match the ones in src.
+	psrc := append([]byte("package p;"), src...)
+	file, err = parser.ParseFile(fset, filename, psrc, parserMode)
+	if err == nil {
+		sourceAdj = func(src []byte, indent int) []byte {
+			// Remove the package clause.
+			// Gofmt has turned the ; into a \n.
+			src = src[indent+len("package p\n"):]
+			return bytes.TrimSpace(src)
+		}
+		return
+	}
+	// If the error is that the source file didn't begin with a
+	// declaration, fall through to try as a statement list.
+	// Stop and return on any other error.
+	if !strings.Contains(err.Error(), "expected declaration") {
+		return
+	}
+
+	// If this is a statement list, make it a source file
+	// by inserting a package clause and turning the list
+	// into a function body.  This handles expressions too.
+	// Insert using a ;, not a newline, so that the line numbers
+	// in fsrc match the ones in src.
+	fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '}')
+	file, err = parser.ParseFile(fset, filename, fsrc, parserMode)
+	if err == nil {
+		sourceAdj = func(src []byte, indent int) []byte {
+			// Cap adjusted indent to zero.
+			if indent < 0 {
+				indent = 0
+			}
+			// Remove the wrapping.
+			// Gofmt has turned the ; into a \n\n.
+			// There will be two non-blank lines with indent, hence 2*indent.
+			src = src[2*indent+len("package p\n\nfunc _() {"):]
+			src = src[:len(src)-(indent+len("\n}\n"))]
+			return bytes.TrimSpace(src)
+		}
+		// Gofmt has also indented the function body one level.
+		// Adjust that with indentAdj.
+		indentAdj = -1
+	}
+
+	// Succeeded, or out of options.
+	return
+}
+
+// Format formats the given package file originally obtained from src
+// and adjusts the result based on the original source via sourceAdj
+// and indentAdj.
+func Format(
+	fset *token.FileSet,
+	file *ast.File,
+	sourceAdj func(src []byte, indent int) []byte,
+	indentAdj int,
+	src []byte,
+	cfg printer.Config,
+) ([]byte, error) {
+	if sourceAdj == nil {
+		// Complete source file.
+		var buf bytes.Buffer
+		err := cfg.Fprint(&buf, fset, file)
+		if err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
+
+	// Partial source file.
+	// Determine and prepend leading space.
+	i, j := 0, 0
+	for j < len(src) && IsSpace(src[j]) {
+		if src[j] == '\n' {
+			i = j + 1 // byte offset of last line in leading space
+		}
+		j++
+	}
+	var res []byte
+	res = append(res, src[:i]...)
+
+	// Determine and prepend indentation of first code line.
+	// Spaces are ignored unless there are no tabs,
+	// in which case spaces count as one tab.
+	indent := 0
+	hasSpace := false
+	for _, b := range src[i:j] {
+		switch b {
+		case ' ':
+			hasSpace = true
+		case '\t':
+			indent++
+		}
+	}
+	if indent == 0 && hasSpace {
+		indent = 1
+	}
+	for i := 0; i < indent; i++ {
+		res = append(res, '\t')
+	}
+
+	// Format the source.
+	// Write it without any leading and trailing space.
+	cfg.Indent = indent + indentAdj
+	var buf bytes.Buffer
+	err := cfg.Fprint(&buf, fset, file)
+	if err != nil {
+		return nil, err
+	}
+	res = append(res, sourceAdj(buf.Bytes(), cfg.Indent)...)
+
+	// Determine and append trailing space.
+	i = len(src)
+	for i > 0 && IsSpace(src[i-1]) {
+		i--
+	}
+	return append(res, src[i:]...), nil
+}
+
+// IsSpace reports whether the byte is a space character.
+// IsSpace defines a space as being among the following bytes: ' ', '\t', '\n' and '\r'.
+func IsSpace(b byte) bool {
+	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
+}