From 1887907fee0b51502a3ef10a63a96543d5dd3e23 Mon Sep 17 00:00:00 2001
From: Nigel Tao <nigeltao@golang.org>
Date: Sun, 16 Oct 2011 20:50:11 +1100
Subject: [PATCH] html: tokenize "a < b" as one whole text token.

R=andybalholm
CC=golang-dev
https://golang.org/cl/5284042
---
 src/pkg/html/token.go      |  47 ++++++++++++-----
 src/pkg/html/token_test.go | 104 +++++++++++++++++++++++++++++++++----
 2 files changed, 128 insertions(+), 23 deletions(-)

diff --git a/src/pkg/html/token.go b/src/pkg/html/token.go
index a02b968dc28..2105cc6f561 100644
--- a/src/pkg/html/token.go
+++ b/src/pkg/html/token.go
@@ -379,15 +379,16 @@ func (z *Tokenizer) nextTag() {
 		z.nextBogusComment()
 		return
 	default:
-		z.tt, z.err = ErrorToken, os.NewError("html: TODO: handle malformed tags")
+		z.nextText()
 		return
 	}
 	// Read the tag name and attribute key/value pairs.
 	z.readTagName()
+	if z.skipWhiteSpace(); z.err != nil {
+		z.tt = ErrorToken
+		return
+	}
 	for {
-		if z.skipWhiteSpace(); z.err != nil {
-			break
-		}
 		c := z.readByte()
 		if z.err != nil || c == '>' {
 			break
@@ -399,6 +400,9 @@ func (z *Tokenizer) nextTag() {
 		if z.pendingAttr[0].start != z.pendingAttr[0].end {
 			z.attr = append(z.attr, z.pendingAttr)
 		}
+		if z.skipWhiteSpace(); z.err != nil {
+			break
+		}
 	}
 	// Check for a self-closing token.
 	if z.err == nil && z.tt == StartTagToken && z.buf[z.raw.end-2] == '/' {
@@ -510,21 +514,40 @@ func (z *Tokenizer) readTagAttrVal() {
 	}
 }
 
-// nextText reads all text up until an '<'.
-// Pre-condition: z.tt == TextToken && z.err == nil && z.raw.start + 1 <= z.raw.end.
+// nextText reads all text up until a start tag "<a", end tag "</a", comment
+// "<!" or XML processing instruction "<?".
+// Pre-condition: z.tt == TextToken && z.err == nil &&
+//   z.raw.start + 1 <= z.raw.end.
 func (z *Tokenizer) nextText() {
 	for {
 		c := z.readByte()
 		if z.err != nil {
-			z.data = z.raw
-			return
+			break
 		}
-		if c == '<' {
-			z.raw.end--
-			z.data = z.raw
-			return
+		if c != '<' {
+			continue
+		}
+		c = z.readByte()
+		if z.err != nil {
+			break
+		}
+		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '!' || c == '?' {
+			z.raw.end -= 2
+			break
+		}
+		if c != '/' {
+			continue
+		}
+		c = z.readByte()
+		if z.err != nil {
+			break
+		}
+		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
+			z.raw.end -= 3
+			break
 		}
 	}
+	z.data = z.raw
 }
 
 // Next scans the next token and returns its type.
diff --git a/src/pkg/html/token_test.go b/src/pkg/html/token_test.go
index 986e6cb748a..09bb75be157 100644
--- a/src/pkg/html/token_test.go
+++ b/src/pkg/html/token_test.go
@@ -21,6 +21,11 @@ type tokenTest struct {
 }
 
 var tokenTests = []tokenTest{
+	{
+		"empty",
+		"",
+		"",
+	},
 	// A single text node. The tokenizer should not break text nodes on whitespace,
 	// nor should it normalize whitespace within a text node.
 	{
@@ -41,6 +46,81 @@ var tokenTests = []tokenTest{
 		"<a>b<c/>d</e>",
 		"<a>$b$<c/>$d$</e>",
 	},
+	// Angle brackets that aren't a tag.
+	{
+		"not a tag #0",
+		"<",
+		"&lt;",
+	},
+	{
+		"not a tag #1",
+		"</",
+		"&lt;/",
+	},
+	/*
+		// TODO: re-enable these tests when we tokenize them correctly.
+		{
+			"not a tag #2",
+			"</>",
+			"",
+		},
+		{
+			"not a tag #3",
+			"a</>b",
+			"a$b",
+		},
+	*/
+	{
+		"not a tag #4",
+		"</ >",
+		"<!-- -->",
+	},
+	{
+		"not a tag #5",
+		"a < b",
+		"a &lt; b",
+	},
+	{
+		"not a tag #6",
+		"<.>",
+		"&lt;.&gt;",
+	},
+	{
+		"not a tag #7",
+		"a<<<b>>>c",
+		"a&lt;&lt;$<b>$&gt;&gt;c",
+	},
+	{
+		"not a tag #8",
+		"if x<0 and y < 0 then x*y>0",
+		"if x&lt;0 and y &lt; 0 then x*y&gt;0",
+	},
+	// EOF in a tag name.
+	{
+		"tag name eof #0",
+		"<a",
+		"",
+	},
+	{
+		"tag name eof #1",
+		"<a ",
+		"",
+	},
+	{
+		"tag name eof #2",
+		"a<b",
+		"a",
+	},
+	{
+		"tag name eof #3",
+		"<a><b",
+		"<a>",
+	},
+	{
+		"tag name eof #4",
+		`<a x`,
+		`<a x="">`,
+	},
 	// Some malformed tags that are missing a '>'.
 	{
 		"malformed tag #0",
@@ -257,8 +337,8 @@ var tokenTests = []tokenTest{
 	},
 	{
 		"Attributes with a solitary single quote",
-		"<p id=can't><p id=won't>",
-		"<p id=\"can&apos;t\">$<p id=\"won&apos;t\">",
+		`<p id=can't><p id=won't>`,
+		`<p id="can&apos;t">$<p id="won&apos;t">`,
 	},
 }
 
@@ -267,15 +347,17 @@ loop:
 	for _, tt := range tokenTests {
 		z := NewTokenizer(bytes.NewBuffer([]byte(tt.html)))
 		z.ReturnComments = true
-		for i, s := range strings.Split(tt.golden, "$") {
-			if z.Next() == ErrorToken {
-				t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error())
-				continue loop
-			}
-			actual := z.Token().String()
-			if s != actual {
-				t.Errorf("%s token %d: want %q got %q", tt.desc, i, s, actual)
-				continue loop
+		if tt.golden != "" {
+			for i, s := range strings.Split(tt.golden, "$") {
+				if z.Next() == ErrorToken {
+					t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error())
+					continue loop
+				}
+				actual := z.Token().String()
+				if s != actual {
+					t.Errorf("%s token %d: want %q got %q", tt.desc, i, s, actual)
+					continue loop
+				}
 			}
 		}
 		z.Next()