From 9a6cef8bbf1d62cbeb786f7a3eb9c009ab190ec9 Mon Sep 17 00:00:00 2001 From: Andrew Balholm Date: Tue, 17 Apr 2012 17:17:22 +1000 Subject: [PATCH] exp/html: more comprehensive tests Currently, the html package only runs a limited subset of the tests in the testdata directory. This tends to limit development of the parser to fixing the bug that causes the first failing test. This CL gives it the ability to run all the tests and produce a log showing the status of each test. (It does it when tests are run with 'go test --update-logs') The status is listed as PASS, FAIL, or PARSE (PARSE means that parsing produced the correct tree, but rendering and re-parsing does not produce the same tree). When 'go test' is run without --update-logs, it runs the tests marked 'PASS' in the logs (and the parsing portion of the tests marked 'PARSE'). Thus it will fail if there has been a regression since the last time the logs were updated. My goal for this CL is to allow develoment of the html package to be less test-driven, while still having the advantages of regression tests. In other words, one can work on any portion of the parser and quickly see whether he is breaking things or improving them. Current statistics of the tests: $ grep ^PASS *.log|wc -l 1017 $ grep ^PARSE *.log|wc -l 46 $ grep ^FAIL *.log|wc -l 181 R=nigeltao CC=golang-dev https://golang.org/cl/6031049 --- src/pkg/exp/html/parse_test.go | 227 +++++++++++++----- src/pkg/exp/html/testlogs/adoption01.dat.log | 13 + src/pkg/exp/html/testlogs/adoption02.dat.log | 2 + src/pkg/exp/html/testlogs/comments01.dat.log | 13 + src/pkg/exp/html/testlogs/doctype01.dat.log | 37 +++ src/pkg/exp/html/testlogs/entities01.dat.log | 67 ++++++ src/pkg/exp/html/testlogs/entities02.dat.log | 25 ++ .../exp/html/testlogs/html5test-com.dat.log | 24 ++ src/pkg/exp/html/testlogs/inbody01.dat.log | 4 + src/pkg/exp/html/testlogs/isindex.dat.log | 3 + ...ing-spec-changes-plain-text-unsafe.dat.log | 1 + .../testlogs/pending-spec-changes.dat.log | 2 + .../html/testlogs/plain-text-unsafe.dat.log | 1 + .../exp/html/testlogs/scriptdata01.dat.log | 26 ++ src/pkg/exp/html/testlogs/tables01.dat.log | 16 ++ src/pkg/exp/html/testlogs/tests1.dat.log | 114 +++++++++ src/pkg/exp/html/testlogs/tests10.dat.log | 54 +++++ src/pkg/exp/html/testlogs/tests11.dat.log | 9 + src/pkg/exp/html/testlogs/tests12.dat.log | 2 + src/pkg/exp/html/testlogs/tests14.dat.log | 7 + src/pkg/exp/html/testlogs/tests15.dat.log | 14 ++ src/pkg/exp/html/testlogs/tests16.dat.log | 189 +++++++++++++++ src/pkg/exp/html/testlogs/tests17.dat.log | 13 + src/pkg/exp/html/testlogs/tests18.dat.log | 20 ++ src/pkg/exp/html/testlogs/tests19.dat.log | 103 ++++++++ src/pkg/exp/html/testlogs/tests2.dat.log | 61 +++++ src/pkg/exp/html/testlogs/tests20.dat.log | 39 +++ src/pkg/exp/html/testlogs/tests21.dat.log | 22 ++ src/pkg/exp/html/testlogs/tests22.dat.log | 5 + src/pkg/exp/html/testlogs/tests23.dat.log | 5 + src/pkg/exp/html/testlogs/tests24.dat.log | 8 + src/pkg/exp/html/testlogs/tests25.dat.log | 20 ++ src/pkg/exp/html/testlogs/tests26.dat.log | 9 + src/pkg/exp/html/testlogs/tests3.dat.log | 24 ++ src/pkg/exp/html/testlogs/tests4.dat.log | 7 + src/pkg/exp/html/testlogs/tests5.dat.log | 16 ++ src/pkg/exp/html/testlogs/tests6.dat.log | 52 ++++ src/pkg/exp/html/testlogs/tests7.dat.log | 30 +++ src/pkg/exp/html/testlogs/tests8.dat.log | 9 + src/pkg/exp/html/testlogs/tests9.dat.log | 27 +++ .../html/testlogs/tests_innerHTML_1.dat.log | 84 +++++++ src/pkg/exp/html/testlogs/tricky01.dat.log | 9 + src/pkg/exp/html/testlogs/webkit01.dat.log | 49 ++++ src/pkg/exp/html/testlogs/webkit02.dat.log | 9 + 44 files changed, 1407 insertions(+), 64 deletions(-) create mode 100644 src/pkg/exp/html/testlogs/adoption01.dat.log create mode 100644 src/pkg/exp/html/testlogs/adoption02.dat.log create mode 100644 src/pkg/exp/html/testlogs/comments01.dat.log create mode 100644 src/pkg/exp/html/testlogs/doctype01.dat.log create mode 100644 src/pkg/exp/html/testlogs/entities01.dat.log create mode 100644 src/pkg/exp/html/testlogs/entities02.dat.log create mode 100644 src/pkg/exp/html/testlogs/html5test-com.dat.log create mode 100644 src/pkg/exp/html/testlogs/inbody01.dat.log create mode 100644 src/pkg/exp/html/testlogs/isindex.dat.log create mode 100644 src/pkg/exp/html/testlogs/pending-spec-changes-plain-text-unsafe.dat.log create mode 100644 src/pkg/exp/html/testlogs/pending-spec-changes.dat.log create mode 100644 src/pkg/exp/html/testlogs/plain-text-unsafe.dat.log create mode 100644 src/pkg/exp/html/testlogs/scriptdata01.dat.log create mode 100644 src/pkg/exp/html/testlogs/tables01.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests1.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests10.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests11.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests12.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests14.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests15.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests16.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests17.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests18.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests19.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests2.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests20.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests21.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests22.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests23.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests24.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests25.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests26.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests3.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests4.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests5.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests6.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests7.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests8.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests9.dat.log create mode 100644 src/pkg/exp/html/testlogs/tests_innerHTML_1.dat.log create mode 100644 src/pkg/exp/html/testlogs/tricky01.dat.log create mode 100644 src/pkg/exp/html/testlogs/webkit01.dat.log create mode 100644 src/pkg/exp/html/testlogs/webkit02.dat.log diff --git a/src/pkg/exp/html/parse_test.go b/src/pkg/exp/html/parse_test.go index f3f966cf58..f5e96f15a8 100644 --- a/src/pkg/exp/html/parse_test.go +++ b/src/pkg/exp/html/parse_test.go @@ -8,13 +8,17 @@ import ( "bufio" "bytes" "errors" + "flag" "fmt" "io" "os" + "path/filepath" "strings" "testing" ) +var updateLogs = flag.Bool("update-logs", false, "Update the log files that show the test results") + // readParseTest reads a single test case from r. func readParseTest(r *bufio.Reader) (text, want, context string, err error) { line, err := r.ReadSlice('\n') @@ -70,12 +74,22 @@ func readParseTest(r *bufio.Reader) (text, want, context string, err error) { if string(line) != "#document\n" { return "", "", "", fmt.Errorf(`got %q want "#document\n"`, line) } + inQuote := false for { line, err = r.ReadSlice('\n') if err != nil && err != io.EOF { return "", "", "", err } - if len(line) == 0 || len(line) == 1 && line[0] == '\n' { + trimmed := bytes.Trim(line, "| \n") + if len(trimmed) > 0 { + if line[0] == '|' && trimmed[0] == '"' { + inQuote = true + } + if trimmed[len(trimmed)-1] == '"' && !(line[0] == '|' && len(trimmed) == 1) { + inQuote = false + } + } + if len(line) == 0 || len(line) == 1 && line[0] == '\n' && !inQuote { break } b = append(b, line...) @@ -168,96 +182,181 @@ func dump(n *Node) (string, error) { return b.String(), nil } +const testDataDir = "testdata/webkit/" +const testLogDir = "testlogs/" + +type parseTestResult int + +const ( + // parseTestFailed indicates that an error occurred during parsing or that + // the parse tree did not match the expected result. + parseTestFailed parseTestResult = iota + // parseTestParseOnly indicates that the first stage of the test (parsing) + // passed, but rendering and re-parsing did not. + parseTestParseOnly + // parseTestPassed indicates that both stages of the test passed. + parseTestPassed +) + +func (r parseTestResult) String() string { + switch r { + case parseTestFailed: + return "FAIL" + case parseTestParseOnly: + return "PARSE" + case parseTestPassed: + return "PASS" + } + return "invalid parseTestResult value" +} + func TestParser(t *testing.T) { - testFiles := []struct { - filename string - // n is the number of test cases to run from that file. - // -1 means all test cases. - n int - }{ - // TODO(nigeltao): Process all the test cases from all the .dat files. - {"adoption01.dat", -1}, - {"doctype01.dat", -1}, - {"tests1.dat", -1}, - {"tests2.dat", -1}, - {"tests3.dat", -1}, - {"tests4.dat", -1}, - {"tests5.dat", -1}, - {"tests6.dat", -1}, - {"tests10.dat", 35}, + testFiles, err := filepath.Glob(testDataDir + "*.dat") + if err != nil { + t.Fatal(err) } for _, tf := range testFiles { - f, err := os.Open("testdata/webkit/" + tf.filename) + f, err := os.Open(tf) if err != nil { t.Fatal(err) } defer f.Close() r := bufio.NewReader(f) - for i := 0; i != tf.n; i++ { + + logName := testLogDir + tf[len(testDataDir):] + ".log" + var lf *os.File + var lbr *bufio.Reader + if *updateLogs { + lf, err = os.Create(logName) + } else { + lf, err = os.Open(logName) + lbr = bufio.NewReader(lf) + } + if err != nil { + t.Fatal(err) + } + defer lf.Close() + + for i := 0; ; i++ { text, want, context, err := readParseTest(r) - if err == io.EOF && tf.n == -1 { + if err == io.EOF { break } if err != nil { t.Fatal(err) } - var doc *Node - if context == "" { - doc, err = Parse(strings.NewReader(text)) + var expectedResult parseTestResult + if !*updateLogs { + var expectedText, expectedResultString string + _, err = fmt.Fscanf(lbr, "%s %q\n", &expectedResultString, &expectedText) if err != nil { t.Fatal(err) } - } else { - contextNode := &Node{ - Type: ElementNode, - Data: context, + if expectedText != text { + t.Fatalf("Log does not match tests: log has %q, tests have %q", expectedText, text) } - nodes, err := ParseFragment(strings.NewReader(text), contextNode) - if err != nil { - t.Fatal(err) - } - doc = &Node{ - Type: DocumentNode, - } - for _, n := range nodes { - doc.Add(n) + switch expectedResultString { + case "FAIL": + // Skip this test. + continue + case "PARSE": + expectedResult = parseTestParseOnly + case "PASS": + expectedResult = parseTestPassed + default: + t.Fatalf("Log has invalid test result: %q", expectedResultString) } } - got, err := dump(doc) - if err != nil { - t.Fatal(err) - } - // Compare the parsed tree to the #document section. - if got != want { - t.Errorf("%s test #%d %q, got vs want:\n----\n%s----\n%s----", tf.filename, i, text, got, want) - continue - } - if renderTestBlacklist[text] || context != "" { - continue - } - // Check that rendering and re-parsing results in an identical tree. - pr, pw := io.Pipe() - go func() { - pw.CloseWithError(Render(pw, doc)) - }() - doc1, err := Parse(pr) - if err != nil { - t.Fatal(err) - } - got1, err := dump(doc1) - if err != nil { - t.Fatal(err) - } - if got != got1 { - t.Errorf("%s test #%d %q, got vs got1:\n----\n%s----\n%s----", tf.filename, i, text, got, got1) - continue + result, err := testParseCase(text, want, context) + + if *updateLogs { + fmt.Fprintf(lf, "%s %q\n", result, text) + } else if result < expectedResult { + t.Errorf("%s test #%d %q, %s", tf, i, text, err) } } } } +// testParseCase tests one test case from the test files. It returns a +// parseTestResult indicating how much of the test passed. If the result +// is not parseTestPassed, it also returns an error that explains the failure. +// text is the HTML to be parsed, want is a dump of the correct parse tree, +// and context is the name of the context node, if any. +func testParseCase(text, want, context string) (result parseTestResult, err error) { + defer func() { + if x := recover(); x != nil { + switch e := x.(type) { + case error: + err = e + default: + err = fmt.Errorf("%v", e) + } + } + }() + + var doc *Node + if context == "" { + doc, err = Parse(strings.NewReader(text)) + if err != nil { + return parseTestFailed, err + } + } else { + contextNode := &Node{ + Type: ElementNode, + Data: context, + } + nodes, err := ParseFragment(strings.NewReader(text), contextNode) + if err != nil { + return parseTestFailed, err + } + doc = &Node{ + Type: DocumentNode, + } + for _, n := range nodes { + doc.Add(n) + } + } + + got, err := dump(doc) + if err != nil { + return parseTestFailed, err + } + // Compare the parsed tree to the #document section. + if got != want { + return parseTestFailed, fmt.Errorf("got vs want:\n----\n%s----\n%s----", got, want) + } + + if renderTestBlacklist[text] || context != "" { + return parseTestPassed, nil + } + + // Set result so that if a panic occurs during the render and re-parse + // the calling function will know that the parsing phase was successful. + result = parseTestParseOnly + + // Check that rendering and re-parsing results in an identical tree. + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(Render(pw, doc)) + }() + doc1, err := Parse(pr) + if err != nil { + return parseTestParseOnly, err + } + got1, err := dump(doc1) + if err != nil { + return parseTestParseOnly, err + } + if got != got1 { + return parseTestParseOnly, fmt.Errorf("got vs got1:\n----\n%s----\n%s----", got, got1) + } + + return parseTestPassed, nil +} + // Some test input result in parse trees are not 'well-formed' despite // following the HTML5 recovery algorithms. Rendering and re-parsing such a // tree will not result in an exact clone of that tree. We blacklist such diff --git a/src/pkg/exp/html/testlogs/adoption01.dat.log b/src/pkg/exp/html/testlogs/adoption01.dat.log new file mode 100644 index 0000000000..e710d38be6 --- /dev/null +++ b/src/pkg/exp/html/testlogs/adoption01.dat.log @@ -0,0 +1,13 @@ +PASS "

" +PASS "1

23

" +PASS "1" +PASS "123" +PASS "1
2
34
5
" +PASS "1

23

" +PASS "

" +PASS "

" +PASS "

" +PASS "

123

45" +PASS "
13
2
" +PASS "AC
B
" +PASS "
" diff --git a/src/pkg/exp/html/testlogs/adoption02.dat.log b/src/pkg/exp/html/testlogs/adoption02.dat.log new file mode 100644 index 0000000000..288019434a --- /dev/null +++ b/src/pkg/exp/html/testlogs/adoption02.dat.log @@ -0,0 +1,2 @@ +PASS "12

34" +PASS "

" diff --git a/src/pkg/exp/html/testlogs/comments01.dat.log b/src/pkg/exp/html/testlogs/comments01.dat.log new file mode 100644 index 0000000000..a72fc7a67e --- /dev/null +++ b/src/pkg/exp/html/testlogs/comments01.dat.log @@ -0,0 +1,13 @@ +PASS "FOOBAZ" +PASS "FOOBAZ" +PASS "FOOBAZ" +PASS "FOOBAZ" +PASS "FOOBAZ" +PASS "FOOBAZ" +PASS "FOOBAZ" +PASS "Hi" +PASS "" +PASS "BAZ" diff --git a/src/pkg/exp/html/testlogs/doctype01.dat.log b/src/pkg/exp/html/testlogs/doctype01.dat.log new file mode 100644 index 0000000000..9654bca5f8 --- /dev/null +++ b/src/pkg/exp/html/testlogs/doctype01.dat.log @@ -0,0 +1,37 @@ +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "Hello" +PASS "" +PASS "" +PASS "\n]>" +PASS "" +PASS "Mine!" +PASS "" +PASS "" +PASS "" +PASS "" diff --git a/src/pkg/exp/html/testlogs/entities01.dat.log b/src/pkg/exp/html/testlogs/entities01.dat.log new file mode 100644 index 0000000000..2f10bfe08d --- /dev/null +++ b/src/pkg/exp/html/testlogs/entities01.dat.log @@ -0,0 +1,67 @@ +PASS "FOO>BAR" +PASS "FOO>BAR" +PASS "FOO> BAR" +PASS "FOO>;;BAR" +PASS "I'm ¬it; I tell you" +PASS "I'm ∉ I tell you" +PASS "FOO& BAR" +PASS "FOO&" +PASS "FOO&&&>BAR" +PASS "FOO)BAR" +PASS "FOOABAR" +PASS "FOOABAR" +PASS "FOO&#BAR" +PASS "FOO&#ZOO" +PASS "FOOºR" +PASS "FOO&#xZOO" +PASS "FOO&#XZOO" +PASS "FOO)BAR" +PASS "FOO䆺R" +PASS "FOOAZOO" +PASS "FOO�ZOO" +PASS "FOOxZOO" +PASS "FOOyZOO" +PASS "FOO€ZOO" +PASS "FOOZOO" +PASS "FOO‚ZOO" +PASS "FOOƒZOO" +PASS "FOO„ZOO" +PASS "FOO…ZOO" +PASS "FOO†ZOO" +PASS "FOO‡ZOO" +PASS "FOOˆZOO" +PASS "FOO‰ZOO" +PASS "FOOŠZOO" +PASS "FOO‹ZOO" +PASS "FOOŒZOO" +PASS "FOOZOO" +PASS "FOOŽZOO" +PASS "FOOZOO" +PASS "FOOZOO" +PASS "FOO‘ZOO" +PASS "FOO’ZOO" +PASS "FOO“ZOO" +PASS "FOO”ZOO" +PASS "FOO•ZOO" +PASS "FOO–ZOO" +PASS "FOO—ZOO" +PASS "FOO˜ZOO" +PASS "FOO™ZOO" +PASS "FOOšZOO" +PASS "FOO›ZOO" +PASS "FOOœZOO" +PASS "FOOZOO" +PASS "FOOžZOO" +PASS "FOOŸZOO" +PASS "FOO ZOO" +PASS "FOO퟿ZOO" +PASS "FOO�ZOO" +PASS "FOO�ZOO" +PASS "FOO�ZOO" +PASS "FOO�ZOO" +PASS "FOOZOO" +PASS "FOO􏿾ZOO" +PASS "FOO􈟔ZOO" +PASS "FOO􏿿ZOO" +PASS "FOO�ZOO" +PASS "FOO�ZOO" diff --git a/src/pkg/exp/html/testlogs/entities02.dat.log b/src/pkg/exp/html/testlogs/entities02.dat.log new file mode 100644 index 0000000000..a7a400007a --- /dev/null +++ b/src/pkg/exp/html/testlogs/entities02.dat.log @@ -0,0 +1,25 @@ +PASS "
" +PASS "
" +PASS "
" +PASS "
" +FAIL "
" +FAIL "
" +FAIL "
" +FAIL "
" +FAIL "
" +PASS "
" +PASS "
" +PASS "
" +PASS "
" +PASS "
" +PASS "
" +PASS "
" +PASS "
" +FAIL "
" +PASS "
" +PASS "
ZZ£_id=23
" +PASS "
ZZ&prod_id=23
" +PASS "
ZZ£_id=23
" +PASS "
ZZ∏_id=23
" +PASS "
ZZ£=23
" +PASS "
ZZ&prod=23
" diff --git a/src/pkg/exp/html/testlogs/html5test-com.dat.log b/src/pkg/exp/html/testlogs/html5test-com.dat.log new file mode 100644 index 0000000000..0742940268 --- /dev/null +++ b/src/pkg/exp/html/testlogs/html5test-com.dat.log @@ -0,0 +1,24 @@ +PASS "" +PASS "
" +PASS "
" +PASS "
" +PASS "" +PASS "" +PASS "⟨⟩" +PASS "'" +PASS "ⅈ" +PASS "𝕂" +PASS "∉" +PASS "" +PASS "" +PASS "" +PASS "-->" +PASS "-->" +PASS "-->" +PASS "-->" +PASS "
  • A
  • B
" +FAIL "
" +PASS "AB

CD" +PASS "

" +PASS "" +PASS "" diff --git a/src/pkg/exp/html/testlogs/inbody01.dat.log b/src/pkg/exp/html/testlogs/inbody01.dat.log new file mode 100644 index 0000000000..f0695812ba --- /dev/null +++ b/src/pkg/exp/html/testlogs/inbody01.dat.log @@ -0,0 +1,4 @@ +PASS "