From 2ddc99baff6cbd351deb9ae5ac0797bded26801b Mon Sep 17 00:00:00 2001 From: yuin Date: Thu, 16 May 2019 19:46:36 +0900 Subject: [PATCH] Add extension tests, Fix bugs in extensions --- README.md | 4 +- _test/options.txt | 27 ++++ ast/inline.go | 29 ++++- commonmark_test.go | 32 ++--- extension/_test/definition_list.txt | 143 +++++++++++++++++++++ extension/_test/footnote.txt | 22 ++++ extension/_test/linkify.txt | 114 +++++++++++++++++ extension/_test/strikethrough.txt | 18 +++ extension/_test/table.txt | 190 ++++++++++++++++++++++++++++ extension/_test/tasklist.txt | 30 +++++ extension/_test/typographer.txt | 20 +++ extension/ast/table.go | 16 ++- extension/definition_list_test.go | 19 +++ extension/footnote_test.go | 19 +++ extension/gfm.go | 14 -- extension/linkify.go | 11 +- extension/linkify_test.go | 19 +++ extension/strikethrough_test.go | 19 +++ extension/table.go | 28 ++-- extension/table_test.go | 19 +++ extension/tasklist_test.go | 19 +++ extension/typographer_test.go | 19 +++ options_test.go | 16 +++ parser/atx_heading.go | 5 +- parser/html_block.go | 58 +-------- parser/parser.go | 3 + parser/raw_html.go | 28 +--- renderer/html/html.go | 12 +- testutil.go | 103 +++++++++++++++ 29 files changed, 912 insertions(+), 144 deletions(-) create mode 100644 _test/options.txt create mode 100644 extension/_test/definition_list.txt create mode 100644 extension/_test/footnote.txt create mode 100644 extension/_test/linkify.txt create mode 100644 extension/_test/strikethrough.txt create mode 100644 extension/_test/table.txt create mode 100644 extension/_test/tasklist.txt create mode 100644 extension/_test/typographer.txt create mode 100644 extension/definition_list_test.go create mode 100644 extension/footnote_test.go create mode 100644 extension/linkify_test.go create mode 100644 extension/strikethrough_test.go create mode 100644 extension/table_test.go create mode 100644 extension/tasklist_test.go create mode 100644 extension/typographer_test.go create mode 100644 options_test.go create mode 100644 testutil.go diff --git a/README.md b/README.md index e4bb79f..aa3db57 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,6 @@ Parser and Renderer options | `parser.WithParagraphTransformers` | A `util.PrioritizedSlice` whose elements are `parser.ParagraphTransformer` | Transformers for transforming paragraph nodes. | | `parser.WithAutoHeadingID` | `-` | Enables auto heading ids. | | `parser.WithAttribute` | `-` | Enables custom attributes. Currently only headings supports attributes. | -| `parser.WithFilterTags` | `...string` | HTML tag names forbidden in HTML blocks and Raw HTMLs. | ### HTML Renderer options @@ -116,7 +115,8 @@ Parser and Renderer options - [Gitmark Flavored Markdown: Task list items](https://github.github.com/gfm/#task-list-items-extension-) - `extension.GFM` - This extension enables Table, Strikethrough, Linkify and TaskList. - In addition, this extension sets some tags to `parser.FilterTags` . + - This extension does not filter tags defined in [6.11Disallowed Raw HTML (extension)](https://github.github.com/gfm/#disallowed-raw-html-extension-). + If you need to filter HTML tags, see [Security](#security) - `extension.DefinitionList` - [PHP Markdown Extra: Definition lists](https://michelf.ca/projects/php-markdown/extra/#def-list) - `extension.Footnote` diff --git a/_test/options.txt b/_test/options.txt new file mode 100644 index 0000000..521fd22 --- /dev/null +++ b/_test/options.txt @@ -0,0 +1,27 @@ +1 +//- - - - - - - - -// +## Title 0 + +## Title1 # {#id_1 .class-1} + +## Title2 {#id_2} + +## Title3 ## {#id_3 .class-3} + +## Title4 ## {attr3=value3} + +## Title5 ## {#id_5 attr5=value5} + +## Title6 ## {#id_6 .class6 attr6=value6} + +## Title7 ## {#id_7 attr7="value \"7"} +//- - - - - - - - -// +

Title 0

+

Title1

+

Title2

+

Title3

+

Title4

+

Title5

+

Title6

+

Title7

+//= = = = = = = = = = = = = = = = = = = = = = = =// diff --git a/ast/inline.go b/ast/inline.go index e1f12e2..7404a3f 100644 --- a/ast/inline.go +++ b/ast/inline.go @@ -362,10 +362,13 @@ const ( // An AutoLink struct represents an autolink of the Markdown text. type AutoLink struct { BaseInline - // Value is a link text of this node. - Value *Text // Type is a type of this autolink. AutoLinkType AutoLinkType + + // Protocol specified a protocol of the link. + Protocol []byte + + value *Text } // Inline implements Inline.Inline. @@ -373,7 +376,7 @@ func (n *AutoLink) Inline() {} // Dump implenets Node.Dump func (n *AutoLink) Dump(source []byte, level int) { - segment := n.Value.Segment + segment := n.value.Segment m := map[string]string{ "Value": string(segment.Value(source)), } @@ -388,11 +391,29 @@ func (n *AutoLink) Kind() NodeKind { return KindAutoLink } +// URL returns an url of this node. +func (n *AutoLink) URL(source []byte) []byte { + if n.Protocol != nil { + s := n.value.Segment + ret := make([]byte, 0, len(n.Protocol)+s.Len()+3) + ret = append(ret, n.Protocol...) + ret = append(ret, ':', '/', '/') + ret = append(ret, n.value.Text(source)...) + return ret + } + return n.value.Text(source) +} + +// Label returns a label of this node. +func (n *AutoLink) Label(source []byte) []byte { + return n.value.Text(source) +} + // NewAutoLink returns a new AutoLink node. func NewAutoLink(typ AutoLinkType, value *Text) *AutoLink { return &AutoLink{ BaseInline: BaseInline{}, - Value: value, + value: value, AutoLinkType: typ, } } diff --git a/commonmark_test.go b/commonmark_test.go index d975900..64d5e3d 100644 --- a/commonmark_test.go +++ b/commonmark_test.go @@ -1,7 +1,6 @@ package goldmark import ( - "bytes" "encoding/json" "io/ioutil" "testing" @@ -27,30 +26,17 @@ func TestSpec(t *testing.T) { if err := json.Unmarshal(bs, &testCases); err != nil { panic(err) } + cases := []MarkdownTestCase{} + for _, c := range testCases { + cases = append(cases, MarkdownTestCase{ + No: c.Example, + Markdown: c.Markdown, + Expected: c.HTML, + }) + } markdown := New(WithRendererOptions( html.WithXHTML(), html.WithUnsafe(), )) - for _, testCase := range testCases { - var out bytes.Buffer - if err := markdown.Convert([]byte(testCase.Markdown), &out); err != nil { - panic(err) - } - if !bytes.Equal(bytes.TrimSpace(out.Bytes()), bytes.TrimSpace([]byte(testCase.HTML))) { - format := `============= case %d ================ -Markdown: ------------ -%s - -Expected: ----------- -%s - -Actual ---------- -%s -` - t.Errorf(format, testCase.Example, testCase.Markdown, testCase.HTML, out.Bytes()) - } - } + DoTestCases(markdown, cases, t) } diff --git a/extension/_test/definition_list.txt b/extension/_test/definition_list.txt new file mode 100644 index 0000000..2ddc3ea --- /dev/null +++ b/extension/_test/definition_list.txt @@ -0,0 +1,143 @@ +1 +//- - - - - - - - -// +Apple +: Pomaceous fruit of plants of the genus Malus in +the family Rosaceae. + +Orange +: The fruit of an evergreen tree of the genus Citrus. +//- - - - - - - - -// +
+
Apple
+
Pomaceous fruit of plants of the genus Malus in +the family Rosaceae.
+
Orange
+
The fruit of an evergreen tree of the genus Citrus.
+
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +2 +//- - - - - - - - -// +Apple +: Pomaceous fruit of plants of the genus Malus in + the family Rosaceae. +: An American computer company. + +Orange +: The fruit of an evergreen tree of the genus Citrus. +//- - - - - - - - -// +
+
Apple
+
Pomaceous fruit of plants of the genus Malus in +the family Rosaceae.
+
An American computer company.
+
Orange
+
The fruit of an evergreen tree of the genus Citrus.
+
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +3 +//- - - - - - - - -// +Term 1 +Term 2 +: Definition a + +Term 3 +: Definition b +//- - - - - - - - -// +
+
Term 1
+
Term 2
+
Definition a
+
Term 3
+
Definition b
+
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +4 +//- - - - - - - - -// +Apple + +: Pomaceous fruit of plants of the genus Malus in + the family Rosaceae. + +Orange + +: The fruit of an evergreen tree of the genus Citrus. +//- - - - - - - - -// +
+
Apple
+
+

Pomaceous fruit of plants of the genus Malus in +the family Rosaceae.

+
+
Orange
+
+

The fruit of an evergreen tree of the genus Citrus.

+
+
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + +5 +//- - - - - - - - -// +Term 1 + +: This is a definition with two paragraphs. Lorem ipsum + dolor sit amet, consectetuer adipiscing elit. Aliquam + hendrerit mi posuere lectus. + + Vestibulum enim wisi, viverra nec, fringilla in, laoreet + vitae, risus. + +: Second definition for term 1, also wrapped in a paragraph + because of the blank line preceding it. + +Term 2 + +: This definition has a code block, a blockquote and a list. + + code block. + + > block quote + > on two lines. + + 1. first list item + 2. second list item +//- - - - - - - - -// +
+
Term 1
+
+

This is a definition with two paragraphs. Lorem ipsum +dolor sit amet, consectetuer adipiscing elit. Aliquam +hendrerit mi posuere lectus.

+

Vestibulum enim wisi, viverra nec, fringilla in, laoreet +vitae, risus.

+
+
+

Second definition for term 1, also wrapped in a paragraph +because of the blank line preceding it.

+
+
Term 2
+
+

This definition has a code block, a blockquote and a list.

+
code block.
+
+
+

block quote +on two lines.

+
+
    +
  1. first list item
    2. +
  2. second list item
    3. +
+
+
+//= = = = = = = = = = = = = = = = = = = = = = = =// + diff --git a/extension/_test/footnote.txt b/extension/_test/footnote.txt new file mode 100644 index 0000000..6a4ea5c --- /dev/null +++ b/extension/_test/footnote.txt @@ -0,0 +1,22 @@ +1 +//- - - - - - - - -// +That's some text with a footnote.[^1] + +[^1]: And that's the footnote. + + That's the second paragraph. +//- - - - - - - - -// +

That's some text with a footnote.1

+
+
+
    +
  1. +

    And that's the footnote.

    +

    That's the second paragraph.

    +
    2. +
+
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + diff --git a/extension/_test/linkify.txt b/extension/_test/linkify.txt new file mode 100644 index 0000000..d421934 --- /dev/null +++ b/extension/_test/linkify.txt @@ -0,0 +1,114 @@ +1 +//- - - - - - - - -// +www.commonmark.org +//- - - - - - - - -// +

www.commonmark.org

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +2 +//- - - - - - - - -// +Visit www.commonmark.org/help for more information. +//- - - - - - - - -// +

Visit www.commonmark.org/help for more information.

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +3 +//- - - - - - - - -// +www.google.com/search?q=Markup+(business) + +www.google.com/search?q=Markup+(business))) + +(www.google.com/search?q=Markup+(business)) + +(www.google.com/search?q=Markup+(business) +//- - - - - - - - -// +

www.google.com/search?q=Markup+(business)

+

www.google.com/search?q=Markup+(business)))

+

(www.google.com/search?q=Markup+(business))

+

(www.google.com/search?q=Markup+(business)

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +4 +//- - - - - - - - -// +www.google.com/search?q=(business))+ok +//- - - - - - - - -// +

www.google.com/search?q=(business))+ok

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +5 +//- - - - - - - - -// +www.google.com/search?q=commonmark&hl=en + +www.google.com/search?q=commonmark&hl; +//- - - - - - - - -// +

www.google.com/search?q=commonmark&hl=en

+

www.google.com/search?q=commonmark&hl;

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +6 +//- - - - - - - - -// +www.commonmark.org/hewww.commonmark.org/he<lp

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +7 +//- - - - - - - - -// +http://commonmark.org + +(Visit https://encrypted.google.com/search?q=Markup+(business)) + +Anonymous FTP is available at ftp://foo.bar.baz. +//- - - - - - - - -// +

http://commonmark.org

+

(Visit https://encrypted.google.com/search?q=Markup+(business))

+

Anonymous FTP is available at ftp://foo.bar.baz.

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +8 +//- - - - - - - - -// +foo@bar.baz +//- - - - - - - - -// +

foo@bar.baz

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +9 +//- - - - - - - - -// +hello@mail+xyz.example isn't valid, but hello+xyz@mail.example is. +//- - - - - - - - -// +

hello@mail+xyz.example isn't valid, but hello+xyz@mail.example is.

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +10 +//- - - - - - - - -// +a.b-c_d@a.b + +a.b-c_d@a.b. + +a.b-c_d@a.b- + +a.b-c_d@a.b_ +//- - - - - - - - -// +

a.b-c_d@a.b

+

a.b-c_d@a.b.

+

a.b-c_d@a.b-

+

a.b-c_d@a.b_

+//= = = = = = = = = = = = = = = = = = = = = = = =// diff --git a/extension/_test/strikethrough.txt b/extension/_test/strikethrough.txt new file mode 100644 index 0000000..dbb48f6 --- /dev/null +++ b/extension/_test/strikethrough.txt @@ -0,0 +1,18 @@ +1 +//- - - - - - - - -// +~~Hi~~ Hello, world! +//- - - - - - - - -// +

Hi Hello, world!

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +2 +//- - - - - - - - -// +This ~~has a + +new paragraph~~. +//- - - - - - - - -// +

This ~~has a

+

new paragraph~~.

+//= = = = = = = = = = = = = = = = = = = = = = = =// diff --git a/extension/_test/table.txt b/extension/_test/table.txt new file mode 100644 index 0000000..ce0d558 --- /dev/null +++ b/extension/_test/table.txt @@ -0,0 +1,190 @@ +1 +//- - - - - - - - -// +| foo | bar | +| --- | --- | +| baz | bim | +//- - - - - - - - -// + + + + + + + + + + + + + +
foobar
bazbim
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +2 +//- - - - - - - - -// +| abc | defghi | +:-: | -----------: +bar | baz +//- - - - - - - - -// + + + + + + + + + + + + + +
abcdefghi
barbaz
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +3 +//- - - - - - - - -// +| f\|oo | +| ------ | +| b `\|` az | +| b **\|** im | +//- - - - - - - - -// + + + + + + + + + + + + + + +
f|oo
b \| az
b | im
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +4 +//- - - - - - - - -// +| abc | def | +| --- | --- | +| bar | baz | +> bar +//- - - - - - - - -// + + + + + + + + + + + + + +
abcdef
barbaz
+
+

bar

+
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +5 +//- - - - - - - - -// +| abc | def | +| --- | --- | +| bar | baz | +bar + +bar +//- - - - - - - - -// + + + + + + + + + + + + + + + + + +
abcdef
barbaz
bar
+

bar

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +6 +//- - - - - - - - -// +| abc | def | +| --- | +| bar | +//- - - - - - - - -// +

| abc | def | +| --- | +| bar |

+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +7 +//- - - - - - - - -// +| abc | def | +| --- | --- | +| bar | +| bar | baz | boo | +//- - - - - - - - -// + + + + + + + + + + + + + + + + + +
abcdef
bar
barbaz
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +8 +//- - - - - - - - -// +| abc | def | +| --- | --- | +//- - - - - - - - -// + + + + + + + +
abcdef
+//= = = = = = = = = = = = = = = = = = = = = = = =// diff --git a/extension/_test/tasklist.txt b/extension/_test/tasklist.txt new file mode 100644 index 0000000..6bbe1b6 --- /dev/null +++ b/extension/_test/tasklist.txt @@ -0,0 +1,30 @@ +1 +//- - - - - - - - -// +- [ ] foo +- [x] bar +//- - - - - - - - -// +
    +
  • foo
    • +
  • bar
    • +
+//= = = = = = = = = = = = = = = = = = = = = = = =// + + + +2 +//- - - - - - - - -// +- [x] foo + - [ ] bar + - [x] baz +- [ ] bim +//- - - - - - - - -// +
    +
  • foo +
      +
    • bar
      • +
    • baz
      • +
    +
    • +
  • bim
    • +
+//= = = = = = = = = = = = = = = = = = = = = = = =// diff --git a/extension/_test/typographer.txt b/extension/_test/typographer.txt new file mode 100644 index 0000000..6445852 --- /dev/null +++ b/extension/_test/typographer.txt @@ -0,0 +1,20 @@ +1 +//- - - - - - - - -// +This should 'be' replaced +//- - - - - - - - -// +

This should ‘be’ replaced

+//= = = = = = = = = = = = = = = = = = = = = = = =// + +2 +//- - - - - - - - -// +This should "be" replaced +//- - - - - - - - -// +

This should “be” replaced

+//= = = = = = = = = = = = = = = = = = = = = = = =// + +3 +//- - - - - - - - -// +**--** *---* a...<< b>> +//- - - - - - - - -// +

– — a…« b»

+//= = = = = = = = = = = = = = = = = = = = = = = =// diff --git a/extension/ast/table.go b/extension/ast/table.go index 7396b86..1d8890b 100644 --- a/extension/ast/table.go +++ b/extension/ast/table.go @@ -102,7 +102,8 @@ func NewTableRow(alignments []Alignment) *TableRow { // A TableHeader struct represents a table header of Markdown(GFM) text. type TableHeader struct { - *TableRow + gast.BaseBlock + Alignments []Alignment } // KindTableHeader is a NodeKind of the TableHeader node. @@ -113,9 +114,20 @@ func (n *TableHeader) Kind() gast.NodeKind { return KindTableHeader } +// Dump implements Node.Dump. +func (n *TableHeader) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + // NewTableHeader returns a new TableHeader node. func NewTableHeader(row *TableRow) *TableHeader { - return &TableHeader{row} + n := &TableHeader{} + for c := row.FirstChild(); c != nil; { + next := c.NextSibling() + n.AppendChild(n, c) + c = next + } + return n } // A TableCell struct represents a table cell of a Markdown(GFM) text. diff --git a/extension/definition_list_test.go b/extension/definition_list_test.go new file mode 100644 index 0000000..1cbf167 --- /dev/null +++ b/extension/definition_list_test.go @@ -0,0 +1,19 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "testing" +) + +func TestDefinitionList(t *testing.T) { + markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + DefinitionList, + ), + ) + goldmark.DoTestCaseFile(markdown, "_test/definition_list.txt", t) +} diff --git a/extension/footnote_test.go b/extension/footnote_test.go new file mode 100644 index 0000000..b5ce90c --- /dev/null +++ b/extension/footnote_test.go @@ -0,0 +1,19 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "testing" +) + +func TestFootnote(t *testing.T) { + markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + Footnote, + ), + ) + goldmark.DoTestCaseFile(markdown, "_test/footnote.txt", t) +} diff --git a/extension/gfm.go b/extension/gfm.go index b16a56d..a570fbd 100644 --- a/extension/gfm.go +++ b/extension/gfm.go @@ -2,7 +2,6 @@ package extension import ( "github.com/yuin/goldmark" - "github.com/yuin/goldmark/parser" ) type gfm struct { @@ -11,20 +10,7 @@ type gfm struct { // GFM is an extension that provides Github Flavored markdown functionalities. var GFM = &gfm{} -var filterTags = []string{ - "title", - "textarea", - "style", - "xmp", - "iframe", - "noembed", - "noframes", - "script", - "plaintext", -} - func (e *gfm) Extend(m goldmark.Markdown) { - m.Parser().AddOptions(parser.WithFilterTags(filterTags...)) Linkify.Extend(m) Table.Extend(m) Strikethrough.Extend(m) diff --git a/extension/linkify.go b/extension/linkify.go index f0544bf..0efe2ce 100644 --- a/extension/linkify.go +++ b/extension/linkify.go @@ -48,13 +48,14 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont } var m []int - typ := ast.AutoLinkType(ast.AutoLinkEmail) - typ = ast.AutoLinkURL + var protocol []byte + var typ ast.AutoLinkType = ast.AutoLinkURL if bytes.HasPrefix(line, protoHTTP) || bytes.HasPrefix(line, protoHTTPS) || bytes.HasPrefix(line, protoFTP) { m = urlRegexp.FindSubmatchIndex(line) } if m == nil && bytes.HasPrefix(line, domainWWW) { m = wwwURLRegxp.FindSubmatchIndex(line) + protocol = []byte("http") } if m != nil { lastChar := line[m[1]-1] @@ -70,7 +71,7 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont } } if closing > 0 { - m[1]-- + m[1] -= closing } } else if lastChar == ';' { i := m[1] - 2 @@ -119,7 +120,9 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont consumes += m[1] block.Advance(consumes) n := ast.NewTextSegment(text.NewSegment(start, start+m[1])) - return ast.NewAutoLink(typ, n) + link := ast.NewAutoLink(typ, n) + link.Protocol = protocol + return link } func (s *linkifyParser) CloseBlock(parent ast.Node, pc parser.Context) { diff --git a/extension/linkify_test.go b/extension/linkify_test.go new file mode 100644 index 0000000..5a80b95 --- /dev/null +++ b/extension/linkify_test.go @@ -0,0 +1,19 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "testing" +) + +func TestLinkify(t *testing.T) { + markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + Linkify, + ), + ) + goldmark.DoTestCaseFile(markdown, "_test/linkify.txt", t) +} diff --git a/extension/strikethrough_test.go b/extension/strikethrough_test.go new file mode 100644 index 0000000..891409e --- /dev/null +++ b/extension/strikethrough_test.go @@ -0,0 +1,19 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "testing" +) + +func TestStrikethrough(t *testing.T) { + markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + Strikethrough, + ), + ) + goldmark.DoTestCaseFile(markdown, "_test/strikethrough.txt", t) +} diff --git a/extension/table.go b/extension/table.go index 8f20f9f..4ff7b73 100644 --- a/extension/table.go +++ b/extension/table.go @@ -41,7 +41,7 @@ func (b *tableParagraphTransformer) Transform(node *gast.Paragraph, reader text. if alignments == nil { return } - header := b.parseRow(lines.At(0), alignments, reader) + header := b.parseRow(lines.At(0), alignments, true, reader) if header == nil || len(alignments) != header.ChildCount() { return } @@ -50,15 +50,14 @@ func (b *tableParagraphTransformer) Transform(node *gast.Paragraph, reader text. table.AppendChild(table, ast.NewTableHeader(header)) if lines.Len() > 2 { for i := 2; i < lines.Len(); i++ { - table.AppendChild(table, b.parseRow(lines.At(i), alignments, reader)) + table.AppendChild(table, b.parseRow(lines.At(i), alignments, false, reader)) } } node.Parent().InsertBefore(node.Parent(), node, table) node.Parent().RemoveChild(node.Parent(), node) - return } -func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments []ast.Alignment, reader text.Reader) *ast.TableRow { +func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments []ast.Alignment, isHeader bool, reader text.Reader) *ast.TableRow { source := reader.Source() line := segment.Value(source) pos := 0 @@ -72,7 +71,16 @@ func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments [] if len(line) > 0 && line[limit-1] == '|' { limit-- } - for i := 0; pos < limit; i++ { + i := 0 + for ; pos < limit; i++ { + alignment := ast.AlignNone + if i >= len(alignments) { + if !isHeader { + return row + } + } else { + alignment = alignments[i] + } closure := util.FindClosure(line[pos:], byte(0), '|', true, false) if closure < 0 { closure = len(line[pos:]) @@ -82,10 +90,13 @@ func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments [] segment = segment.TrimLeftSpace(source) segment = segment.TrimRightSpace(source) node.Lines().Append(segment) - node.Alignment = alignments[i] + node.Alignment = alignment row.AppendChild(row, node) pos += closure + 1 } + for ; i < len(alignments); i++ { + row.AppendChild(row, ast.NewTableCell()) + } return row } @@ -175,9 +186,6 @@ func (r *TableHTMLRenderer) renderTableHeader(w util.BufWriter, source []byte, n if n.NextSibling() != nil { w.WriteString("\n") } - if n.Parent().LastChild() == n { - w.WriteString("\n") - } } return gast.WalkContinue, nil } @@ -197,7 +205,7 @@ func (r *TableHTMLRenderer) renderTableRow(w util.BufWriter, source []byte, n ga func (r *TableHTMLRenderer) renderTableCell(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { n := node.(*ast.TableCell) tag := "td" - if n.Parent().Parent().FirstChild() == n.Parent() { + if n.Parent().Kind() == ast.KindTableHeader { tag = "th" } if entering { diff --git a/extension/table_test.go b/extension/table_test.go new file mode 100644 index 0000000..7e38896 --- /dev/null +++ b/extension/table_test.go @@ -0,0 +1,19 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "testing" +) + +func TestTable(t *testing.T) { + markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + Table, + ), + ) + goldmark.DoTestCaseFile(markdown, "_test/table.txt", t) +} diff --git a/extension/tasklist_test.go b/extension/tasklist_test.go new file mode 100644 index 0000000..6fc74d5 --- /dev/null +++ b/extension/tasklist_test.go @@ -0,0 +1,19 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "testing" +) + +func TestTaskList(t *testing.T) { + markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + TaskList, + ), + ) + goldmark.DoTestCaseFile(markdown, "_test/tasklist.txt", t) +} diff --git a/extension/typographer_test.go b/extension/typographer_test.go new file mode 100644 index 0000000..6646314 --- /dev/null +++ b/extension/typographer_test.go @@ -0,0 +1,19 @@ +package extension + +import ( + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/renderer/html" + "testing" +) + +func TestTypographer(t *testing.T) { + markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + Typographer, + ), + ) + goldmark.DoTestCaseFile(markdown, "_test/typographer.txt", t) +} diff --git a/options_test.go b/options_test.go new file mode 100644 index 0000000..f1a0b0f --- /dev/null +++ b/options_test.go @@ -0,0 +1,16 @@ +package goldmark + +import ( + "github.com/yuin/goldmark/parser" + "testing" +) + +func TestDefinitionList(t *testing.T) { + markdown := New( + WithParserOptions( + parser.WithAttribute(), + parser.WithAutoHeadingID(), + ), + ) + DoTestCaseFile(markdown, "_test/options.txt", t) +} diff --git a/parser/atx_heading.go b/parser/atx_heading.go index 9c63ba1..fbcfc5c 100644 --- a/parser/atx_heading.go +++ b/parser/atx_heading.go @@ -125,7 +125,7 @@ func (b *atxHeadingParser) Open(parent ast.Node, reader text.Reader, pc Context) break } node.SetAttribute(line[as+ai[0]:as+ai[1]], - line[as+ai[2]:as+ai[3]]) + util.UnescapePunctuations(line[as+ai[2]:as+ai[3]])) as += ai[3] + skip } for ; as < stop && util.IsSpace(line[as]); as++ { @@ -208,7 +208,8 @@ func parseLastLineAttributes(node ast.Node, reader text.Reader, pc Context) { indicies := util.FindAttributeIndiciesReverse(line, true) if indicies != nil { for _, index := range indicies { - node.SetAttribute(line[index[0]:index[1]], line[index[2]:index[3]]) + node.SetAttribute(line[index[0]:index[1]], + util.UnescapePunctuations(line[index[2]:index[3]])) } lastLine.Stop = lastLine.Start + indicies[0][0] - 1 lastLine.TrimRightSpace(reader.Source()) diff --git a/parser/html_block.go b/parser/html_block.go index 831ba7d..eb822ef 100644 --- a/parser/html_block.go +++ b/parser/html_block.go @@ -9,48 +9,6 @@ import ( "strings" ) -// An HTMLConfig struct is a data structure that holds configuration of the renderers related to raw htmls. -type HTMLConfig struct { - FilterTags map[string]bool -} - -// SetOption implements SetOptioner. -func (b *HTMLConfig) SetOption(name OptionName, value interface{}) { - switch name { - case optFilterTags: - b.FilterTags = value.(map[string]bool) - } -} - -// A HTMLOption interface sets options for the raw HTML parsers. -type HTMLOption interface { - Option - SetHTMLOption(*HTMLConfig) -} - -const optFilterTags OptionName = "FilterTags" - -type withFilterTags struct { - value map[string]bool -} - -func (o *withFilterTags) SetParserOption(c *Config) { - c.Options[optFilterTags] = o.value -} - -func (o *withFilterTags) SetHTMLOption(p *HTMLConfig) { - p.FilterTags = o.value -} - -// WithFilterTags is a functional otpion that specify forbidden tag names. -func WithFilterTags(names ...string) HTMLOption { - m := map[string]bool{} - for _, name := range names { - m[name] = true - } - return &withFilterTags{m} -} - var allowedBlockTags = map[string]bool{ "address": true, "article": true, @@ -137,17 +95,14 @@ var htmlBlockType6Regexp = regexp.MustCompile(`^[ ]{0,3}|/>)\s*\n?$`) type htmlBlockParser struct { - HTMLConfig } +var defaultHtmlBlockParser = &htmlBlockParser{} + // NewHTMLBlockParser return a new BlockParser that can parse html // blocks. -func NewHTMLBlockParser(opts ...HTMLOption) BlockParser { - p := &htmlBlockParser{} - for _, o := range opts { - o.SetHTMLOption(&p.HTMLConfig) - } - return p +func NewHTMLBlockParser() BlockParser { + return defaultHtmlBlockParser } func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { @@ -191,11 +146,6 @@ func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) } } if node != nil { - if b.FilterTags != nil { - if _, ok := b.FilterTags[tagName]; ok { - return nil, NoChildren - } - } reader.Advance(segment.Len() - 1) node.Lines().Append(segment) return node, NoChildren diff --git a/parser/parser.go b/parser/parser.go index 615c1de..6352353 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -84,6 +84,9 @@ func (s *ids) Generate(value, prefix []byte) []byte { continue } if util.IsAlphaNumeric(v) { + if 'A' <= v && v <= 'Z' { + v += 'a' - 'A' + } result = append(result, v) } else if util.IsSpace(v) { result = append(result, '-') diff --git a/parser/raw_html.go b/parser/raw_html.go index fcc76e2..d7ba414 100644 --- a/parser/raw_html.go +++ b/parser/raw_html.go @@ -9,17 +9,14 @@ import ( ) type rawHTMLParser struct { - HTMLConfig } +var defaultRawHTMLParser = &rawHTMLParser{} + // NewRawHTMLParser return a new InlineParser that can parse // inline htmls -func NewRawHTMLParser(opts ...HTMLOption) InlineParser { - p := &rawHTMLParser{} - for _, o := range opts { - o.SetHTMLOption(&p.HTMLConfig) - } - return p +func NewRawHTMLParser() InlineParser { + return defaultRawHTMLParser } func (s *rawHTMLParser) Trigger() []byte { @@ -74,22 +71,7 @@ var dummyMatch = [][]byte{} func (s *rawHTMLParser) parseMultiLineRegexp(reg *regexp.Regexp, block text.Reader, pc Context) ast.Node { sline, ssegment := block.Position() - var m [][]byte - if s.FilterTags != nil { - m = block.FindSubMatch(reg) - } else { - if block.Match(reg) { - m = dummyMatch - } - } - - if m != nil { - if s.FilterTags != nil && len(m) > 1 { - tagName := string(m[1]) - if _, ok := s.FilterTags[tagName]; ok { - return nil - } - } + if block.Match(reg) { node := ast.NewRawHTML() eline, esegment := block.Position() block.SetPosition(sline, ssegment) diff --git a/renderer/html/html.go b/renderer/html/html.go index 0fd3dcd..8551b78 100644 --- a/renderer/html/html.go +++ b/renderer/html/html.go @@ -352,14 +352,14 @@ func (r *Renderer) renderAutoLink(w util.BufWriter, source []byte, node ast.Node return ast.WalkContinue, nil } w.WriteString(``) - w.Write(util.EscapeHTML(value)) + w.Write(util.EscapeHTML(label)) w.WriteString(``) return ast.WalkContinue, nil } @@ -493,7 +493,7 @@ func (r *Renderer) RenderAttributes(w util.BufWriter, node ast.Node) { w.WriteString(" ") w.Write(attr.Name) w.WriteString(`="`) - w.Write(attr.Value) + w.Write(util.EscapeHTML(attr.Value)) w.WriteByte('"') } } diff --git a/testutil.go b/testutil.go new file mode 100644 index 0000000..4917e58 --- /dev/null +++ b/testutil.go @@ -0,0 +1,103 @@ +package goldmark + +import ( + "bufio" + "bytes" + "fmt" + "github.com/yuin/goldmark/util" + "os" + "strconv" + "strings" + testing "testing" +) + +type MarkdownTestCase struct { + No int + Markdown string + Expected string +} + +const attributeSeparator = "//- - - - - - - - -//" +const caseSeparator = "//= = = = = = = = = = = = = = = = = = = = = = = =//" + +func DoTestCaseFile(m Markdown, filename string, t *testing.T) { + fp, err := os.Open(filename) + if err != nil { + panic(err) + } + defer fp.Close() + + scanner := bufio.NewScanner(fp) + c := MarkdownTestCase{ + No: -1, + Markdown: "", + Expected: "", + } + cases := []MarkdownTestCase{} + line := 0 + for scanner.Scan() { + line++ + if util.IsBlank([]byte(scanner.Text())) { + continue + } + c.No, err = strconv.Atoi(scanner.Text()) + if err != nil { + panic(fmt.Sprintf("%s: invalid case No at line %d", filename, line)) + } + if !scanner.Scan() { + panic(fmt.Sprintf("%s: invalid case at line %d", filename, line)) + } + line++ + if scanner.Text() != attributeSeparator { + panic(fmt.Sprintf("%s: invalid separator '%s' at line %d", filename, scanner.Text(), line)) + } + buf := []string{} + for scanner.Scan() { + line++ + text := scanner.Text() + if text == attributeSeparator { + break + } + buf = append(buf, text) + } + c.Markdown = strings.Join(buf, "\n") + buf = []string{} + for scanner.Scan() { + line++ + text := scanner.Text() + if text == caseSeparator { + break + } + buf = append(buf, text) + } + c.Expected = strings.Join(buf, "\n") + cases = append(cases, c) + } + DoTestCases(m, cases, t) +} + +func DoTestCases(m Markdown, cases []MarkdownTestCase, t *testing.T) { + for _, testCase := range cases { + var out bytes.Buffer + if err := m.Convert([]byte(testCase.Markdown), &out); err != nil { + panic(err) + } + if !bytes.Equal(bytes.TrimSpace(out.Bytes()), bytes.TrimSpace([]byte(testCase.Expected))) { + format := `============= case %d ================ +Markdown: +----------- +%s + +Expected: +---------- +%s + +Actual +--------- +%s +` + t.Errorf(format, testCase.No, testCase.Markdown, testCase.Expected, out.Bytes()) + } + + } +}