Add extension tests, Fix bugs in extensions

README.md

@@ -93,7 +93,6 @@ Parser and Renderer options
 | `parser.WithParagraphTransformers` | A `util.PrioritizedSlice` whose elements are `parser.ParagraphTransformer` | Transformers for transforming paragraph nodes. | 
 | `parser.WithAutoHeadingID` | `-` | Enables auto heading ids. |
 | `parser.WithAttribute` | `-` | Enables custom attributes. Currently only headings supports attributes. |
-| `parser.WithFilterTags` | `...string` | HTML tag names forbidden in HTML blocks and Raw HTMLs. |
 
 ### HTML Renderer options
 
@@ -116,7 +115,8 @@ Parser and Renderer options
   - [Gitmark Flavored Markdown: Task list items](https://github.github.com/gfm/#task-list-items-extension-)
 - `extension.GFM`
   - This extension enables Table, Strikethrough, Linkify and TaskList.
-    In addition, this extension sets some tags to `parser.FilterTags` .
+  - This extension does not filter tags defined in [6.11Disallowed Raw HTML (extension)](https://github.github.com/gfm/#disallowed-raw-html-extension-).
+    If you need to filter HTML tags, see [Security](#security)
 - `extension.DefinitionList`
   - [PHP Markdown Extra: Definition lists](https://michelf.ca/projects/php-markdown/extra/#def-list)
 - `extension.Footnote`

_test/options.txt

@@ -0,0 +1,27 @@
+1
+//- - - - - - - - -//
+## Title 0
+
+## Title1 # {#id_1 .class-1}
+
+## Title2 {#id_2}
+
+## Title3 ## {#id_3 .class-3}
+
+## Title4 ## {attr3=value3}
+
+## Title5 ## {#id_5 attr5=value5}
+
+## Title6 ## {#id_6 .class6 attr6=value6}
+
+## Title7 ## {#id_7 attr7="value \"7"}
+//- - - - - - - - -//
+<h2 id="title-0">Title 0</h2>
+<h2 id="id_1" class="class-1">Title1</h2>
+<h2 id="id_2">Title2</h2>
+<h2 id="id_3" class="class-3">Title3</h2>
+<h2 attr3="value3" id="title4">Title4</h2>
+<h2 id="id_5" attr5="value5">Title5</h2>
+<h2 id="id_6" class="class6" attr6="value6">Title6</h2>
+<h2 id="id_7" attr7="value &quot;7">Title7</h2>
+//= = = = = = = = = = = = = = = = = = = = = = = =//

ast/inline.go

@@ -362,10 +362,13 @@ const (
 // An AutoLink struct represents an autolink of the Markdown text.
 type AutoLink struct {
 	BaseInline
-	// Value is a link text of this node.
-	Value *Text
 	// Type is a type of this autolink.
 	AutoLinkType AutoLinkType
+
+	// Protocol specified a protocol of the link.
+	Protocol []byte
+
+	value *Text
 }
 
 // Inline implements Inline.Inline.
@@ -373,7 +376,7 @@ func (n *AutoLink) Inline() {}
 
 // Dump implenets Node.Dump
 func (n *AutoLink) Dump(source []byte, level int) {
-	segment := n.Value.Segment
+	segment := n.value.Segment
 	m := map[string]string{
 		"Value": string(segment.Value(source)),
 	}
@@ -388,11 +391,29 @@ func (n *AutoLink) Kind() NodeKind {
 	return KindAutoLink
 }
 
+// URL returns an url of this node.
+func (n *AutoLink) URL(source []byte) []byte {
+	if n.Protocol != nil {
+		s := n.value.Segment
+		ret := make([]byte, 0, len(n.Protocol)+s.Len()+3)
+		ret = append(ret, n.Protocol...)
+		ret = append(ret, ':', '/', '/')
+		ret = append(ret, n.value.Text(source)...)
+		return ret
+	}
+	return n.value.Text(source)
+}
+
+// Label returns a label of this node.
+func (n *AutoLink) Label(source []byte) []byte {
+	return n.value.Text(source)
+}
+
 // NewAutoLink returns a new AutoLink node.
 func NewAutoLink(typ AutoLinkType, value *Text) *AutoLink {
 	return &AutoLink{
 		BaseInline:   BaseInline{},
-		Value:        value,
+		value:        value,
 		AutoLinkType: typ,
 	}
 }

commonmark_test.go

@@ -1,7 +1,6 @@
 package goldmark
 
 import (
-	"bytes"
 	"encoding/json"
 	"io/ioutil"
 	"testing"
@@ -27,30 +26,17 @@ func TestSpec(t *testing.T) {
 	if err := json.Unmarshal(bs, &testCases); err != nil {
 		panic(err)
 	}
+	cases := []MarkdownTestCase{}
+	for _, c := range testCases {
+		cases = append(cases, MarkdownTestCase{
+			No:       c.Example,
+			Markdown: c.Markdown,
+			Expected: c.HTML,
+		})
+	}
 	markdown := New(WithRendererOptions(
 		html.WithXHTML(),
 		html.WithUnsafe(),
 	))
-	for _, testCase := range testCases {
+	DoTestCases(markdown, cases, t)
-		var out bytes.Buffer
-		if err := markdown.Convert([]byte(testCase.Markdown), &out); err != nil {
-			panic(err)
-		}
-		if !bytes.Equal(bytes.TrimSpace(out.Bytes()), bytes.TrimSpace([]byte(testCase.HTML))) {
-			format := `============= case %d ================
-Markdown:
------------
-%s
-
-Expected:
-----------
-%s
-
-Actual
----------
-%s
-`
-			t.Errorf(format, testCase.Example, testCase.Markdown, testCase.HTML, out.Bytes())
-		}
-	}
 }

extension/_test/definition_list.txt

@@ -0,0 +1,143 @@
+1
+//- - - - - - - - -//
+Apple
+:   Pomaceous fruit of plants of the genus Malus in 
+the family Rosaceae.
+
+Orange
+:   The fruit of an evergreen tree of the genus Citrus.
+//- - - - - - - - -//
+<dl>
+<dt>Apple</dt>
+<dd>Pomaceous fruit of plants of the genus Malus in
+the family Rosaceae.</dd>
+<dt>Orange</dt>
+<dd>The fruit of an evergreen tree of the genus Citrus.</dd>
+</dl>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+2
+//- - - - - - - - -//
+Apple
+:   Pomaceous fruit of plants of the genus Malus in 
+    the family Rosaceae.
+:   An American computer company.
+
+Orange
+:   The fruit of an evergreen tree of the genus Citrus.
+//- - - - - - - - -//
+<dl>
+<dt>Apple</dt>
+<dd>Pomaceous fruit of plants of the genus Malus in
+the family Rosaceae.</dd>
+<dd>An American computer company.</dd>
+<dt>Orange</dt>
+<dd>The fruit of an evergreen tree of the genus Citrus.</dd>
+</dl>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+3
+//- - - - - - - - -//
+Term 1
+Term 2
+:   Definition a
+
+Term 3
+:   Definition b
+//- - - - - - - - -//
+<dl>
+<dt>Term 1</dt>
+<dt>Term 2</dt>
+<dd>Definition a</dd>
+<dt>Term 3</dt>
+<dd>Definition b</dd>
+</dl>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+4
+//- - - - - - - - -//
+Apple
+
+:   Pomaceous fruit of plants of the genus Malus in 
+    the family Rosaceae.
+
+Orange
+
+:    The fruit of an evergreen tree of the genus Citrus.
+//- - - - - - - - -//
+<dl>
+<dt>Apple</dt>
+<dd>
+<p>Pomaceous fruit of plants of the genus Malus in
+the family Rosaceae.</p>
+</dd>
+<dt>Orange</dt>
+<dd>
+<p>The fruit of an evergreen tree of the genus Citrus.</p>
+</dd>
+</dl>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+5
+//- - - - - - - - -//
+Term 1
+
+:   This is a definition with two paragraphs. Lorem ipsum 
+    dolor sit amet, consectetuer adipiscing elit. Aliquam 
+    hendrerit mi posuere lectus.
+
+    Vestibulum enim wisi, viverra nec, fringilla in, laoreet
+    vitae, risus.
+
+:   Second definition for term 1, also wrapped in a paragraph
+    because of the blank line preceding it.
+
+Term 2
+
+:   This definition has a code block, a blockquote and a list.
+
+        code block.
+
+    > block quote
+    > on two lines.
+
+    1.  first list item
+    2.  second list item
+//- - - - - - - - -//
+<dl>
+<dt>Term 1</dt>
+<dd>
+<p>This is a definition with two paragraphs. Lorem ipsum
+dolor sit amet, consectetuer adipiscing elit. Aliquam
+hendrerit mi posuere lectus.</p>
+<p>Vestibulum enim wisi, viverra nec, fringilla in, laoreet
+vitae, risus.</p>
+</dd>
+<dd>
+<p>Second definition for term 1, also wrapped in a paragraph
+because of the blank line preceding it.</p>
+</dd>
+<dt>Term 2</dt>
+<dd>
+<p>This definition has a code block, a blockquote and a list.</p>
+<pre><code>code block.
+</code></pre>
+<blockquote>
+<p>block quote
+on two lines.</p>
+</blockquote>
+<ol>
+<li>first list item</li>
+<li>second list item</li>
+</ol>
+</dd>
+</dl>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+

extension/_test/footnote.txt

@@ -0,0 +1,22 @@
+1
+//- - - - - - - - -//
+That's some text with a footnote.[^1]
+
+[^1]: And that's the footnote.
+
+    That's the second paragraph.
+//- - - - - - - - -//
+<p>That's some text with a footnote.<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup></p>
+<section class="footnotes" role="doc-endnotes">
+<hr>
+<ol>
+<li id="fn:1" role="doc-endnote">
+<p>And that's the footnote.</p>
+<p>That's the second paragraph.</p>
+</li>
+</ol>
+<section>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+

extension/_test/linkify.txt

@@ -0,0 +1,114 @@
+1
+//- - - - - - - - -//
+www.commonmark.org
+//- - - - - - - - -//
+<p><a href="http://www.commonmark.org">www.commonmark.org</a></p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+2
+//- - - - - - - - -//
+Visit www.commonmark.org/help for more information.
+//- - - - - - - - -//
+<p>Visit <a href="http://www.commonmark.org/help">www.commonmark.org/help</a> for more information.</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+3
+//- - - - - - - - -//
+www.google.com/search?q=Markup+(business)
+
+www.google.com/search?q=Markup+(business)))
+
+(www.google.com/search?q=Markup+(business))
+
+(www.google.com/search?q=Markup+(business)
+//- - - - - - - - -//
+<p><a href="http://www.google.com/search?q=Markup+(business)">www.google.com/search?q=Markup+(business)</a></p>
+<p><a href="http://www.google.com/search?q=Markup+(business)">www.google.com/search?q=Markup+(business)</a>))</p>
+<p>(<a href="http://www.google.com/search?q=Markup+(business)">www.google.com/search?q=Markup+(business)</a>)</p>
+<p>(<a href="http://www.google.com/search?q=Markup+(business)">www.google.com/search?q=Markup+(business)</a></p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+4
+//- - - - - - - - -//
+www.google.com/search?q=(business))+ok
+//- - - - - - - - -//
+<p><a href="http://www.google.com/search?q=(business))+ok">www.google.com/search?q=(business))+ok</a></p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+5
+//- - - - - - - - -//
+www.google.com/search?q=commonmark&hl=en
+
+www.google.com/search?q=commonmark&hl;
+//- - - - - - - - -//
+<p><a href="http://www.google.com/search?q=commonmark&amp;hl=en">www.google.com/search?q=commonmark&amp;hl=en</a></p>
+<p><a href="http://www.google.com/search?q=commonmark">www.google.com/search?q=commonmark</a>&amp;hl;</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+6
+//- - - - - - - - -//
+www.commonmark.org/he<lp
+//- - - - - - - - -//
+<p><a href="http://www.commonmark.org/he">www.commonmark.org/he</a>&lt;lp</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+7
+//- - - - - - - - -//
+http://commonmark.org
+
+(Visit https://encrypted.google.com/search?q=Markup+(business))
+
+Anonymous FTP is available at ftp://foo.bar.baz.
+//- - - - - - - - -//
+<p><a href="http://commonmark.org">http://commonmark.org</a></p>
+<p>(Visit <a href="https://encrypted.google.com/search?q=Markup+(business)">https://encrypted.google.com/search?q=Markup+(business)</a>)</p>
+<p>Anonymous FTP is available at <a href="ftp://foo.bar.baz">ftp://foo.bar.baz</a>.</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+8
+//- - - - - - - - -//
+foo@bar.baz
+//- - - - - - - - -//
+<p><a href="mailto:foo@bar.baz">foo@bar.baz</a></p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+9
+//- - - - - - - - -//
+hello@mail+xyz.example isn't valid, but hello+xyz@mail.example is.
+//- - - - - - - - -//
+<p>hello@mail+xyz.example isn't valid, but <a href="mailto:hello+xyz@mail.example">hello+xyz@mail.example</a> is.</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+10
+//- - - - - - - - -//
+a.b-c_d@a.b
+
+a.b-c_d@a.b.
+
+a.b-c_d@a.b-
+
+a.b-c_d@a.b_
+//- - - - - - - - -//
+<p><a href="mailto:a.b-c_d@a.b">a.b-c_d@a.b</a></p>
+<p><a href="mailto:a.b-c_d@a.b">a.b-c_d@a.b</a>.</p>
+<p>a.b-c_d@a.b-</p>
+<p>a.b-c_d@a.b_</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//

extension/_test/strikethrough.txt

@@ -0,0 +1,18 @@
+1
+//- - - - - - - - -//
+~~Hi~~ Hello, world!
+//- - - - - - - - -//
+<p><del>Hi</del> Hello, world!</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+2
+//- - - - - - - - -//
+This ~~has a
+
+new paragraph~~.
+//- - - - - - - - -//
+<p>This ~~has a</p>
+<p>new paragraph~~.</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//

extension/_test/table.txt

@@ -0,0 +1,190 @@
+1
+//- - - - - - - - -//
+| foo | bar |
+| --- | --- |
+| baz | bim |
+//- - - - - - - - -//
+<table>
+<thead>
+<tr>
+<th>foo</th>
+<th>bar</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>baz</td>
+<td>bim</td>
+</tr>
+</tbody>
+</table>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+2
+//- - - - - - - - -//
+| abc | defghi |
+:-: | -----------:
+bar | baz
+//- - - - - - - - -//
+<table>
+<thead>
+<tr>
+<th align="center">abc</th>
+<th align="right">defghi</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="center">bar</td>
+<td align="right">baz</td>
+</tr>
+</tbody>
+</table>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+3
+//- - - - - - - - -//
+| f\|oo  |
+| ------ |
+| b `\|` az |
+| b **\|** im |
+//- - - - - - - - -//
+<table>
+<thead>
+<tr>
+<th>f|oo</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>b <code>\|</code> az</td>
+</tr>
+<tr>
+<td>b <strong>|</strong> im</td>
+</tr>
+</tbody>
+</table>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+4
+//- - - - - - - - -//
+| abc | def |
+| --- | --- |
+| bar | baz |
+> bar
+//- - - - - - - - -//
+<table>
+<thead>
+<tr>
+<th>abc</th>
+<th>def</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>bar</td>
+<td>baz</td>
+</tr>
+</tbody>
+</table>
+<blockquote>
+<p>bar</p>
+</blockquote>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+5
+//- - - - - - - - -//
+| abc | def |
+| --- | --- |
+| bar | baz |
+bar
+
+bar
+//- - - - - - - - -//
+<table>
+<thead>
+<tr>
+<th>abc</th>
+<th>def</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>bar</td>
+<td>baz</td>
+</tr>
+<tr>
+<td>bar</td>
+<td></td>
+</tr>
+</tbody>
+</table>
+<p>bar</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+6
+//- - - - - - - - -//
+| abc | def |
+| --- |
+| bar |
+//- - - - - - - - -//
+<p>| abc | def |
+| --- |
+| bar |</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+7
+//- - - - - - - - -//
+| abc | def |
+| --- | --- |
+| bar |
+| bar | baz | boo |
+//- - - - - - - - -//
+<table>
+<thead>
+<tr>
+<th>abc</th>
+<th>def</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>bar</td>
+<td></td>
+</tr>
+<tr>
+<td>bar</td>
+<td>baz</td>
+</tr>
+</tbody>
+</table>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+8
+//- - - - - - - - -//
+| abc | def |
+| --- | --- |
+//- - - - - - - - -//
+<table>
+<thead>
+<tr>
+<th>abc</th>
+<th>def</th>
+</tr>
+</thead>
+</table>
+//= = = = = = = = = = = = = = = = = = = = = = = =//

extension/_test/tasklist.txt

@@ -0,0 +1,30 @@
+1
+//- - - - - - - - -//
+- [ ] foo
+- [x] bar
+//- - - - - - - - -//
+<ul>
+<li><input disabled="" type="checkbox">foo</li>
+<li><input checked="" disabled="" type="checkbox">bar</li>
+</ul>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+
+
+2
+//- - - - - - - - -//
+- [x] foo
+  - [ ] bar
+  - [x] baz
+- [ ] bim
+//- - - - - - - - -//
+<ul>
+<li><input checked="" disabled="" type="checkbox">foo
+<ul>
+<li><input disabled="" type="checkbox">bar</li>
+<li><input checked="" disabled="" type="checkbox">baz</li>
+</ul>
+</li>
+<li><input disabled="" type="checkbox">bim</li>
+</ul>
+//= = = = = = = = = = = = = = = = = = = = = = = =//

extension/_test/typographer.txt

@@ -0,0 +1,20 @@
+1
+//- - - - - - - - -//
+This should 'be' replaced
+//- - - - - - - - -//
+<p>This should &lsquo;be&rsquo; replaced</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+2
+//- - - - - - - - -//
+This should "be" replaced
+//- - - - - - - - -//
+<p>This should &ldquo;be&rdquo; replaced</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//
+
+3
+//- - - - - - - - -//
+**--** *---* a...<< b>>
+//- - - - - - - - -//
+<p><strong>&ndash;</strong> <em>&mdash;</em> a&hellip;&laquo; b&raquo;</p>
+//= = = = = = = = = = = = = = = = = = = = = = = =//

extension/ast/table.go

@@ -102,7 +102,8 @@ func NewTableRow(alignments []Alignment) *TableRow {
 
 // A TableHeader struct represents a table header of Markdown(GFM) text.
 type TableHeader struct {
-	*TableRow
+	gast.BaseBlock
+	Alignments []Alignment
 }
 
 // KindTableHeader is a NodeKind of the TableHeader node.
@@ -113,9 +114,20 @@ func (n *TableHeader) Kind() gast.NodeKind {
 	return KindTableHeader
 }
 
+// Dump implements Node.Dump.
+func (n *TableHeader) Dump(source []byte, level int) {
+	gast.DumpHelper(n, source, level, nil, nil)
+}
+
 // NewTableHeader returns a new TableHeader node.
 func NewTableHeader(row *TableRow) *TableHeader {
-	return &TableHeader{row}
+	n := &TableHeader{}
+	for c := row.FirstChild(); c != nil; {
+		next := c.NextSibling()
+		n.AppendChild(n, c)
+		c = next
+	}
+	return n
 }
 
 // A TableCell struct represents a table cell of a Markdown(GFM) text.

extension/definition_list_test.go

@@ -0,0 +1,19 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"testing"
+)
+
+func TestDefinitionList(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+		),
+		goldmark.WithExtensions(
+			DefinitionList,
+		),
+	)
+	goldmark.DoTestCaseFile(markdown, "_test/definition_list.txt", t)
+}

extension/footnote_test.go

@@ -0,0 +1,19 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"testing"
+)
+
+func TestFootnote(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+		),
+		goldmark.WithExtensions(
+			Footnote,
+		),
+	)
+	goldmark.DoTestCaseFile(markdown, "_test/footnote.txt", t)
+}

extension/gfm.go

@@ -2,7 +2,6 @@ package extension
 
 import (
 	"github.com/yuin/goldmark"
-	"github.com/yuin/goldmark/parser"
 )
 
 type gfm struct {
@@ -11,20 +10,7 @@ type gfm struct {
 // GFM is an extension that provides Github Flavored markdown functionalities.
 var GFM = &gfm{}
 
-var filterTags = []string{
-	"title",
-	"textarea",
-	"style",
-	"xmp",
-	"iframe",
-	"noembed",
-	"noframes",
-	"script",
-	"plaintext",
-}
-
 func (e *gfm) Extend(m goldmark.Markdown) {
-	m.Parser().AddOptions(parser.WithFilterTags(filterTags...))
 	Linkify.Extend(m)
 	Table.Extend(m)
 	Strikethrough.Extend(m)

extension/linkify.go

@@ -48,13 +48,14 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont
 	}
 
 	var m []int
-	typ := ast.AutoLinkType(ast.AutoLinkEmail)
+	var protocol []byte
-	typ = ast.AutoLinkURL
+	var typ ast.AutoLinkType = ast.AutoLinkURL
 	if bytes.HasPrefix(line, protoHTTP) || bytes.HasPrefix(line, protoHTTPS) || bytes.HasPrefix(line, protoFTP) {
 		m = urlRegexp.FindSubmatchIndex(line)
 	}
 	if m == nil && bytes.HasPrefix(line, domainWWW) {
 		m = wwwURLRegxp.FindSubmatchIndex(line)
+		protocol = []byte("http")
 	}
 	if m != nil {
 		lastChar := line[m[1]-1]
@@ -70,7 +71,7 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont
 				}
 			}
 			if closing > 0 {
-				m[1]--
+				m[1] -= closing
 			}
 		} else if lastChar == ';' {
 			i := m[1] - 2
@@ -119,7 +120,9 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont
 	consumes += m[1]
 	block.Advance(consumes)
 	n := ast.NewTextSegment(text.NewSegment(start, start+m[1]))
-	return ast.NewAutoLink(typ, n)
+	link := ast.NewAutoLink(typ, n)
+	link.Protocol = protocol
+	return link
 }
 
 func (s *linkifyParser) CloseBlock(parent ast.Node, pc parser.Context) {

extension/linkify_test.go

@@ -0,0 +1,19 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"testing"
+)
+
+func TestLinkify(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+		),
+		goldmark.WithExtensions(
+			Linkify,
+		),
+	)
+	goldmark.DoTestCaseFile(markdown, "_test/linkify.txt", t)
+}

extension/strikethrough_test.go

@@ -0,0 +1,19 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"testing"
+)
+
+func TestStrikethrough(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+		),
+		goldmark.WithExtensions(
+			Strikethrough,
+		),
+	)
+	goldmark.DoTestCaseFile(markdown, "_test/strikethrough.txt", t)
+}

extension/table.go

@@ -41,7 +41,7 @@ func (b *tableParagraphTransformer) Transform(node *gast.Paragraph, reader text.
 	if alignments == nil {
 		return
 	}
-	header := b.parseRow(lines.At(0), alignments, reader)
+	header := b.parseRow(lines.At(0), alignments, true, reader)
 	if header == nil || len(alignments) != header.ChildCount() {
 		return
 	}
@@ -50,15 +50,14 @@ func (b *tableParagraphTransformer) Transform(node *gast.Paragraph, reader text.
 	table.AppendChild(table, ast.NewTableHeader(header))
 	if lines.Len() > 2 {
 		for i := 2; i < lines.Len(); i++ {
-			table.AppendChild(table, b.parseRow(lines.At(i), alignments, reader))
+			table.AppendChild(table, b.parseRow(lines.At(i), alignments, false, reader))
 		}
 	}
 	node.Parent().InsertBefore(node.Parent(), node, table)
 	node.Parent().RemoveChild(node.Parent(), node)
-	return
 }
 
-func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments []ast.Alignment, reader text.Reader) *ast.TableRow {
+func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments []ast.Alignment, isHeader bool, reader text.Reader) *ast.TableRow {
 	source := reader.Source()
 	line := segment.Value(source)
 	pos := 0
@@ -72,7 +71,16 @@ func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments []
 	if len(line) > 0 && line[limit-1] == '|' {
 		limit--
 	}
-	for i := 0; pos < limit; i++ {
+	i := 0
+	for ; pos < limit; i++ {
+		alignment := ast.AlignNone
+		if i >= len(alignments) {
+			if !isHeader {
+				return row
+			}
+		} else {
+			alignment = alignments[i]
+		}
 		closure := util.FindClosure(line[pos:], byte(0), '|', true, false)
 		if closure < 0 {
 			closure = len(line[pos:])
@@ -82,10 +90,13 @@ func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments []
 		segment = segment.TrimLeftSpace(source)
 		segment = segment.TrimRightSpace(source)
 		node.Lines().Append(segment)
-		node.Alignment = alignments[i]
+		node.Alignment = alignment
 		row.AppendChild(row, node)
 		pos += closure + 1
 	}
+	for ; i < len(alignments); i++ {
+		row.AppendChild(row, ast.NewTableCell())
+	}
 	return row
 }
 
@@ -175,9 +186,6 @@ func (r *TableHTMLRenderer) renderTableHeader(w util.BufWriter, source []byte, n
 		if n.NextSibling() != nil {
 			w.WriteString("<tbody>\n")
 		}
-		if n.Parent().LastChild() == n {
-			w.WriteString("</tbody>\n")
-		}
 	}
 	return gast.WalkContinue, nil
 }
@@ -197,7 +205,7 @@ func (r *TableHTMLRenderer) renderTableRow(w util.BufWriter, source []byte, n ga
 func (r *TableHTMLRenderer) renderTableCell(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) {
 	n := node.(*ast.TableCell)
 	tag := "td"
-	if n.Parent().Parent().FirstChild() == n.Parent() {
+	if n.Parent().Kind() == ast.KindTableHeader {
 		tag = "th"
 	}
 	if entering {

extension/table_test.go

@@ -0,0 +1,19 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"testing"
+)
+
+func TestTable(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+		),
+		goldmark.WithExtensions(
+			Table,
+		),
+	)
+	goldmark.DoTestCaseFile(markdown, "_test/table.txt", t)
+}

extension/tasklist_test.go

@@ -0,0 +1,19 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"testing"
+)
+
+func TestTaskList(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+		),
+		goldmark.WithExtensions(
+			TaskList,
+		),
+	)
+	goldmark.DoTestCaseFile(markdown, "_test/tasklist.txt", t)
+}

extension/typographer_test.go

@@ -0,0 +1,19 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"testing"
+)
+
+func TestTypographer(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+		),
+		goldmark.WithExtensions(
+			Typographer,
+		),
+	)
+	goldmark.DoTestCaseFile(markdown, "_test/typographer.txt", t)
+}

options_test.go

@@ -0,0 +1,16 @@
+package goldmark
+
+import (
+	"github.com/yuin/goldmark/parser"
+	"testing"
+)
+
+func TestDefinitionList(t *testing.T) {
+	markdown := New(
+		WithParserOptions(
+			parser.WithAttribute(),
+			parser.WithAutoHeadingID(),
+		),
+	)
+	DoTestCaseFile(markdown, "_test/options.txt", t)
+}

parser/atx_heading.go

@@ -125,7 +125,7 @@ func (b *atxHeadingParser) Open(parent ast.Node, reader text.Reader, pc Context)
 						break
 					}
 					node.SetAttribute(line[as+ai[0]:as+ai[1]],
-						line[as+ai[2]:as+ai[3]])
+						util.UnescapePunctuations(line[as+ai[2]:as+ai[3]]))
 					as += ai[3] + skip
 				}
 				for ; as < stop && util.IsSpace(line[as]); as++ {
@@ -208,7 +208,8 @@ func parseLastLineAttributes(node ast.Node, reader text.Reader, pc Context) {
 	indicies := util.FindAttributeIndiciesReverse(line, true)
 	if indicies != nil {
 		for _, index := range indicies {
-			node.SetAttribute(line[index[0]:index[1]], line[index[2]:index[3]])
+			node.SetAttribute(line[index[0]:index[1]],
+				util.UnescapePunctuations(line[index[2]:index[3]]))
 		}
 		lastLine.Stop = lastLine.Start + indicies[0][0] - 1
 		lastLine.TrimRightSpace(reader.Source())

parser/html_block.go

@@ -9,48 +9,6 @@ import (
 	"strings"
 )
 
-// An HTMLConfig struct is a data structure that holds configuration of the renderers related to raw htmls.
-type HTMLConfig struct {
-	FilterTags map[string]bool
-}
-
-// SetOption implements SetOptioner.
-func (b *HTMLConfig) SetOption(name OptionName, value interface{}) {
-	switch name {
-	case optFilterTags:
-		b.FilterTags = value.(map[string]bool)
-	}
-}
-
-// A HTMLOption interface sets options for the raw HTML parsers.
-type HTMLOption interface {
-	Option
-	SetHTMLOption(*HTMLConfig)
-}
-
-const optFilterTags OptionName = "FilterTags"
-
-type withFilterTags struct {
-	value map[string]bool
-}
-
-func (o *withFilterTags) SetParserOption(c *Config) {
-	c.Options[optFilterTags] = o.value
-}
-
-func (o *withFilterTags) SetHTMLOption(p *HTMLConfig) {
-	p.FilterTags = o.value
-}
-
-// WithFilterTags is a functional otpion that specify forbidden tag names.
-func WithFilterTags(names ...string) HTMLOption {
-	m := map[string]bool{}
-	for _, name := range names {
-		m[name] = true
-	}
-	return &withFilterTags{m}
-}
-
 var allowedBlockTags = map[string]bool{
 	"address":    true,
 	"article":    true,
@@ -137,17 +95,14 @@ var htmlBlockType6Regexp = regexp.MustCompile(`^[ ]{0,3}</?([a-zA-Z0-9]+)(?:\s.*
 var htmlBlockType7Regexp = regexp.MustCompile(`^[ ]{0,3}<(/)?([a-zA-Z0-9]+)(` + attributePattern + `*)(:?>|/>)\s*\n?$`)
 
 type htmlBlockParser struct {
-	HTMLConfig
 }
 
+var defaultHtmlBlockParser = &htmlBlockParser{}
+
 // NewHTMLBlockParser return a new BlockParser that can parse html
 // blocks.
-func NewHTMLBlockParser(opts ...HTMLOption) BlockParser {
+func NewHTMLBlockParser() BlockParser {
-	p := &htmlBlockParser{}
+	return defaultHtmlBlockParser
-	for _, o := range opts {
-		o.SetHTMLOption(&p.HTMLConfig)
-	}
-	return p
 }
 
 func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) {
@@ -191,11 +146,6 @@ func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context)
 		}
 	}
 	if node != nil {
-		if b.FilterTags != nil {
-			if _, ok := b.FilterTags[tagName]; ok {
-				return nil, NoChildren
-			}
-		}
 		reader.Advance(segment.Len() - 1)
 		node.Lines().Append(segment)
 		return node, NoChildren

parser/parser.go

@@ -84,6 +84,9 @@ func (s *ids) Generate(value, prefix []byte) []byte {
 			continue
 		}
 		if util.IsAlphaNumeric(v) {
+			if 'A' <= v && v <= 'Z' {
+				v += 'a' - 'A'
+			}
 			result = append(result, v)
 		} else if util.IsSpace(v) {
 			result = append(result, '-')

parser/raw_html.go

@@ -9,17 +9,14 @@ import (
 )
 
 type rawHTMLParser struct {
-	HTMLConfig
 }
 
+var defaultRawHTMLParser = &rawHTMLParser{}
+
 // NewRawHTMLParser return a new InlineParser that can parse
 // inline htmls
-func NewRawHTMLParser(opts ...HTMLOption) InlineParser {
+func NewRawHTMLParser() InlineParser {
-	p := &rawHTMLParser{}
+	return defaultRawHTMLParser
-	for _, o := range opts {
-		o.SetHTMLOption(&p.HTMLConfig)
-	}
-	return p
 }
 
 func (s *rawHTMLParser) Trigger() []byte {
@@ -74,22 +71,7 @@ var dummyMatch = [][]byte{}
 
 func (s *rawHTMLParser) parseMultiLineRegexp(reg *regexp.Regexp, block text.Reader, pc Context) ast.Node {
 	sline, ssegment := block.Position()
-	var m [][]byte
-	if s.FilterTags != nil {
-		m = block.FindSubMatch(reg)
-	} else {
 	if block.Match(reg) {
-			m = dummyMatch
-		}
-	}
-
-	if m != nil {
-		if s.FilterTags != nil && len(m) > 1 {
-			tagName := string(m[1])
-			if _, ok := s.FilterTags[tagName]; ok {
-				return nil
-			}
-		}
 		node := ast.NewRawHTML()
 		eline, esegment := block.Position()
 		block.SetPosition(sline, ssegment)

renderer/html/html.go

@@ -352,14 +352,14 @@ func (r *Renderer) renderAutoLink(w util.BufWriter, source []byte, node ast.Node
 		return ast.WalkContinue, nil
 	}
 	w.WriteString(`<a href="`)
-	segment := n.Value.Segment
+	url := n.URL(source)
-	value := segment.Value(source)
+	label := n.Label(source)
-	if n.AutoLinkType == ast.AutoLinkEmail && !bytes.HasPrefix(bytes.ToLower(value), []byte("mailto:")) {
+	if n.AutoLinkType == ast.AutoLinkEmail && !bytes.HasPrefix(bytes.ToLower(url), []byte("mailto:")) {
 		w.WriteString("mailto:")
 	}
-	w.Write(util.EscapeHTML(util.URLEscape(value, false)))
+	w.Write(util.EscapeHTML(util.URLEscape(url, false)))
 	w.WriteString(`">`)
-	w.Write(util.EscapeHTML(value))
+	w.Write(util.EscapeHTML(label))
 	w.WriteString(`</a>`)
 	return ast.WalkContinue, nil
 }
@@ -493,7 +493,7 @@ func (r *Renderer) RenderAttributes(w util.BufWriter, node ast.Node) {
 		w.WriteString(" ")
 		w.Write(attr.Name)
 		w.WriteString(`="`)
-		w.Write(attr.Value)
+		w.Write(util.EscapeHTML(attr.Value))
 		w.WriteByte('"')
 	}
 }

testutil.go

@@ -0,0 +1,103 @@
+package goldmark
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"github.com/yuin/goldmark/util"
+	"os"
+	"strconv"
+	"strings"
+	testing "testing"
+)
+
+type MarkdownTestCase struct {
+	No       int
+	Markdown string
+	Expected string
+}
+
+const attributeSeparator = "//- - - - - - - - -//"
+const caseSeparator = "//= = = = = = = = = = = = = = = = = = = = = = = =//"
+
+func DoTestCaseFile(m Markdown, filename string, t *testing.T) {
+	fp, err := os.Open(filename)
+	if err != nil {
+		panic(err)
+	}
+	defer fp.Close()
+
+	scanner := bufio.NewScanner(fp)
+	c := MarkdownTestCase{
+		No:       -1,
+		Markdown: "",
+		Expected: "",
+	}
+	cases := []MarkdownTestCase{}
+	line := 0
+	for scanner.Scan() {
+		line++
+		if util.IsBlank([]byte(scanner.Text())) {
+			continue
+		}
+		c.No, err = strconv.Atoi(scanner.Text())
+		if err != nil {
+			panic(fmt.Sprintf("%s: invalid case No at line %d", filename, line))
+		}
+		if !scanner.Scan() {
+			panic(fmt.Sprintf("%s: invalid case at line %d", filename, line))
+		}
+		line++
+		if scanner.Text() != attributeSeparator {
+			panic(fmt.Sprintf("%s: invalid separator '%s' at line %d", filename, scanner.Text(), line))
+		}
+		buf := []string{}
+		for scanner.Scan() {
+			line++
+			text := scanner.Text()
+			if text == attributeSeparator {
+				break
+			}
+			buf = append(buf, text)
+		}
+		c.Markdown = strings.Join(buf, "\n")
+		buf = []string{}
+		for scanner.Scan() {
+			line++
+			text := scanner.Text()
+			if text == caseSeparator {
+				break
+			}
+			buf = append(buf, text)
+		}
+		c.Expected = strings.Join(buf, "\n")
+		cases = append(cases, c)
+	}
+	DoTestCases(m, cases, t)
+}
+
+func DoTestCases(m Markdown, cases []MarkdownTestCase, t *testing.T) {
+	for _, testCase := range cases {
+		var out bytes.Buffer
+		if err := m.Convert([]byte(testCase.Markdown), &out); err != nil {
+			panic(err)
+		}
+		if !bytes.Equal(bytes.TrimSpace(out.Bytes()), bytes.TrimSpace([]byte(testCase.Expected))) {
+			format := `============= case %d ================
+Markdown:
+-----------
+%s
+
+Expected:
+----------
+%s
+
+Actual
+---------
+%s
+`
+			t.Errorf(format, testCase.No, testCase.Markdown, testCase.Expected, out.Bytes())
+		}
+
+	}
+}