Add CJK extension

README.md

@@ -180,6 +180,8 @@ Parser and Renderer options
     - [PHP Markdown Extra: Footnotes](https://michelf.ca/projects/php-markdown/extra/#footnotes)
 - `extension.Typographer`
     - This extension substitutes punctuations with typographic entities like [smartypants](https://daringfireball.net/projects/smartypants/).
+- `extension.CJK`
+    - This extension is a shortcut for CJK related functionalities.
 
 ### Attributes
 The `parser.WithAttribute` option allows you to define attributes on some elements.
@@ -369,6 +371,17 @@ footnote-prefix: article1
 # My article
 
 ```
+
+### CJK extension
+CommonMark gives compatibilities a high priority and original markdown was designed by westerners. So CommonMark lacks considerations for languages like CJK.
+
+This extension provides additional options for CJK users.
+
+| Functional option | Type | Description |
+| ----------------- | ---- | ----------- |
+| `extension.WithEastAsianLineBreaks` | `-` | Soft line breaks are rendered as a newline. Some asian users will see it as an unnecessary space. With this option, soft line breaks between east asian wide characters will be ignored. |
+| `extension.WithEscapedSpace` | `-` | Without spaces around an emphasis started with east asian punctuations, it is not interpreted as an emphasis(as defined in CommonMark spec). With this option, you can avoid this inconvenient behavior by putting 'not rendered' spaces around an emphasis like `太郎は\ **「こんにちわ」**\ といった`. |
+
  
 Security
 --------------------

ast/inline.go

@@ -91,7 +91,7 @@ func (n *Text) SetSoftLineBreak(v bool) {
 	if v {
 		n.flags |= textSoftLineBreak
 	} else {
-		n.flags = n.flags &^ textHardLineBreak
+		n.flags = n.flags &^ textSoftLineBreak
 	}
 }
 

extension/cjk.go

@@ -0,0 +1,49 @@
+package extension
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+)
+
+// A CJKOption sets options for CJK support mostly for HTML based renderers.
+type CJKOption func(*cjk)
+
+// WithEastAsianLineBreaks is a functional option that indicates whether softline breaks
+// between east asian wide characters should be ignored.
+func WithEastAsianLineBreaks() CJKOption {
+	return func(c *cjk) {
+		c.EastAsianLineBreaks = true
+	}
+}
+
+// WithEscapedSpace is a functional option that indicates that a '\' escaped half-space(0x20) should not be rendered.
+func WithEscapedSpace() CJKOption {
+	return func(c *cjk) {
+		c.EscapedSpace = true
+	}
+}
+
+type cjk struct {
+	EastAsianLineBreaks bool
+	EscapedSpace        bool
+}
+
+var CJK = NewCJK(WithEastAsianLineBreaks(), WithEscapedSpace())
+
+// NewCJK returns a new extension with given options.
+func NewCJK(opts ...CJKOption) goldmark.Extender {
+	e := &cjk{}
+	for _, opt := range opts {
+		opt(e)
+	}
+	return e
+}
+
+func (e *cjk) Extend(m goldmark.Markdown) {
+	if e.EastAsianLineBreaks {
+		m.Renderer().AddOptions(html.WithEastAsianLineBreaks())
+	}
+	if e.EscapedSpace {
+		m.Renderer().AddOptions(html.WithWriter(html.NewWriter(html.WithEscapedSpace())))
+	}
+}

extension/cjk_test.go

@@ -0,0 +1,154 @@
+package extension
+
+import (
+	"testing"
+
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/renderer/html"
+	"github.com/yuin/goldmark/testutil"
+)
+
+func TestEscapedSpace(t *testing.T) {
+	markdown := goldmark.New(goldmark.WithRendererOptions(
+		html.WithXHTML(),
+		html.WithUnsafe(),
+	))
+	no := 1
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "Without spaces around an emphasis started with east asian punctuations, it is not interpreted as an emphasis(as defined in CommonMark spec)",
+			Markdown:    "太郎は**「こんにちわ」**と言った\nんです",
+			Expected:    "<p>太郎は**「こんにちわ」**と言った\nんです</p>",
+		},
+		t,
+	)
+
+	no = 2
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "With spaces around an emphasis started with east asian punctuations, it is interpreted as an emphasis(but remains unnecessary spaces)",
+			Markdown:    "太郎は **「こんにちわ」** と言った\nんです",
+			Expected:    "<p>太郎は <strong>「こんにちわ」</strong> と言った\nんです</p>",
+		},
+		t,
+	)
+
+	// Enables EscapedSpace
+	markdown = goldmark.New(goldmark.WithRendererOptions(
+		html.WithXHTML(),
+		html.WithUnsafe(),
+	),
+		goldmark.WithExtensions(NewCJK(WithEscapedSpace())),
+	)
+
+	no = 3
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "With spaces around an emphasis started with east asian punctuations,it is interpreted as an emphasis",
+			Markdown:    "太郎は\\ **「こんにちわ」**\\ と言った\nんです",
+			Expected:    "<p>太郎は<strong>「こんにちわ」</strong>と言った\nんです</p>",
+		},
+		t,
+	)
+}
+
+func TestEastAsianLineBreaks(t *testing.T) {
+	markdown := goldmark.New(goldmark.WithRendererOptions(
+		html.WithXHTML(),
+		html.WithUnsafe(),
+	))
+	no := 1
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "Soft line breaks are rendered as a newline, so some asian users will see it as an unnecessary space",
+			Markdown:    "太郎は\\ **「こんにちわ」**\\ と言った\nんです",
+			Expected:    "<p>太郎は\\ <strong>「こんにちわ」</strong>\\ と言った\nんです</p>",
+		},
+		t,
+	)
+
+	// Enables EastAsianLineBreaks
+
+	markdown = goldmark.New(goldmark.WithRendererOptions(
+		html.WithXHTML(),
+		html.WithUnsafe(),
+	),
+		goldmark.WithExtensions(NewCJK(WithEastAsianLineBreaks())),
+	)
+
+	no = 2
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "Soft line breaks between east asian wide characters are ignored",
+			Markdown:    "太郎は\\ **「こんにちわ」**\\ と言った\nんです",
+			Expected:    "<p>太郎は\\ <strong>「こんにちわ」</strong>\\ と言ったんです</p>",
+		},
+		t,
+	)
+
+	no = 3
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "Soft line breaks between western characters are rendered as a newline",
+			Markdown:    "太郎は\\ **「こんにちわ」**\\ と言ったa\nbんです",
+			Expected:    "<p>太郎は\\ <strong>「こんにちわ」</strong>\\ と言ったa\nbんです</p>",
+		},
+		t,
+	)
+
+	no = 4
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "Soft line breaks between a western character and an east asian wide character are rendered as a newline",
+			Markdown:    "太郎は\\ **「こんにちわ」**\\ と言ったa\nんです",
+			Expected:    "<p>太郎は\\ <strong>「こんにちわ」</strong>\\ と言ったa\nんです</p>",
+		},
+		t,
+	)
+
+	no = 5
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "Soft line breaks between an east asian wide character and a western character are rendered as a newline",
+			Markdown:    "太郎は\\ **「こんにちわ」**\\ と言った\nbんです",
+			Expected:    "<p>太郎は\\ <strong>「こんにちわ」</strong>\\ と言った\nbんです</p>",
+		},
+		t,
+	)
+
+	// WithHardWraps take precedence over WithEastAsianLineBreaks
+	markdown = goldmark.New(goldmark.WithRendererOptions(
+		html.WithHardWraps(),
+		html.WithXHTML(),
+		html.WithUnsafe(),
+	),
+		goldmark.WithExtensions(NewCJK(WithEastAsianLineBreaks())),
+	)
+	no = 6
+	testutil.DoTestCase(
+		markdown,
+		testutil.MarkdownTestCase{
+			No:          no,
+			Description: "WithHardWraps take precedence over WithEastAsianLineBreaks",
+			Markdown:    "太郎は\\ **「こんにちわ」**\\ と言った\nんです",
+			Expected:    "<p>太郎は\\ <strong>「こんにちわ」</strong>\\ と言った<br />\nんです</p>",
+		},
+		t,
+	)
+}

renderer/html/html.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"strconv"
+	"unicode/utf8"
 
 	"github.com/yuin/goldmark/ast"
 	"github.com/yuin/goldmark/renderer"
@@ -12,19 +13,21 @@ import (
 
 // A Config struct has configurations for the HTML based renderers.
 type Config struct {
-	Writer    Writer
+	Writer              Writer
-	HardWraps bool
+	HardWraps           bool
-	XHTML     bool
+	EastAsianLineBreaks bool
-	Unsafe    bool
+	XHTML               bool
+	Unsafe              bool
 }
 
 // NewConfig returns a new Config with defaults.
 func NewConfig() Config {
 	return Config{
-		Writer:    DefaultWriter,
+		Writer:              DefaultWriter,
-		HardWraps: false,
+		HardWraps:           false,
-		XHTML:     false,
+		EastAsianLineBreaks: false,
-		Unsafe:    false,
+		XHTML:               false,
+		Unsafe:              false,
 	}
 }
 
@@ -33,6 +36,8 @@ func (c *Config) SetOption(name renderer.OptionName, value interface{}) {
 	switch name {
 	case optHardWraps:
 		c.HardWraps = value.(bool)
+	case optEastAsianLineBreaks:
+		c.EastAsianLineBreaks = value.(bool)
 	case optXHTML:
 		c.XHTML = value.(bool)
 	case optUnsafe:
@@ -94,6 +99,29 @@ func WithHardWraps() interface {
 	return &withHardWraps{}
 }
 
+// EastAsianLineBreaks is an option name used in WithEastAsianLineBreaks.
+const optEastAsianLineBreaks renderer.OptionName = "EastAsianLineBreaks"
+
+type withEastAsianLineBreaks struct {
+}
+
+func (o *withEastAsianLineBreaks) SetConfig(c *renderer.Config) {
+	c.Options[optEastAsianLineBreaks] = true
+}
+
+func (o *withEastAsianLineBreaks) SetHTMLOption(c *Config) {
+	c.EastAsianLineBreaks = true
+}
+
+// WithEastAsianLineBreaks is a functional option that indicates whether softline breaks
+// between east asian wide characters should be ignored.
+func WithEastAsianLineBreaks() interface {
+	renderer.Option
+	Option
+} {
+	return &withEastAsianLineBreaks{}
+}
+
 // XHTML is an option name used in WithXHTML.
 const optXHTML renderer.OptionName = "XHTML"
 
@@ -615,7 +643,8 @@ func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, en
 	if n.IsRaw() {
 		r.Writer.RawWrite(w, segment.Value(source))
 	} else {
-		r.Writer.Write(w, segment.Value(source))
+		value := segment.Value(source)
+		r.Writer.Write(w, value)
 		if n.HardLineBreak() || (n.SoftLineBreak() && r.HardWraps) {
 			if r.XHTML {
 				_, _ = w.WriteString("<br />\n")
@@ -623,7 +652,21 @@ func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, en
 				_, _ = w.WriteString("<br>\n")
 			}
 		} else if n.SoftLineBreak() {
-			_ = w.WriteByte('\n')
+			if r.EastAsianLineBreaks && len(value) != 0 {
+				sibling := node.NextSibling()
+				if sibling != nil && sibling.Kind() == ast.KindText {
+					if siblingText := sibling.(*ast.Text).Text(source); len(siblingText) != 0 {
+						thisLastRune := util.ToRune(value, len(value)-1)
+						siblingFirstRune, _ := utf8.DecodeRune(siblingText)
+						if !(util.IsEastAsianWideRune(thisLastRune) &&
+							util.IsEastAsianWideRune(siblingFirstRune)) {
+							_ = w.WriteByte('\n')
+						}
+					}
+				}
+			} else {
+				_ = w.WriteByte('\n')
+			}
 		}
 	}
 	return ast.WalkContinue, nil
@@ -683,7 +726,33 @@ type Writer interface {
 
 var replacementCharacter = []byte("\ufffd")
 
+// A WriterConfig struct has configurations for the HTML based writers.
+type WriterConfig struct {
+	// EscapedSpace is an option that indicates that a '\' escaped half-space(0x20) should not be rendered.
+	EscapedSpace bool
+}
+
+// A WriterOption interface sets options for HTML based writers.
+type WriterOption func(*WriterConfig)
+
+// WithEscapedSpace is a WriterOption indicates that a '\' escaped half-space(0x20) should not be rendered.
+func WithEscapedSpace() WriterOption {
+	return func(c *WriterConfig) {
+		c.EscapedSpace = true
+	}
+}
+
 type defaultWriter struct {
+	WriterConfig
+}
+
+// NewWriter returns a new Writer.
+func NewWriter(opts ...WriterOption) Writer {
+	w := &defaultWriter{}
+	for _, opt := range opts {
+		opt(&w.WriterConfig)
+	}
+	return w
 }
 
 func escapeRune(writer util.BufWriter, r rune) {
@@ -746,6 +815,12 @@ func (d *defaultWriter) Write(writer util.BufWriter, source []byte) {
 				escaped = false
 				continue
 			}
+			if d.EscapedSpace && c == ' ' {
+				d.RawWrite(writer, source[n:i-1])
+				n = i + 1
+				escaped = false
+				continue
+			}
 		}
 		if c == '\x00' {
 			d.RawWrite(writer, source[n:i])
@@ -811,8 +886,8 @@ func (d *defaultWriter) Write(writer util.BufWriter, source []byte) {
 	d.RawWrite(writer, source[n:])
 }
 
-// DefaultWriter is a default implementation of the Writer.
+// DefaultWriter is a default instance of the Writer.
-var DefaultWriter = &defaultWriter{}
+var DefaultWriter = NewWriter()
 
 var bDataImage = []byte("data:image/")
 var bPng = []byte("png;")

util/util.go

@@ -184,25 +184,25 @@ func IndentPositionPadding(bs []byte, currentPos, paddingv, width int) (pos, pad
 //
 // Deprecated: This function has bugs. Use util.IndentPositionPadding and util.FirstNonSpacePosition.
 func DedentPosition(bs []byte, currentPos, width int) (pos, padding int) {
-       if width == 0 {
+	if width == 0 {
-               return 0, 0
+		return 0, 0
-       }
+	}
-       w := 0
+	w := 0
-       l := len(bs)
+	l := len(bs)
-       i := 0
+	i := 0
-       for ; i < l; i++ {
+	for ; i < l; i++ {
-               if bs[i] == '\t' {
+		if bs[i] == '\t' {
-                       w += TabWidth(currentPos + w)
+			w += TabWidth(currentPos + w)
-               } else if bs[i] == ' ' {
+		} else if bs[i] == ' ' {
-                       w++
+			w++
-               } else {
+		} else {
-                       break
+			break
-               }
+		}
-       }
+	}
-       if w >= width {
+	if w >= width {
-               return i, w - width
+		return i, w - width
-       }
+	}
-       return i, 0
+	return i, 0
 }
 
 // DedentPositionPadding dedents lines by the given width.
@@ -211,26 +211,26 @@ func DedentPosition(bs []byte, currentPos, width int) (pos, padding int) {
 //
 // Deprecated: This function has bugs. Use util.IndentPositionPadding and util.FirstNonSpacePosition.
 func DedentPositionPadding(bs []byte, currentPos, paddingv, width int) (pos, padding int) {
-       if width == 0 {
+	if width == 0 {
-               return 0, paddingv
+		return 0, paddingv
-       }
+	}
 
-       w := 0
+	w := 0
-       i := 0
+	i := 0
-       l := len(bs)
+	l := len(bs)
-       for ; i < l; i++ {
+	for ; i < l; i++ {
-               if bs[i] == '\t' {
+		if bs[i] == '\t' {
-                       w += TabWidth(currentPos + w)
+			w += TabWidth(currentPos + w)
-               } else if bs[i] == ' ' {
+		} else if bs[i] == ' ' {
-                       w++
+			w++
-               } else {
+		} else {
-                       break
+			break
-               }
+		}
-       }
+	}
-       if w >= width {
+	if w >= width {
-               return i - paddingv, w - width
+		return i - paddingv, w - width
-       }
+	}
-       return i - paddingv, 0
+	return i - paddingv, 0
 }
 
 // IndentWidth calculate an indent width for the given line.
@@ -834,6 +834,15 @@ func IsAlphaNumeric(c byte) bool {
 	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
 }
 
+// IsEastAsianWideRune returns trhe if the given rune is an east asian wide character, otherwise false.
+func IsEastAsianWideRune(r rune) bool {
+	return unicode.Is(unicode.Hiragana, r) ||
+		unicode.Is(unicode.Katakana, r) ||
+		unicode.Is(unicode.Han, r) ||
+		unicode.Is(unicode.Lm, r) ||
+		unicode.Is(unicode.Hangul, r)
+}
+
 // A BufWriter is a subset of the bufio.Writer .
 type BufWriter interface {
 	io.Writer