Browse Source

Refactor renders (#15175)

* Refactor renders

* Some performance optimization

* Fix comment

* Transform reader

* Fix csv test

* Fix test

* Fix tests

* Improve optimaziation

* Fix test

* Fix test

* Detect file encoding with reader

* Improve optimaziation

* reduce memory usage

* improve code

* fix build

* Fix test

* Fix for go1.15

* Fix render

* Fix comment

* Fix lint

* Fix test

* Don't use NormalEOF when unnecessary

* revert change on util.go

* Apply suggestions from code review

Co-authored-by: zeripath <art27@cantab.net>

* rename function

* Take NormalEOF back

Co-authored-by: zeripath <art27@cantab.net>
pull/15394/merge
Lunny Xiao 4 weeks ago
committed by GitHub
parent
commit
9d99f6ab19
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 1026 additions and 626 deletions
  1. +1
    -1
      contrib/pr/checkout.go
  2. +8
    -2
      models/issue_comment.go
  3. +4
    -1
      models/repo.go
  4. +5
    -4
      models/repo_generate.go
  5. +31
    -18
      modules/charset/charset.go
  6. +21
    -5
      modules/csv/csv.go
  7. +5
    -2
      modules/csv/csv_test.go
  8. +84
    -39
      modules/markup/csv/csv.go
  9. +8
    -3
      modules/markup/csv/csv_test.go
  10. +26
    -34
      modules/markup/external/external.go
  11. +98
    -138
      modules/markup/html.go
  12. +32
    -27
      modules/markup/html_internal_test.go
  13. +50
    -14
      modules/markup/html_test.go
  14. +60
    -36
      modules/markup/markdown/markdown.go
  15. +52
    -21
      modules/markup/markdown/markdown_test.go
  16. +0
    -143
      modules/markup/markup.go
  17. +32
    -28
      modules/markup/orgmode/orgmode.go
  18. +9
    -2
      modules/markup/orgmode/orgmode_test.go
  19. +201
    -0
      modules/markup/renderer.go
  20. +0
    -0
      modules/markup/renderer_test.go
  21. +6
    -2
      modules/notification/mail/mail.go
  22. +6
    -6
      modules/setting/markup.go
  23. +23
    -6
      modules/templates/helper.go
  24. +14
    -24
      routers/api/v1/misc/markdown.go
  25. +1
    -1
      routers/init.go
  26. +10
    -1
      routers/org/home.go
  27. +1
    -9
      routers/repo/compare.go
  28. +46
    -9
      routers/repo/issue.go
  29. +3
    -10
      routers/repo/lfs.go
  30. +17
    -2
      routers/repo/milestone.go
  31. +17
    -2
      routers/repo/projects.go
  32. +17
    -2
      routers/repo/release.go
  33. +41
    -10
      routers/repo/view.go
  34. +27
    -4
      routers/repo/wiki.go
  35. +10
    -1
      routers/user/home.go
  36. +10
    -1
      routers/user/profile.go
  37. +8
    -2
      services/gitdiff/csv_test.go
  38. +21
    -11
      services/mailer/mail.go
  39. +5
    -1
      services/mailer/mail_issue.go
  40. +10
    -1
      services/mailer/mail_release.go
  41. +6
    -3
      services/mailer/mail_test.go

+ 1
- 1
contrib/pr/checkout.go View File

@ -114,7 +114,7 @@ func runPR() {
log.Printf("[PR] Setting up router\n")
//routers.GlobalInit()
external.RegisterParsers()
external.RegisterRenderers()
markup.Init()
c := routes.NormalRoutes()


+ 8
- 2
models/issue_comment.go View File

@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/references"
"code.gitea.io/gitea/modules/structs"
@ -1178,8 +1179,13 @@ func findCodeComments(e Engine, opts FindCommentsOptions, issue *Issue, currentU
return nil, err
}
comment.RenderedContent = string(markdown.Render([]byte(comment.Content), issue.Repo.Link(),
issue.Repo.ComposeMetas()))
var err error
if comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
URLPrefix: issue.Repo.Link(),
Metas: issue.Repo.ComposeMetas(),
}, comment.Content); err != nil {
return nil, err
}
}
return comments[:n], nil
}


+ 4
- 1
models/repo.go View File

@ -863,7 +863,10 @@ func (repo *Repository) getUsersWithAccessMode(e Engine, mode AccessMode) (_ []*
// DescriptionHTML does special handles to description and return HTML string.
func (repo *Repository) DescriptionHTML() template.HTML {
desc, err := markup.RenderDescriptionHTML([]byte(repo.Description), repo.HTMLURL(), repo.ComposeMetas())
desc, err := markup.RenderDescriptionHTML(&markup.RenderContext{
URLPrefix: repo.HTMLURL(),
Metas: repo.ComposeMetas(),
}, repo.Description)
if err != nil {
log.Error("Failed to render description for %s (ID: %d): %v", repo.Name, repo.ID, err)
return template.HTML(markup.Sanitize(repo.Description))


+ 5
- 4
models/repo_generate.go View File

@ -5,13 +5,14 @@
package models
import (
"bufio"
"bytes"
"strconv"
"strings"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
"github.com/gobwas/glob"
)
@ -49,9 +50,9 @@ func (gt GiteaTemplate) Globs() []glob.Glob {
}
gt.globs = make([]glob.Glob, 0)
lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
scanner := bufio.NewScanner(bytes.NewReader(gt.Content))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}


+ 31
- 18
modules/charset/charset.go View File

@ -7,6 +7,8 @@ package charset
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"strings"
"unicode/utf8"
@ -21,6 +23,33 @@ import (
// UTF8BOM is the utf-8 byte-order marker
var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
var buf = make([]byte, 2048)
n, err := rd.Read(buf)
if err != nil {
return rd
}
charsetLabel, err := DetectEncoding(buf[:n])
if err != nil || charsetLabel == "UTF-8" {
return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
}
encoding, _ := charset.Lookup(charsetLabel)
if encoding == nil {
return io.MultiReader(bytes.NewReader(buf[:n]), rd)
}
return transform.NewReader(
io.MultiReader(
bytes.NewReader(RemoveBOMIfPresent(buf[:n])),
rd,
),
encoding.NewDecoder(),
)
}
// ToUTF8WithErr converts content to UTF8 encoding
func ToUTF8WithErr(content []byte) (string, error) {
charsetLabel, err := DetectEncoding(content)
@ -49,24 +78,8 @@ func ToUTF8WithErr(content []byte) (string, error) {
// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
func ToUTF8WithFallback(content []byte) []byte {
charsetLabel, err := DetectEncoding(content)
if err != nil || charsetLabel == "UTF-8" {
return RemoveBOMIfPresent(content)
}
encoding, _ := charset.Lookup(charsetLabel)
if encoding == nil {
return content
}
// If there is an error, we concatenate the nicely decoded part and the
// original left over. This way we won't lose data.
result, n, err := transform.Bytes(encoding.NewDecoder(), content)
if err != nil {
return append(result, content[n:]...)
}
return RemoveBOMIfPresent(result)
bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))
return bs
}
// ToUTF8 converts content to UTF8 encoding and ignore error


+ 21
- 5
modules/csv/csv.go View File

@ -7,7 +7,9 @@ package csv
import (
"bytes"
"encoding/csv"
stdcsv "encoding/csv"
"errors"
"io"
"regexp"
"strings"
@ -18,17 +20,31 @@ import (
var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
// CreateReader creates a csv.Reader with the given delimiter.
func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
rd := csv.NewReader(bytes.NewReader(rawBytes))
func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
rd := stdcsv.NewReader(input)
rd.Comma = delimiter
rd.TrimLeadingSpace = true
return rd
}
// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
delimiter := guessDelimiter(rawBytes)
return CreateReader(rawBytes, delimiter)
func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
var data = make([]byte, 1e4)
size, err := rd.Read(data)
if err != nil {
return nil, err
}
delimiter := guessDelimiter(data[:size])
var newInput io.Reader
if size < 1e4 {
newInput = bytes.NewReader(data[:size])
} else {
newInput = io.MultiReader(bytes.NewReader(data), rd)
}
return CreateReader(newInput, delimiter), nil
}
// guessDelimiter scores the input CSV data against delimiters, and returns the best match.


+ 5
- 2
modules/csv/csv_test.go View File

@ -5,20 +5,23 @@
package csv
import (
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCreateReader(t *testing.T) {
rd := CreateReader([]byte{}, ',')
rd := CreateReader(bytes.NewReader([]byte{}), ',')
assert.Equal(t, ',', rd.Comma)
}
func TestCreateReaderAndGuessDelimiter(t *testing.T) {
input := "a;b;c\n1;2;3\n4;5;6"
rd := CreateReaderAndGuessDelimiter([]byte(input))
rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input))
assert.NoError(t, err)
assert.Equal(t, ';', rd.Comma)
}


+ 84
- 39
modules/markup/csv/csv.go View File

@ -5,9 +5,11 @@
package markup
import (
"bufio"
"bytes"
"html"
"io"
"io/ioutil"
"strconv"
"code.gitea.io/gitea/modules/csv"
@ -16,55 +18,89 @@ import (
)
func init() {
markup.RegisterParser(Parser{})
markup.RegisterRenderer(Renderer{})
}
// Parser implements markup.Parser for csv files
type Parser struct {
// Renderer implements markup.Renderer for csv files
type Renderer struct {
}
// Name implements markup.Parser
func (Parser) Name() string {
// Name implements markup.Renderer
func (Renderer) Name() string {
return "csv"
}
// NeedPostProcess implements markup.Parser
func (Parser) NeedPostProcess() bool { return false }
// NeedPostProcess implements markup.Renderer
func (Renderer) NeedPostProcess() bool { return false }
// Extensions implements markup.Parser
func (Parser) Extensions() []string {
// Extensions implements markup.Renderer
func (Renderer) Extensions() []string {
return []string{".csv", ".tsv"}
}
// Render implements markup.Parser
func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
var tmpBlock bytes.Buffer
if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) {
tmpBlock.WriteString("<pre>")
tmpBlock.WriteString(html.EscapeString(string(rawBytes)))
tmpBlock.WriteString("</pre>")
return tmpBlock.Bytes()
func writeField(w io.Writer, element, class, field string) error {
if _, err := io.WriteString(w, "<"); err != nil {
return err
}
if _, err := io.WriteString(w, element); err != nil {
return err
}
if len(class) > 0 {
if _, err := io.WriteString(w, " class=\""); err != nil {
return err
}
if _, err := io.WriteString(w, class); err != nil {
return err
}
if _, err := io.WriteString(w, "\""); err != nil {
return err
}
}
if _, err := io.WriteString(w, ">"); err != nil {
return err
}
if _, err := io.WriteString(w, html.EscapeString(field)); err != nil {
return err
}
if _, err := io.WriteString(w, "</"); err != nil {
return err
}
if _, err := io.WriteString(w, element); err != nil {
return err
}
_, err := io.WriteString(w, ">")
return err
}
rd := csv.CreateReaderAndGuessDelimiter(rawBytes)
// Render implements markup.Renderer
func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
var tmpBlock = bufio.NewWriter(output)
writeField := func(element, class, field string) {
tmpBlock.WriteString("<")
tmpBlock.WriteString(element)
if len(class) > 0 {
tmpBlock.WriteString(" class=\"")
tmpBlock.WriteString(class)
tmpBlock.WriteString("\"")
// FIXME: don't read all to memory
rawBytes, err := ioutil.ReadAll(input)
if err != nil {
return err
}
if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) {
if _, err := tmpBlock.WriteString("<pre>"); err != nil {
return err
}
tmpBlock.WriteString(">")
tmpBlock.WriteString(html.EscapeString(field))
tmpBlock.WriteString("</")
tmpBlock.WriteString(element)
tmpBlock.WriteString(">")
if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
return err
}
_, err = tmpBlock.WriteString("</pre>")
return err
}
rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes))
if err != nil {
return err
}
tmpBlock.WriteString(`<table class="data-table">`)
if _, err := tmpBlock.WriteString(`<table class="data-table">`); err != nil {
return err
}
row := 1
for {
fields, err := rd.Read()
@ -74,20 +110,29 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string,
if err != nil {
continue
}
tmpBlock.WriteString("<tr>")
if _, err := tmpBlock.WriteString("<tr>"); err != nil {
return err
}
element := "td"
if row == 1 {
element = "th"
}
writeField(element, "line-num", strconv.Itoa(row))
if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row)); err != nil {
return err
}
for _, field := range fields {
writeField(element, "", field)
if err := writeField(tmpBlock, element, "", field); err != nil {
return err
}
}
if _, err := tmpBlock.WriteString("</tr>"); err != nil {
return err
}
tmpBlock.WriteString("</tr>")
row++
}
tmpBlock.WriteString("</table>")
return tmpBlock.Bytes()
if _, err = tmpBlock.WriteString("</table>"); err != nil {
return err
}
return tmpBlock.Flush()
}

+ 8
- 3
modules/markup/csv/csv_test.go View File

@ -5,13 +5,16 @@
package markup
import (
"strings"
"testing"
"code.gitea.io/gitea/modules/markup"
"github.com/stretchr/testify/assert"
)
func TestRenderCSV(t *testing.T) {
var parser Parser
var render Renderer
var kases = map[string]string{
"a": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>a</th></tr></table>",
"1,2": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr></table>",
@ -20,7 +23,9 @@ func TestRenderCSV(t *testing.T) {
}
for k, v := range kases {
res := parser.Render([]byte(k), "", nil, false)
assert.EqualValues(t, v, string(res))
var buf strings.Builder
err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf)
assert.NoError(t, err)
assert.EqualValues(t, v, buf.String())
}
}

+ 26
- 34
modules/markup/external/external.go View File

@ -5,7 +5,7 @@
package external
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
@ -19,32 +19,32 @@ import (
"code.gitea.io/gitea/modules/util"
)
// RegisterParsers registers all supported third part parsers according settings
func RegisterParsers() {
for _, parser := range setting.ExternalMarkupParsers {
if parser.Enabled && parser.Command != "" && len(parser.FileExtensions) > 0 {
markup.RegisterParser(&Parser{parser})
// RegisterRenderers registers all supported third part renderers according settings
func RegisterRenderers() {
for _, renderer := range setting.ExternalMarkupRenderers {
if renderer.Enabled && renderer.Command != "" && len(renderer.FileExtensions) > 0 {
markup.RegisterRenderer(&Renderer{renderer})
}
}
}
// Parser implements markup.Parser for external tools
type Parser struct {
setting.MarkupParser
// Renderer implements markup.Renderer for external tools
type Renderer struct {
setting.MarkupRenderer
}
// Name returns the external tool name
func (p *Parser) Name() string {
func (p *Renderer) Name() string {
return p.MarkupName
}
// NeedPostProcess implements markup.Parser
func (p *Parser) NeedPostProcess() bool {
return p.MarkupParser.NeedPostProcess
// NeedPostProcess implements markup.Renderer
func (p *Renderer) NeedPostProcess() bool {
return p.MarkupRenderer.NeedPostProcess
}
// Extensions returns the supported extensions of the tool
func (p *Parser) Extensions() []string {
func (p *Renderer) Extensions() []string {
return p.FileExtensions
}
@ -56,14 +56,10 @@ func envMark(envName string) string {
}
// Render renders the data of the document to HTML via the external tool.
func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
var (
bs []byte
buf = bytes.NewBuffer(bs)
rd = bytes.NewReader(rawBytes)
urlRawPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), urlPrefix,
urlRawPrefix = strings.Replace(ctx.URLPrefix, "/src/", "/raw/", 1)
command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), ctx.URLPrefix,
envMark("GITEA_PREFIX_RAW"), urlRawPrefix).Replace(p.Command)
commands = strings.Fields(command)
args = commands[1:]
@ -73,8 +69,7 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
// write to temp file
f, err := ioutil.TempFile("", "gitea_input")
if err != nil {
log.Error("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err)
return []byte("")
return fmt.Errorf("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err)
}
tmpPath := f.Name()
defer func() {
@ -83,17 +78,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
}
}()
_, err = io.Copy(f, rd)
_, err = io.Copy(f, input)
if err != nil {
f.Close()
log.Error("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err)
return []byte("")
return fmt.Errorf("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err)
}
err = f.Close()
if err != nil {
log.Error("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err)
return []byte("")
return fmt.Errorf("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err)
}
args = append(args, f.Name())
}
@ -101,16 +94,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
cmd := exec.Command(commands[0], args...)
cmd.Env = append(
os.Environ(),
"GITEA_PREFIX_SRC="+urlPrefix,
"GITEA_PREFIX_SRC="+ctx.URLPrefix,
"GITEA_PREFIX_RAW="+urlRawPrefix,
)
if !p.IsInputFile {
cmd.Stdin = rd
cmd.Stdin = input
}
cmd.Stdout = buf
cmd.Stdout = output
if err := cmd.Run(); err != nil {
log.Error("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err)
return []byte("")
return fmt.Errorf("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err)
}
return buf.Bytes()
return nil
}

+ 98
- 138
modules/markup/html.go View File

@ -7,6 +7,8 @@ package markup
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/url"
"path"
"path/filepath"
@ -144,7 +146,7 @@ func (p *postProcessError) Error() string {
return "PostProcess: " + p.context + ", " + p.err.Error()
}
type processor func(ctx *postProcessCtx, node *html.Node)
type processor func(ctx *RenderContext, node *html.Node)
var defaultProcessors = []processor{
fullIssuePatternProcessor,
@ -159,34 +161,17 @@ var defaultProcessors = []processor{
emojiShortCodeProcessor,
}
type postProcessCtx struct {
metas map[string]string
urlPrefix string
isWikiMarkdown bool
// processors used by this context.
procs []processor
}
// PostProcess does the final required transformations to the passed raw HTML
// data, and ensures its validity. Transformations include: replacing links and
// emails with HTML links, parsing shortlinks in the format of [[Link]], like
// MediaWiki, linking issues in the format #ID, and mentions in the format
// @user, and others.
func PostProcess(
rawHTML []byte,
urlPrefix string,
metas map[string]string,
isWikiMarkdown bool,
) ([]byte, error) {
// create the context from the parameters
ctx := &postProcessCtx{
metas: metas,
urlPrefix: urlPrefix,
isWikiMarkdown: isWikiMarkdown,
procs: defaultProcessors,
}
return ctx.postProcess(rawHTML)
ctx *RenderContext,
input io.Reader,
output io.Writer,
) error {
return postProcess(ctx, defaultProcessors, input, output)
}
var commitMessageProcessors = []processor{
@ -205,23 +190,18 @@ var commitMessageProcessors = []processor{
// the shortLinkProcessor and will add a defaultLinkProcessor if defaultLink is
// set, which changes every text node into a link to the passed default link.
func RenderCommitMessage(
rawHTML []byte,
urlPrefix, defaultLink string,
metas map[string]string,
) ([]byte, error) {
ctx := &postProcessCtx{
metas: metas,
urlPrefix: urlPrefix,
procs: commitMessageProcessors,
}
if defaultLink != "" {
ctx *RenderContext,
content string,
) (string, error) {
var procs = commitMessageProcessors
if ctx.DefaultLink != "" {
// we don't have to fear data races, because being
// commitMessageProcessors of fixed len and cap, every time we append
// something to it the slice is realloc+copied, so append always
// generates the slice ex-novo.
ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink))
procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink))
}
return ctx.postProcess(rawHTML)
return renderProcessString(ctx, procs, content)
}
var commitMessageSubjectProcessors = []processor{
@ -245,83 +225,72 @@ var emojiProcessors = []processor{
// emailAddressProcessor, will add a defaultLinkProcessor if defaultLink is set,
// which changes every text node into a link to the passed default link.
func RenderCommitMessageSubject(
rawHTML []byte,
urlPrefix, defaultLink string,
metas map[string]string,
) ([]byte, error) {
ctx := &postProcessCtx{
metas: metas,
urlPrefix: urlPrefix,
procs: commitMessageSubjectProcessors,
}
if defaultLink != "" {
ctx *RenderContext,
content string,
) (string, error) {
var procs = commitMessageSubjectProcessors
if ctx.DefaultLink != "" {
// we don't have to fear data races, because being
// commitMessageSubjectProcessors of fixed len and cap, every time we
// append something to it the slice is realloc+copied, so append always
// generates the slice ex-novo.
ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink))
procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink))
}
return ctx.postProcess(rawHTML)
return renderProcessString(ctx, procs, content)
}
// RenderIssueTitle to process title on individual issue/pull page
func RenderIssueTitle(
rawHTML []byte,
urlPrefix string,
metas map[string]string,
) ([]byte, error) {
ctx := &postProcessCtx{
metas: metas,
urlPrefix: urlPrefix,
procs: []processor{
issueIndexPatternProcessor,
sha1CurrentPatternProcessor,
emojiShortCodeProcessor,
emojiProcessor,
},
ctx *RenderContext,
title string,
) (string, error) {
return renderProcessString(ctx, []processor{
issueIndexPatternProcessor,
sha1CurrentPatternProcessor,
emojiShortCodeProcessor,
emojiProcessor,
}, title)
}
func renderProcessString(ctx *RenderContext, procs []processor, content string) (string, error) {
var buf strings.Builder
if err := postProcess(ctx, procs, strings.NewReader(content), &buf); err != nil {
return "", err
}
return ctx.postProcess(rawHTML)
return buf.String(), nil
}
// RenderDescriptionHTML will use similar logic as PostProcess, but will
// use a single special linkProcessor.
func RenderDescriptionHTML(
rawHTML []byte,
urlPrefix string,
metas map[string]string,
) ([]byte, error) {
ctx := &postProcessCtx{
metas: metas,
urlPrefix: urlPrefix,
procs: []processor{
descriptionLinkProcessor,
emojiShortCodeProcessor,
emojiProcessor,
},
}
return ctx.postProcess(rawHTML)
ctx *RenderContext,
content string,
) (string, error) {
return renderProcessString(ctx, []processor{
descriptionLinkProcessor,
emojiShortCodeProcessor,
emojiProcessor,
}, content)
}
// RenderEmoji for when we want to just process emoji and shortcodes
// in various places it isn't already run through the normal markdown procesor
func RenderEmoji(
rawHTML []byte,
) ([]byte, error) {
ctx := &postProcessCtx{
procs: emojiProcessors,
}
return ctx.postProcess(rawHTML)
content string,
) (string, error) {
return renderProcessString(&RenderContext{}, emojiProcessors, content)
}
var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`)
var nulCleaner = strings.NewReplacer("\000", "")
func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
if ctx.procs == nil {
ctx.procs = defaultProcessors
func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error {
// FIXME: don't read all content to memory
rawHTML, err := ioutil.ReadAll(input)
if err != nil {
return err
}
// give a generous extra 50 bytes
res := bytes.NewBuffer(make([]byte, 0, len(rawHTML)+50))
// prepend "<html><body>"
_, _ = res.WriteString("<html><body>")
@ -335,11 +304,11 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
// parse the HTML
nodes, err := html.ParseFragment(res, nil)
if err != nil {
return nil, &postProcessError{"invalid HTML", err}
return &postProcessError{"invalid HTML", err}
}
for _, node := range nodes {
ctx.visitNode(node, true)
visitNode(ctx, procs, node, true)
}
newNodes := make([]*html.Node, 0, len(nodes))
@ -365,25 +334,17 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) {
}
}
nodes = newNodes
// Create buffer in which the data will be placed again. We know that the
// length will be at least that of res; to spare a few alloc+copy, we
// reuse res, resetting its length to 0.
res.Reset()
// Render everything to buf.
for _, node := range nodes {
err = html.Render(res, node)
for _, node := range newNodes {
err = html.Render(output, node)
if err != nil {
return nil, &postProcessError{"error rendering processed HTML", err}
return &postProcessError{"error rendering processed HTML", err}
}
}
// Everything done successfully, return parsed data.
return res.Bytes(), nil
return nil
}
func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
func visitNode(ctx *RenderContext, procs []processor, node *html.Node, visitText bool) {
// Add user-content- to IDs if they don't already have them
for idx, attr := range node.Attr {
if attr.Key == "id" && !(strings.HasPrefix(attr.Val, "user-content-") || blackfridayExtRegex.MatchString(attr.Val)) {
@ -399,7 +360,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
switch node.Type {
case html.TextNode:
if visitText {
ctx.textNode(node)
textNode(ctx, procs, node)
}
case html.ElementNode:
if node.Data == "img" {
@ -410,8 +371,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
}
link := []byte(attr.Val)
if len(link) > 0 && !IsLink(link) {
prefix := ctx.urlPrefix
if ctx.isWikiMarkdown {
prefix := ctx.URLPrefix
if ctx.IsWiki {
prefix = util.URLJoin(prefix, "wiki", "raw")
}
prefix = strings.Replace(prefix, "/src/", "/media/", 1)
@ -449,7 +410,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
}
}
for n := node.FirstChild; n != nil; n = n.NextSibling {
ctx.visitNode(n, visitText)
visitNode(ctx, procs, n, visitText)
}
}
// ignore everything else
@ -457,8 +418,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) {
// textNode runs the passed node through various processors, in order to handle
// all kinds of special links handled by the post-processing.
func (ctx *postProcessCtx) textNode(node *html.Node) {
for _, processor := range ctx.procs {
func textNode(ctx *RenderContext, procs []processor, node *html.Node) {
for _, processor := range procs {
processor(ctx, node)
}
}
@ -609,7 +570,7 @@ func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) {
}
}
func mentionProcessor(ctx *postProcessCtx, node *html.Node) {
func mentionProcessor(ctx *RenderContext, node *html.Node) {
// We replace only the first mention; other mentions will be addressed later
found, loc := references.FindFirstMentionBytes([]byte(node.Data))
if !found {
@ -617,26 +578,26 @@ func mentionProcessor(ctx *postProcessCtx, node *html.Node) {
}
mention := node.Data[loc.Start:loc.End]
var teams string
teams, ok := ctx.metas["teams"]
teams, ok := ctx.Metas["teams"]
// FIXME: util.URLJoin may not be necessary here:
// - setting.AppURL is defined to have a terminal '/' so unless mention[1:]
// is an AppSubURL link we can probably fallback to concatenation.
// team mention should follow @orgName/teamName style
if ok && strings.Contains(mention, "/") {
mentionOrgAndTeam := strings.Split(mention, "/")
if mentionOrgAndTeam[0][1:] == ctx.metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") {
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention"))
if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") {
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention"))
}
return
}
replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention"))
}
func shortLinkProcessor(ctx *postProcessCtx, node *html.Node) {
func shortLinkProcessor(ctx *RenderContext, node *html.Node) {
shortLinkProcessorFull(ctx, node, false)
}
func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
func shortLinkProcessorFull(ctx *RenderContext, node *html.Node, noLink bool) {
m := shortLinkPattern.FindStringSubmatchIndex(node.Data)
if m == nil {
return
@ -741,13 +702,13 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
link = url.PathEscape(link)
}
}
urlPrefix := ctx.urlPrefix
urlPrefix := ctx.URLPrefix
if image {
if !absoluteLink {
if IsSameDomain(urlPrefix) {
urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
}
if ctx.isWikiMarkdown {
if ctx.IsWiki {
link = util.URLJoin("wiki", "raw", link)
}
link = util.URLJoin(urlPrefix, link)
@ -778,7 +739,7 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
}
} else {
if !absoluteLink {
if ctx.isWikiMarkdown {
if ctx.IsWiki {
link = util.URLJoin("wiki", link)
}
link = util.URLJoin(urlPrefix, link)
@ -794,8 +755,8 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
replaceContent(node, m[0], m[1], linkNode)
}
func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) {
if ctx.metas == nil {
func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
return
}
m := getIssueFullPattern().FindStringSubmatchIndex(node.Data)
@ -811,7 +772,7 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) {
matchOrg := linkParts[len(linkParts)-4]
matchRepo := linkParts[len(linkParts)-3]
if matchOrg == ctx.metas["user"] && matchRepo == ctx.metas["repo"] {
if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] {
// TODO if m[4]:m[5] is not nil, then link is to a comment,
// and we should indicate that in the text somehow
replaceContent(node, m[0], m[1], createLink(link, id, "ref-issue"))
@ -822,8 +783,8 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) {
}
}
func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
if ctx.metas == nil {
func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
return
}
@ -832,8 +793,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
ref *references.RenderizableReference
)
_, exttrack := ctx.metas["format"]
alphanum := ctx.metas["style"] == IssueNameStyleAlphanumeric
_, exttrack := ctx.Metas["format"]
alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric
// Repos with external issue trackers might still need to reference local PRs
// We need to concern with the first one that shows up in the text, whichever it is
@ -853,8 +814,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
var link *html.Node
reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
if exttrack && !ref.IsPull {
ctx.metas["index"] = ref.Issue
link = createLink(com.Expand(ctx.metas["format"], ctx.metas), reftext, "ref-issue")
ctx.Metas["index"] = ref.Issue
link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue")
} else {
// Path determines the type of link that will be rendered. It's unknown at this point whether
// the linked item is actually a PR or an issue. Luckily it's of no real consequence because
@ -864,7 +825,7 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
path = "pulls"
}
if ref.Owner == "" {
link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], path, ref.Issue), reftext, "ref-issue")
link = createLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue")
} else {
link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue")
}
@ -893,8 +854,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
}
// fullSha1PatternProcessor renders SHA containing URLs
func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) {
if ctx.metas == nil {
func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
return
}
m := anySHA1Pattern.FindStringSubmatchIndex(node.Data)
@ -944,8 +905,7 @@ func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) {
}
// emojiShortCodeProcessor for rendering text like :smile: into emoji
func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) {
func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) {
m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data)
if m == nil {
return
@ -968,7 +928,7 @@ func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) {
}
// emoji processor to match emoji and add emoji class
func emojiProcessor(ctx *postProcessCtx, node *html.Node) {
func emojiProcessor(ctx *RenderContext, node *html.Node) {
m := emoji.FindEmojiSubmatchIndex(node.Data)
if m == nil {
return
@ -983,8 +943,8 @@ func emojiProcessor(ctx *postProcessCtx, node *html.Node) {
// sha1CurrentPatternProcessor renders SHA1 strings to corresponding links that
// are assumed to be in the same repository.
func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
if ctx.metas == nil || ctx.metas["user"] == "" || ctx.metas["repo"] == "" || ctx.metas["repoPath"] == "" {
func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" {
return
}
m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data)
@ -1000,7 +960,7 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
// as used by git and github for linking and thus we have to do similar.
// Because of this, we check to make sure that a matched hash is actually
// a commit in the repository before making it a link.
if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.metas["repoPath"]); err != nil {
if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil {
if !strings.Contains(err.Error(), "fatal: Needed a single revision") {
log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err)
}
@ -1008,11 +968,11 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
}
replaceContent(node, m[2], m[3],
createCodeLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "commit", hash), base.ShortSha(hash), "commit"))
createCodeLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], "commit", hash), base.ShortSha(hash), "commit"))
}
// emailAddressProcessor replaces raw email addresses with a mailto: link.
func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) {
func emailAddressProcessor(ctx *RenderContext, node *html.Node) {
m := emailRegex.FindStringSubmatchIndex(node.Data)
if m == nil {
return
@ -1023,7 +983,7 @@ func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) {
// linkProcessor creates links for any HTTP or HTTPS URL not captured by
// markdown.
func linkProcessor(ctx *postProcessCtx, node *html.Node) {
func linkProcessor(ctx *RenderContext, node *html.Node) {
m := common.LinkRegex.FindStringIndex(node.Data)
if m == nil {
return
@ -1033,7 +993,7 @@ func linkProcessor(ctx *postProcessCtx, node *html.Node) {
}
func genDefaultLinkProcessor(defaultLink string) processor {
return func(ctx *postProcessCtx, node *html.Node) {
return func(ctx *RenderContext, node *html.Node) {
ch := &html.Node{
Parent: node,
Type: html.TextNode,
@ -1052,7 +1012,7 @@ func genDefaultLinkProcessor(defaultLink string) processor {
}
// descriptionLinkProcessor creates links for DescriptionHTML
func descriptionLinkProcessor(ctx *postProcessCtx, node *html.Node) {
func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) {
m := common.LinkRegex.FindStringIndex(node.Data)
if m == nil {
return


+ 32
- 27
modules/markup/html_internal_test.go View File

@ -61,8 +61,8 @@ var localMetas = map[string]string{
func TestRender_IssueIndexPattern(t *testing.T) {
// numeric: render inputs without valid mentions
test := func(s string) {
testRenderIssueIndexPattern(t, s, s, nil)
testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: numericMetas})
testRenderIssueIndexPattern(t, s, s, &RenderContext{})
testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas})
}
// should not render anything when there are no mentions
@ -109,13 +109,13 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, path), "ref-issue", index, marker)
}
expectedNil := fmt.Sprintf(expectedFmt, links...)
testRenderIssueIndexPattern(t, s, expectedNil, &postProcessCtx{metas: localMetas})
testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{Metas: localMetas})
for i, index := range indices {
links[i] = numericIssueLink(prefix, "ref-issue", index, marker)
}
expectedNum := fmt.Sprintf(expectedFmt, links...)
testRenderIssueIndexPattern(t, s, expectedNum, &postProcessCtx{metas: numericMetas})
testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{Metas: numericMetas})
}
// should render freestanding mentions
@ -150,7 +150,7 @@ func TestRender_IssueIndexPattern3(t *testing.T) {
// alphanumeric: render inputs without valid mentions
test := func(s string) {
testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: alphanumericMetas})
testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: alphanumericMetas})
}
test("")
test("this is a test")
@ -181,25 +181,22 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue", name)
}
expected := fmt.Sprintf(expectedFmt, links...)
testRenderIssueIndexPattern(t, s, expected, &postProcessCtx{metas: alphanumericMetas})
testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
}
test("OTT-1234 test", "%s test", "OTT-1234")
test("test T-12 issue", "test %s issue", "T-12")
test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890")
}
func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *postProcessCtx) {
if ctx == nil {
ctx = new(postProcessCtx)
}
ctx.procs = []processor{issueIndexPatternProcessor}
if ctx.urlPrefix == "" {
ctx.urlPrefix = AppSubURL
func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) {
if ctx.URLPrefix == "" {
ctx.URLPrefix = AppSubURL
}
res, err := ctx.postProcess([]byte(input))
var buf strings.Builder
err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf)
assert.NoError(t, err)
assert.Equal(t, expected, string(res))
assert.Equal(t, expected, buf.String())
}
func TestRender_AutoLink(t *testing.T) {
@ -207,12 +204,22 @@ func TestRender_AutoLink(t *testing.T) {
setting.AppSubURL = AppSubURL
test := func(input, expected string) {
buffer, err := PostProcess([]byte(input), setting.AppSubURL, localMetas, false)
var buffer strings.Builder
err := PostProcess(&RenderContext{
URLPrefix: setting.AppSubURL,
Metas: localMetas,
}, strings.NewReader(input), &buffer)
assert.Equal(t, err, nil)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
buffer, err = PostProcess([]byte(input), setting.AppSubURL, localMetas, true)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String()))
buffer.Reset()
err = PostProcess(&RenderContext{
URLPrefix: setting.AppSubURL,
Metas: localMetas,
IsWiki: true,
}, strings.NewReader(input), &buffer)
assert.Equal(t, err, nil)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String()))
}
// render valid issue URLs
@ -235,15 +242,13 @@ func TestRender_FullIssueURLs(t *testing.T) {
setting.AppSubURL = AppSubURL
test := func(input, expected string) {
ctx := new(postProcessCtx)
ctx.procs = []processor{fullIssuePatternProcessor}
if ctx.urlPrefix == "" {
ctx.urlPrefix = AppSubURL
}
ctx.metas = localMetas
result, err := ctx.postProcess([]byte(input))
var result strings.Builder
err := postProcess(&RenderContext{
URLPrefix: AppSubURL,
Metas: localMetas,
}, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result)
assert.NoError(t, err)
assert.Equal(t, expected, string(result))
assert.Equal(t, expected, result.String())
}
test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6",
"Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6")


+ 50
- 14
modules/markup/html_test.go View File

@ -28,7 +28,12 @@ func TestRender_Commits(t *testing.T) {
setting.AppSubURL = AppSubURL
test := func(input, expected string) {
buffer := RenderString(".md", input, setting.AppSubURL, localMetas)
buffer, err := RenderString(&RenderContext{
Filename: ".md",
URLPrefix: setting.AppSubURL,
Metas: localMetas,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
}
@ -59,7 +64,12 @@ func TestRender_CrossReferences(t *testing.T) {
setting.AppSubURL = AppSubURL
test := func(input, expected string) {
buffer := RenderString("a.md", input, setting.AppSubURL, localMetas)
buffer, err := RenderString(&RenderContext{
Filename: "a.md",
URLPrefix: setting.AppSubURL,
Metas: localMetas,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
}
@ -91,7 +101,11 @@ func TestRender_links(t *testing.T) {
setting.AppSubURL = AppSubURL
test := func(input, expected string) {
buffer := RenderString("a.md", input, setting.AppSubURL, nil)
buffer, err := RenderString(&RenderContext{
Filename: "a.md",
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
}
// Text that should be turned into URL
@ -187,8 +201,12 @@ func TestRender_email(t *testing.T) {
setting.AppSubURL = AppSubURL
test := func(input, expected string) {
buffer := RenderString("a.md", input, setting.AppSubURL, nil)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
res, err := RenderString(&RenderContext{
Filename: "a.md",
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res))
}
// Text that should be turned into email link
@ -242,7 +260,11 @@ func TestRender_emoji(t *testing.T) {
test := func(input, expected string) {
expected = strings.ReplaceAll(expected, "&", "&amp;")
buffer := RenderString("a.md", input, setting.AppSubURL, nil)
buffer, err := RenderString(&RenderContext{
Filename: "a.md",
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
}
@ -291,9 +313,17 @@ func TestRender_ShortLinks(t *testing.T) {
tree := util.URLJoin(AppSubURL, "src", "master")
test := func(input, expected, expectedWiki string) {
buffer := markdown.RenderString(input, tree, nil)
buffer, err := markdown.RenderString(&RenderContext{
URLPrefix: tree,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
buffer = markdown.RenderWiki([]byte(input), setting.AppSubURL, localMetas)
buffer, err = markdown.RenderString(&RenderContext{
URLPrefix: setting.AppSubURL,
Metas: localMetas,
IsWiki: true,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer))
}
@ -395,16 +425,22 @@ func Test_ParseClusterFuzz(t *testing.T) {
data := "<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
val, err := PostProcess([]byte(data), "https://example.com", localMetas, false)
var res strings.Builder
err := PostProcess(&RenderContext{
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
assert.NoError(t, err)
assert.NotContains(t, string(val), "<html")
assert.NotContains(t, res.String(), "<html")
data = "<!DOCTYPE html>\n<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
val, err = PostProcess([]byte(data), "https://example.com", localMetas, false)
res.Reset()
err = PostProcess(&RenderContext{
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
assert.NoError(t, err)
assert.NotContains(t, string(val), "<html")
assert.NotContains(t, res.String(), "<html")
}

+ 60
- 36
modules/markup/markdown/markdown.go View File

@ -8,6 +8,7 @@ package markdown
import (
"fmt"
"io"
"io/ioutil"
"strings"
"sync"
@ -73,17 +74,17 @@ func (l *limitWriter) CloseWithError(err error) error {
return l.w.CloseWithError(err)
}
// NewGiteaParseContext creates a parser.Context with the gitea context set
func NewGiteaParseContext(urlPrefix string, metas map[string]string, isWiki bool) parser.Context {
// newParserContext creates a parser.Context with the render context set
func newParserContext(ctx *markup.RenderContext) parser.Context {
pc := parser.NewContext(parser.WithIDs(newPrefixedIDs()))
pc.Set(urlPrefixKey, urlPrefix)
pc.Set(isWikiKey, isWiki)
pc.Set(renderMetasKey, metas)
pc.Set(urlPrefixKey, ctx.URLPrefix)
pc.Set(isWikiKey, ctx.IsWiki)
pc.Set(renderMetasKey, ctx.Metas)
return pc
}
// actualRender renders Markdown to HTML without handling special links.
func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) []byte {
func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
once.Do(func() {
converter = goldmark.New(
goldmark.WithExtensions(extension.Table,
@ -169,7 +170,7 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa
limit: setting.UI.MaxDisplayFileSize * 3,
}
// FIXME: should we include a timeout that closes the pipe to abort the parser and sanitizer if it takes too long?
// FIXME: should we include a timeout that closes the pipe to abort the renderer and sanitizer if it takes too long?
go func() {
defer func() {
err := recover()
@ -184,18 +185,26 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa
_ = lw.CloseWithError(fmt.Errorf("%v", err))
}()
pc := NewGiteaParseContext(urlPrefix, metas, wikiMarkdown)
if err := converter.Convert(giteautil.NormalizeEOL(body), lw, parser.WithContext(pc)); err != nil {
// FIXME: Don't read all to memory, but goldmark doesn't support
pc := newParserContext(ctx)
buf, err := ioutil.ReadAll(input)
if err != nil {
log.Error("Unable to ReadAll: %v", err)
return
}
if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil {
log.Error("Unable to render: %v", err)
_ = lw.CloseWithError(err)
return
}
_ = lw.Close()
}()