From f7fa825ecfe14765068738365077526c7f32e7b1 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 29 Jun 2021 00:18:51 +0300 Subject: [PATCH 01/29] rst: add missing line/column info for some warnings --- compiler/docgen.nim | 1 + compiler/lineinfos.nim | 5 ++ compiler/main.nim | 7 +-- lib/packages/docutils/rst.nim | 83 ++++++++++++++++++-------------- lib/packages/docutils/rstast.nim | 18 +++++-- lib/packages/docutils/rstgen.nim | 9 ++-- tests/stdlib/trst.nim | 26 ++++++++++ tests/stdlib/trstgen.nim | 29 +++++++++-- 8 files changed, 127 insertions(+), 51 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 1ae00e22e3afa..cc55de9fb9e5a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -168,6 +168,7 @@ template declareClosures = of meFootnoteMismatch: k = errFootnoteMismatch of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel of mwUnknownSubstitution: k = warnUnknownSubstitutionX + of mwBrokenLink: k = warnBrokenLink of mwUnsupportedLanguage: k = warnLanguageXNotSupported of mwUnsupportedField: k = warnFieldXNotSupported of mwRstStyle: k = warnRstStyle diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 801c20d2557ea..26cefd127885f 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -48,6 +48,7 @@ type warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated", warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX", + warnBrokenLink = "BrokenLink", warnLanguageXNotSupported = "LanguageXNotSupported", warnFieldXNotSupported = "FieldXNotSupported", warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored", @@ -113,6 +114,7 @@ const warnUnknownMagic: "unknown magic '$1' might crash the compiler", warnRedefinitionOfLabel: "redefinition of label '$1'", warnUnknownSubstitutionX: "unknown substitution '$1'", + warnBrokenLink: "broken link '$1'", warnLanguageXNotSupported: "language '$1' not supported", warnFieldXNotSupported: "field '$1' not supported", warnRstStyle: "RST style: $1", @@ -196,6 +198,9 @@ const warnMax* = pred(hintSuccess) hintMin* = hintSuccess hintMax* = high(TMsgKind) + rstWarnings* = {warnRedefinitionOfLabel, warnUnknownSubstitutionX, + warnBrokenLink, warnLanguageXNotSupported, + warnFieldXNotSupported, warnRstStyle} type TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints diff --git a/compiler/main.nim b/compiler/main.nim index 1c8034c6cd7b7..2f29e6b27eff4 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -295,8 +295,7 @@ proc mainCommand*(graph: ModuleGraph) = commandBuildIndex(conf, $conf.outDir) of cmdRst2html: # XXX: why are warnings disabled by default for rst2html and rst2tex? - for warn in [warnUnknownSubstitutionX, warnLanguageXNotSupported, - warnFieldXNotSupported, warnRstStyle]: + for warn in rstWarnings: conf.setNoteDefaults(warn, true) conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218 when defined(leanCompiler): @@ -305,9 +304,7 @@ proc mainCommand*(graph: ModuleGraph) = loadConfigs(DocConfig, cache, conf, graph.idgen) commandRst2Html(cache, conf) of cmdRst2tex, cmdDoc2tex: - for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX, - warnLanguageXNotSupported, - warnFieldXNotSupported, warnRstStyle]: + for warn in rstWarnings: conf.setNoteDefaults(warn, true) when defined(leanCompiler): conf.quitOrRaise "compiler wasn't built with documentation generator" diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 9abdf3fc3329e..e5a4bafa47d12 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -229,6 +229,7 @@ type meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", mwRedefinitionOfLabel = "redefinition of label '$1'", mwUnknownSubstitution = "unknown substitution '$1'", + mwBrokenLink = "broken link '$1'", mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", mwRstStyle = "RST style: $1" @@ -605,10 +606,13 @@ proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = p.col + currentTok(p).col, msgKind, arg) proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = - ## Print warnings for footnotes/substitutions. - ## TODO: their line/column info is not known, to fix it. s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg) +proc rstMessage(s: PRstSharedState, loc: PRstLocation, + msgKind: MsgKind, arg: string) = + ## Print warnings for footnotes/substitutions/references. + s.msgHandler(loc.filename, loc.line, loc.col, msgKind, arg) + proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = p.s.msgHandler(p.s.filename, p.line + line, p.col + col, msgKind, arg) @@ -935,9 +939,10 @@ proc getAutoSymbol(s: PRstSharedState, order: int): string = if fnote.autoSymIdx == order: return fnote.label -proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode = +proc newRstNodeA(p: var RstParser, kind: RstNodeKind, + loc: PRstLocation = nil): PRstNode = ## create node and consume the current anchor - result = newRstNode(kind) + result = newRstNode(kind, loc=loc) if p.curAnchor != "": result.anchor = p.curAnchor p.curAnchor = "" @@ -1155,9 +1160,15 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind = if result == rnUnknownRole: rstMessage(p, mwUnsupportedLanguage, sym) +proc newLocation(p: RstParser, i = p.idx): PRstLocation = + PRstLocation( + col: p.col + p.tok[i].col, + line: p.line + p.tok[i].line, + filename: p.s.filename) + proc toInlineCode(n: PRstNode, language: string): PRstNode = ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). - result = newRstNode(rnInlineCode) + result = newRstNode(rnInlineCode, loc=n.loc) let args = newRstNode(rnDirArg) var lang = language if language == "cpp": lang = "c++" @@ -1179,6 +1190,7 @@ proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode = result = newRstNode(kind, newSons) proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = + ## Finalizes node `n` that was tentatively determined as interpreted text. var newKind = n.kind var newSons = n.sons @@ -1207,12 +1219,10 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = newKind = rnHyperlink newSons = @[a, b] setRef(p, rstnodeToRefname(a), b) - elif n.kind == rnInterpretedText: - newKind = rnRef - else: + result = newRstNode(newKind, newSons) + else: # some link that will be resolved in `resolveSubs` newKind = rnRef - newSons = @[n] - result = newRstNode(newKind, newSons) + result = newRstNode(newKind, newSons, loc=n.loc) elif match(p, p.idx, ":w:"): # a role: let (roleName, lastIdx) = getRefname(p, p.idx+1) @@ -1300,20 +1310,19 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) = else: # check for reference (probably, long one like some.ref.with.dots_ ) var saveIdx = p.idx - var isRef = false + var reference: PRstNode = nil inc p.idx while currentTok(p).kind in {tkWord, tkPunct}: if currentTok(p).kind == tkPunct: if isInlineMarkupEnd(p, "_", exact=true): - isRef = true + reference = newRstNode(rnRef, loc=newLocation(p, saveIdx)) break if not validRefnamePunct(currentTok(p).symbol): break inc p.idx - if isRef: - let r = newRstNode(rnRef) - for i in saveIdx..p.idx-1: r.add newLeaf(p.tok[i].symbol) - father.add r + if reference != nil: + for i in saveIdx..p.idx-1: reference.add newLeaf(p.tok[i].symbol) + father.add reference inc p.idx # skip final _ else: # 1 normal word father.add newLeaf(p.tok[saveIdx].symbol) @@ -1387,6 +1396,7 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, else: rstMessage(p, meExpected, postfix, line, col) proc parseMarkdownCodeblock(p: var RstParser): PRstNode = + result = newRstNodeA(p, rnCodeBlock, loc=newLocation(p)) var args = newRstNode(rnDirArg) if currentTok(p).kind == tkWord: args.add(newLeaf(p)) @@ -1411,7 +1421,6 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode = inc p.idx var lb = newRstNode(rnLiteralBlock) lb.add(n) - result = newRstNodeA(p, rnCodeBlock) result.add(args) result.add(PRstNode(nil)) result.add(lb) @@ -1495,6 +1504,7 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode = proc parseInline(p: var RstParser, father: PRstNode) = var n: PRstNode # to be used in `if` condition + let saveIdx = p.idx case currentTok(p).kind of tkPunct: if isInlineMarkupStart(p, "***"): @@ -1537,12 +1547,12 @@ proc parseInline(p: var RstParser, father: PRstNode) = n = n.toOtherRole(k, roleName) father.add(n) elif isInlineMarkupStart(p, "`"): - var n = newRstNode(rnInterpretedText) + var n = newRstNode(rnInterpretedText, loc=newLocation(p, p.idx+1)) parseUntil(p, n, "`", false) # bug #17260 n = parsePostfix(p, n) father.add(n) elif isInlineMarkupStart(p, "|"): - var n = newRstNode(rnSubstitutionReferences) + var n = newRstNode(rnSubstitutionReferences, loc=newLocation(p, p.idx+1)) parseUntil(p, n, "|", false) father.add(n) elif roSupportMarkdown in p.s.options and @@ -1551,16 +1561,16 @@ proc parseInline(p: var RstParser, father: PRstNode) = discard "parseMarkdownLink already processed it" elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and (n = parseFootnoteName(p, reference=true); n != nil): - var nn = newRstNode(rnFootnoteRef) + var nn = newRstNode(rnFootnoteRef, loc=newLocation(p, saveIdx+1)) nn.add n let (fnType, _) = getFootnoteType(n) case fnType of fnAutoSymbol: p.s.lineFootnoteSymRef.add curLine(p) - nn.order = p.s.lineFootnoteSymRef.len + nn.loc.order = p.s.lineFootnoteSymRef.len of fnAutoNumber: p.s.lineFootnoteNumRef.add curLine(p) - nn.order = p.s.lineFootnoteNumRef.len + nn.loc.order = p.s.lineFootnoteNumRef.len else: discard father.add(nn) else: @@ -1693,7 +1703,7 @@ proc parseField(p: var RstParser): PRstNode = ## Returns a parsed rnField node. ## ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody. - result = newRstNode(rnField) + result = newRstNode(rnField, loc=newLocation(p)) var col = currentTok(p).col var fieldname = newRstNode(rnFieldName) parseUntil(p, fieldname, ":", false) @@ -2469,6 +2479,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to ## check them before accessing. result = newRstNodeA(p, k) + if k == rnCodeBlock: result.loc = newLocation(p) var args: PRstNode = nil var options: PRstNode = nil if hasArg in flags: @@ -2634,7 +2645,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) assert result.sons[1].kind == rnFieldList # Hook the extra field and specify the Nim language as value. - var extraNode = newRstNode(rnField) + var extraNode = newRstNode(rnField, loc=newLocation(p)) extraNode.add(newRstNode(rnFieldName)) extraNode.add(newRstNode(rnFieldBody)) extraNode.sons[0].add newLeaf("default-language") @@ -2872,7 +2883,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = var key = addNodes(n) var e = getEnv(key) if e != "": result = newLeaf(e) - else: rstMessage(s, mwUnknownSubstitution, key) + else: rstMessage(s, n.loc, mwUnknownSubstitution, key) of rnHeadline, rnOverline: # fix up section levels depending on presence of a title and subtitle if s.hTitleCnt == 2: @@ -2890,12 +2901,14 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = let text = newRstNode(rnInner, n.sons) result.sons = @[text, y] else: - let s = findMainAnchor(s, refn) - if s != "": + let anchor = findMainAnchor(s, refn) + if anchor != "": result = newRstNode(rnInternalRef) let text = newRstNode(rnInner, n.sons) - result.sons = @[text, # visible text of reference - newLeaf(s)] # link itself + result.sons = @[text, # visible text of reference + newLeaf(anchor)] # link itself + else: + rstMessage(s, n.loc, mwBrokenLink, refn) of rnFootnote: var (fnType, num) = getFootnoteType(n.sons[0]) case fnType @@ -2922,20 +2935,20 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = result.add(nn) var refn = fnType.prefix # create new rnFootnoteRef, add final label, and finalize target refn: - result = newRstNode(rnFootnoteRef) + result = newRstNode(rnFootnoteRef, loc=n.loc) case fnType of fnManualNumber: addLabel num refn.add $num of fnAutoNumber: - addLabel getFootnoteNum(s, n.order) - refn.add $n.order + addLabel getFootnoteNum(s, n.loc.order) + refn.add $n.loc.order of fnAutoNumberLabel: addLabel getFootnoteNum(s, rstnodeToRefname(n)) refn.add rstnodeToRefname(n) of fnAutoSymbol: - addLabel getAutoSymbol(s, n.order) - refn.add $n.order + addLabel getAutoSymbol(s, n.loc.order) + refn.add $n.loc.order of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) @@ -2943,7 +2956,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = if anch != "": result.add newLeaf(anch) # add link else: - rstMessage(s, mwUnknownSubstitution, refn) + rstMessage(s, n.loc, mwBrokenLink, refn) result.add newLeaf(refn) # add link of rnLeaf: discard diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 2489ce40c7ce6..8a68d950c1499 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -77,6 +77,11 @@ type PRstNode* = ref RstNode ## an RST node RstNodeSeq* = seq[PRstNode] + PRstLocation* = ref object ## location inside a file + filename*: string + line*: int + col*: int + order*: int ## only for `rnFootnoteRef` RstNode* {.acyclic, final.} = object ## AST node (result of RST parsing) case kind*: RstNodeKind ## the node's kind of rnLeaf, rnSmiley: @@ -92,9 +97,13 @@ type level*: int ## level of headings starting from 1 (main ## chapter) to larger ones (minor sub-sections) ## level=0 means it's document title or subtitle - of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem: + of rnFootnote, rnCitation, rnOptionListItem: order*: int ## footnote order (for auto-symbol footnotes and ## auto-numbered ones without a label) + of rnRef, rnSubstitutionReferences, rnFootnoteRef, + rnInterpretedText, rnField, rnInlineCode, rnCodeBlock: + loc*: PRstLocation ## To have line/column info for warnings at + ## nodes that are post-processed after parsing else: discard anchor*: string ## anchor, internal link target @@ -105,8 +114,9 @@ proc len*(n: PRstNode): int = result = len(n.sons) proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], - anchor = ""): PRstNode = + anchor = "", loc: PRstLocation = nil): PRstNode = result = PRstNode(kind: kind, sons: sons) + if loc != nil: result.loc = loc proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = assert kind in {rnLeaf, rnSmiley} @@ -388,8 +398,10 @@ proc renderRstToStr*(node: PRstNode, indent=0): string = result.add " adType=" & node.adType of rnHeadline, rnOverline, rnMarkdownHeadline: result.add " level=" & $node.level - of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem: + of rnFootnote, rnCitation, rnOptionListItem: result.add (if node.order == 0: "" else: " order=" & $node.order) + of rnFootnoteRef: + result.add (if node.loc.order == 0: "" else: " order=" & $node.loc.order) else: discard result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index b9c8f688ba006..0ed31a876b580 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -908,9 +908,8 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = [d.config.getOrDefault"doc.smiley_format" % n.text]) proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = - # TODO: proper column/line info template err(msg: string) = - d.msgHandler(d.filename, 1, 0, meInvalidRstField, msg) + d.msgHandler(n.loc.filename, n.loc.line, n.loc.col, meInvalidRstField, msg) let value = n.getFieldValue var number: int let nChars = parseInt(value, number) @@ -958,7 +957,8 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage else: - d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument) + d.msgHandler(n.loc.filename, n.loc.line, n.loc.col, + mwUnsupportedField, n.getArgument) proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = ## Iterates over all code block fields and returns processed params. @@ -1069,7 +1069,8 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) = dispA(d.target, result, blockStart, blockStart, []) if params.lang == langNone: if len(params.langStr) > 0: - d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr) + d.msgHandler(n.loc.filename, n.loc.line, n.loc.col, + mwUnsupportedLanguage, params.langStr) for letter in m.text: escChar(d.target, result, letter, emText) else: renderCodeLang(result, params.lang, m.text, d.target) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index fe99d6cfa75a0..65725138af165 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -5,6 +5,8 @@ discard """ [Suite] RST indentation +[Suite] Warnings + [Suite] RST include directive [Suite] RST escaping @@ -356,6 +358,30 @@ suite "RST indentation": # "template..." should be parsed as a definition list attached to ":test:": check inputWrong.toAst != ast +suite "Warnings": + test "warnings for broken footnotes/links/substitutions": + let input = dedent""" + firstParagraph + + footnoteRef [som]_ + + link `a broken Link`_ + + substitution |undefined subst| + + link short.link_ + + lastParagraph + """ + var warnings = new seq[string] + let output = input.toAst(warnings=warnings) + check(warnings[] == @[ + "input(3, 14) Warning: broken link 'citation-som'", + "input(5, 7) Warning: broken link 'a-broken-link'", + "input(7, 15) Warning: unknown substitution 'undefined subst'", + "input(9, 6) Warning: broken link 'shortdotlink'" + ]) + suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index d5d902e1d9f7e..fba977f9b4514 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1034,9 +1034,7 @@ Test1 """ var warnings8 = new seq[string] let output8 = input8.toHtml(warnings=warnings8) - # TODO: the line 1 is arbitrary because reference lines are not preserved - check(warnings8[] == @["input(1, 1) Warning: unknown substitution " & - "\'citation-som\'"]) + check(warnings8[] == @["input(3, 7) Warning: broken link 'citation-som'"]) # check that footnote group does not break parsing of other directives: let input9 = dedent """ @@ -1144,10 +1142,33 @@ Test1 """ var error = new string let output = input.toHtml(error=error) - check(error[] == "input(1, 1) Error: invalid field: " & + check(error[] == "input(2, 3) Error: invalid field: " & "extra arguments were given to number-lines: ' let a = 1'") check "" == output + test "code-block warning": + let input = dedent """ + .. code:: Nim + :unsupportedField: anything + + .. code:: unsupportedLang + + anything + + ```anotherLang + someCode + ``` + """ + let warnings = new seq[string] + let output = input.toHtml(warnings=warnings) + check(warnings[] == @[ + "input(2, 4) Warning: field 'unsupportedField' not supported", + "input(4, 11) Warning: language 'unsupportedLang' not supported", + "input(8, 4) Warning: language 'anotherLang' not supported" + ]) + check(output == "
\nanything
" & + "

\nsomeCode\n

\n") + test "RST admonitions": # check that all admonitions are implemented let input0 = dedent """ From e64c7fb2f59b27f533b26639c81d2d7e441ea3a5 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 29 Jun 2021 00:32:50 +0300 Subject: [PATCH 02/29] add workaround --- lib/packages/docutils/rst.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index e5a4bafa47d12..17b8fa568d2cd 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1160,12 +1160,15 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind = if result == rnUnknownRole: rstMessage(p, mwUnsupportedLanguage, sym) -proc newLocation(p: RstParser, i = p.idx): PRstLocation = +proc newLocation(p: RstParser, i: int): PRstLocation = PRstLocation( col: p.col + p.tok[i].col, line: p.line + p.tok[i].line, filename: p.s.filename) +proc newLocation(p: RstParser): PRstLocation = + newLocation(p, p.idx) + proc toInlineCode(n: PRstNode, language: string): PRstNode = ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). result = newRstNode(rnInlineCode, loc=n.loc) From af49d7b561c27c0170aed641fa82445b9736693b Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 3 Jul 2021 00:09:47 +0300 Subject: [PATCH 03/29] use TLineInfo/FileIndex for storing file names --- compiler/docgen.nim | 32 ++++----- lib/packages/docutils/rst.nim | 111 +++++++++++++++++++------------ lib/packages/docutils/rstast.nim | 22 +++--- lib/packages/docutils/rstgen.nim | 37 +++++------ tests/stdlib/trst.nim | 4 +- tests/stdlib/trstgen.nim | 84 ++++++++++++----------- 6 files changed, 164 insertions(+), 126 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index cc55de9fb9e5a..adb6b1089f226 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -183,11 +183,9 @@ template declareClosures = proc parseRst(text, filename: string, line, column: int, - rstOptions: RstParseOptions; conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() - result = rstParsePass1(text, filename, line, column, rstOptions, - sharedState) + result = rstParsePass1(text, filename, line, column, sharedState) proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, guessTarget: bool): AbsoluteFile = @@ -203,20 +201,26 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile, proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex} proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, - outExt: string = HtmlExt, module: PSym = nil): PDoc = + outExt: string = HtmlExt, module: PSym = nil, + isPureRst=false): PDoc = declareClosures() new(result) result.module = module result.conf = conf result.cache = cache result.outDir = conf.outDir.string - const options = {roSupportRawDirective, roSupportMarkdown, - roPreferMarkdown, roNimFile} + result.isPureRst = isPureRst + var options: RstParseOptions = {} + if isPureRst: + options = {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown} + else: + options = {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, + roNimFile} result.sharedState = newRstSharedState( options, filename.string, docgenFindFile, compilerMsgHandler) initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml), - conf.configVars, filename.string, options, + conf.configVars, filename.string, docgenFindFile, compilerMsgHandler) if conf.configVars.hasKey("doc.googleAnalytics"): @@ -300,8 +304,7 @@ proc genComment(d: PDoc, n: PNode): PRstNode = result = parseRst(n.comment, toFullPath(d.conf, n.info), toLinenumber(n.info), toColumn(n.info) + DocColOffset, - d.options, d.conf, - d.sharedState) + d.conf, d.sharedState) proc genRecCommentAux(d: PDoc, n: PNode): PRstNode = if n == nil: return nil @@ -1172,6 +1175,9 @@ proc finishGenerateDoc*(d: var PDoc) = d.jEntriesFinal.add entry.json d.jEntriesPre[i].rst = nil + # pass file map `files` to ``rstgen.nim`` for its warnings + d.files = move(d.sharedState.files) + proc add(d: PDoc; j: JsonItem) = if j.json != nil or j.rst != nil: d.jEntriesPre.add j @@ -1427,14 +1433,10 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) = proc commandRstAux(cache: IdentCache, conf: ConfigRef; filename: AbsoluteFile, outExt: string) = var filen = addFileExt(filename, "txt") - var d = newDocumentor(filen, cache, conf, outExt) - - d.isPureRst = true + var d = newDocumentor(filen, cache, conf, outExt, isPureRst=true) let rst = parseRst(readFile(filen.string), filen.string, line=LineRstInit, column=ColRstInit, - {roSupportRawDirective, roSupportMarkdown, - roPreferMarkdown}, conf, - d.sharedState) + conf, d.sharedState) d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] finishGenerateDoc(d) writeOutput(d) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 17b8fa568d2cd..4f8022f4c7dd8 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -196,7 +196,7 @@ import os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars + std/private/miscdollars, ../../../compiler/lineinfos from highlite import SourceLanguage, getSourceLanguage type @@ -510,7 +510,9 @@ type # number, order of occurrence msgHandler: MsgHandler # How to handle errors. findFile: FindFileHandler # How to find files. - filename: string + files*: seq[string] # map FileIndex -> file name (for storing + # file names for warnings after 1st stage) + currFileIdx: FileIndex # current index in `files` hasToc*: bool PRstSharedState* = ref RstSharedState @@ -580,6 +582,23 @@ proc whichRoleAux(sym: string): RstNodeKind = else: # unknown role result = rnUnknownRole +proc setCurrFilename(s: PRstSharedState, file1: string) = + for i, file2 in s.files: + if file1 == file2: + s.currFileIdx = i.FileIndex + return + s.files.add file1 + s.currFileIdx = (s.files.len - 1).FileIndex + +proc getFilename(files: seq[string], fid: FileIndex): string = + if fid.int < files.len: + result = files[fid.int] + else: + result = "input" + +proc currFilename(s: PRstSharedState): string = + getFilename(s.files, s.currFileIdx) + proc newRstSharedState*(options: RstParseOptions, filename: string, findFile: FindFileHandler, @@ -590,35 +609,38 @@ proc newRstSharedState*(options: RstParseOptions, currRoleKind: whichRoleAux(r), options: options, msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, - filename: filename, findFile: if not isNil(findFile): findFile else: defaultFindFile ) + setCurrFilename(result, filename) proc curLine(p: RstParser): int = p.line + currentTok(p).line proc findRelativeFile(p: RstParser; filename: string): string = - result = p.s.filename.splitFile.dir / filename + result = p.s.currFilename.splitFile.dir / filename if not fileExists(result): result = p.s.findFile(filename) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = - p.s.msgHandler(p.s.filename, curLine(p), + p.s.msgHandler(p.s.currFilename, curLine(p), p.col + currentTok(p).col, msgKind, arg) proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = - s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg) + s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) -proc rstMessage(s: PRstSharedState, loc: PRstLocation, - msgKind: MsgKind, arg: string) = - ## Print warnings for footnotes/substitutions/references. - s.msgHandler(loc.filename, loc.line, loc.col, msgKind, arg) +proc rstMessage*(files: seq[string], f: MsgHandler, + li: ref TLineInfo | TLineInfo, + msgKind: MsgKind, arg: string) = + ## Print warnings using `li: TLineInfo`, i.e. in 2nd-pass warnings for + ## footnotes/substitutions/references or from ``rstgen.nim``. + let file = getFilename(files, li.fileIndex) + f(file, li.line.int, li.col.int, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = - p.s.msgHandler(p.s.filename, p.line + line, + p.s.msgHandler(p.s.currFilename, p.line + line, p.col + col, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind) = - p.s.msgHandler(p.s.filename, curLine(p), + p.s.msgHandler(p.s.currFilename, curLine(p), p.col + currentTok(p).col, msgKind, currentTok(p).symbol) @@ -940,9 +962,9 @@ proc getAutoSymbol(s: PRstSharedState, order: int): string = return fnote.label proc newRstNodeA(p: var RstParser, kind: RstNodeKind, - loc: PRstLocation = nil): PRstNode = + li: ref TLineInfo = nil): PRstNode = ## create node and consume the current anchor - result = newRstNode(kind, loc=loc) + result = newRstNode(kind, li=li) if p.curAnchor != "": result.anchor = p.curAnchor p.curAnchor = "" @@ -1160,18 +1182,18 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind = if result == rnUnknownRole: rstMessage(p, mwUnsupportedLanguage, sym) -proc newLocation(p: RstParser, i: int): PRstLocation = - PRstLocation( - col: p.col + p.tok[i].col, - line: p.line + p.tok[i].line, - filename: p.s.filename) +proc newLocation(p: RstParser, iTok: int): ref TLineInfo = + new result + result.col = int16(p.col + p.tok[iTok].col) + result.line = uint16(p.line + p.tok[iTok].line) + result.fileIndex = p.s.currFileIdx -proc newLocation(p: RstParser): PRstLocation = +proc newLocation(p: RstParser): ref TLineInfo = newLocation(p, p.idx) proc toInlineCode(n: PRstNode, language: string): PRstNode = ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). - result = newRstNode(rnInlineCode, loc=n.loc) + result = newRstNode(rnInlineCode, li=n.li) let args = newRstNode(rnDirArg) var lang = language if language == "cpp": lang = "c++" @@ -1225,7 +1247,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef - result = newRstNode(newKind, newSons, loc=n.loc) + result = newRstNode(newKind, newSons, li=n.li) elif match(p, p.idx, ":w:"): # a role: let (roleName, lastIdx) = getRefname(p, p.idx+1) @@ -1318,7 +1340,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) = while currentTok(p).kind in {tkWord, tkPunct}: if currentTok(p).kind == tkPunct: if isInlineMarkupEnd(p, "_", exact=true): - reference = newRstNode(rnRef, loc=newLocation(p, saveIdx)) + reference = newRstNode(rnRef, li=newLocation(p, saveIdx)) break if not validRefnamePunct(currentTok(p).symbol): break @@ -1399,7 +1421,7 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, else: rstMessage(p, meExpected, postfix, line, col) proc parseMarkdownCodeblock(p: var RstParser): PRstNode = - result = newRstNodeA(p, rnCodeBlock, loc=newLocation(p)) + result = newRstNodeA(p, rnCodeBlock, li=newLocation(p)) var args = newRstNode(rnDirArg) if currentTok(p).kind == tkWord: args.add(newLeaf(p)) @@ -1550,12 +1572,12 @@ proc parseInline(p: var RstParser, father: PRstNode) = n = n.toOtherRole(k, roleName) father.add(n) elif isInlineMarkupStart(p, "`"): - var n = newRstNode(rnInterpretedText, loc=newLocation(p, p.idx+1)) + var n = newRstNode(rnInterpretedText, li=newLocation(p, p.idx+1)) parseUntil(p, n, "`", false) # bug #17260 n = parsePostfix(p, n) father.add(n) elif isInlineMarkupStart(p, "|"): - var n = newRstNode(rnSubstitutionReferences, loc=newLocation(p, p.idx+1)) + var n = newRstNode(rnSubstitutionReferences, li=newLocation(p, p.idx+1)) parseUntil(p, n, "|", false) father.add(n) elif roSupportMarkdown in p.s.options and @@ -1564,7 +1586,9 @@ proc parseInline(p: var RstParser, father: PRstNode) = discard "parseMarkdownLink already processed it" elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and (n = parseFootnoteName(p, reference=true); n != nil): - var nn = newRstNode(rnFootnoteRef, loc=newLocation(p, saveIdx+1)) + var nn = newRstNode(rnFootnoteRef) + nn.loc = new PFootnoteRefInfo + nn.loc.li = newLocation(p, saveIdx+1)[] nn.add n let (fnType, _) = getFootnoteType(n) case fnType @@ -1706,7 +1730,7 @@ proc parseField(p: var RstParser): PRstNode = ## Returns a parsed rnField node. ## ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody. - result = newRstNode(rnField, loc=newLocation(p)) + result = newRstNode(rnField, li=newLocation(p)) var col = currentTok(p).col var fieldname = newRstNode(rnFieldName) parseUntil(p, fieldname, ":", false) @@ -2482,7 +2506,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to ## check them before accessing. result = newRstNodeA(p, k) - if k == rnCodeBlock: result.loc = newLocation(p) + if k == rnCodeBlock: result.li = newLocation(p) var args: PRstNode = nil var options: PRstNode = nil if hasArg in flags: @@ -2607,7 +2631,8 @@ proc dirInclude(p: var RstParser): PRstNode = var q: RstParser initParser(q, p.s) - q.s.filename = path + let saveFileIdx = p.s.currFileIdx + setCurrFilename(p.s, path) getTokens( inputString[startPosition..endPosition].strip(), q.tok) @@ -2615,6 +2640,7 @@ proc dirInclude(p: var RstParser): PRstNode = #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: # InternalError("Too many binary zeros in include file") result = parseDoc(q) + p.s.currFileIdx = saveFileIdx proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = ## Parses a code block. @@ -2648,7 +2674,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) assert result.sons[1].kind == rnFieldList # Hook the extra field and specify the Nim language as value. - var extraNode = newRstNode(rnField, loc=newLocation(p)) + var extraNode = newRstNode(rnField, li=newLocation(p)) extraNode.add(newRstNode(rnFieldName)) extraNode.add(newRstNode(rnFieldBody)) extraNode.sons[0].add newLeaf("default-language") @@ -2853,7 +2879,6 @@ proc parseDotDot(p: var RstParser): PRstNode = proc rstParsePass1*(fragment, filename: string, line, column: int, - options: RstParseOptions, sharedState: PRstSharedState): PRstNode = ## Parses an RST `fragment`. ## The result should be further processed by @@ -2886,7 +2911,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = var key = addNodes(n) var e = getEnv(key) if e != "": result = newLeaf(e) - else: rstMessage(s, n.loc, mwUnknownSubstitution, key) + else: rstMessage(s.files, s.msgHandler, n.li, mwUnknownSubstitution, key) of rnHeadline, rnOverline: # fix up section levels depending on presence of a title and subtitle if s.hTitleCnt == 2: @@ -2911,7 +2936,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = result.sons = @[text, # visible text of reference newLeaf(anchor)] # link itself else: - rstMessage(s, n.loc, mwBrokenLink, refn) + rstMessage(s.files, s.msgHandler, n.li, mwBrokenLink, refn) of rnFootnote: var (fnType, num) = getFootnoteType(n.sons[0]) case fnType @@ -2938,7 +2963,8 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = result.add(nn) var refn = fnType.prefix # create new rnFootnoteRef, add final label, and finalize target refn: - result = newRstNode(rnFootnoteRef, loc=n.loc) + result = newRstNode(rnFootnoteRef) + result.loc = n.loc case fnType of fnManualNumber: addLabel num @@ -2959,7 +2985,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = if anch != "": result.add newLeaf(anch) # add link else: - rstMessage(s, n.loc, mwBrokenLink, refn) + rstMessage(s.files, s.msgHandler, n.loc.li, mwBrokenLink, refn) result.add newLeaf(refn) # add link of rnLeaf: discard @@ -2990,11 +3016,14 @@ proc rstParse*(text, filename: string, line, column: int, hasToc: var bool, options: RstParseOptions, findFile: FindFileHandler = nil, - msgHandler: MsgHandler = nil): PRstNode = - ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`. + msgHandler: MsgHandler = nil): (PRstNode, seq[string]) = + ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, + ## note that 2nd tuple element should be fed to `initRstGenerator` + ## argument `files` (it is being filled here at least with `filename` + ## and possibly with other files from RST ``.. include::`` statement). var sharedState = newRstSharedState(options, filename, findFile, msgHandler) - let unresolved = rstParsePass1(text, filename, line, column, - options, sharedState) + let unresolved = rstParsePass1(text, filename, line, column, sharedState) preparePass2(sharedState, unresolved) - result = resolveSubs(sharedState, unresolved) + result[0] = resolveSubs(sharedState, unresolved) + result[1] = sharedState.files hasToc = sharedState.hasToc diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 8a68d950c1499..8027564a0d049 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -9,7 +9,7 @@ ## This module implements an AST for the `reStructuredText`:idx: parser. -import strutils, json +import strutils, json, ../../../compiler/lineinfos type RstNodeKind* = enum ## the possible node kinds of an PRstNode @@ -77,11 +77,10 @@ type PRstNode* = ref RstNode ## an RST node RstNodeSeq* = seq[PRstNode] - PRstLocation* = ref object ## location inside a file - filename*: string - line*: int - col*: int - order*: int ## only for `rnFootnoteRef` + PFootnoteRefInfo* = ref object ## location inside a file, only for + ## `rnFootnoteRef` (packed for memory saving) + li*: TLineInfo + order*: int RstNode* {.acyclic, final.} = object ## AST node (result of RST parsing) case kind*: RstNodeKind ## the node's kind of rnLeaf, rnSmiley: @@ -100,10 +99,12 @@ type of rnFootnote, rnCitation, rnOptionListItem: order*: int ## footnote order (for auto-symbol footnotes and ## auto-numbered ones without a label) - of rnRef, rnSubstitutionReferences, rnFootnoteRef, + of rnRef, rnSubstitutionReferences, rnInterpretedText, rnField, rnInlineCode, rnCodeBlock: - loc*: PRstLocation ## To have line/column info for warnings at + li*: ref TLineInfo ## To have line/column info for warnings at ## nodes that are post-processed after parsing + of rnFootnoteRef: + loc*: PFootnoteRefInfo # almost the same else: discard anchor*: string ## anchor, internal link target @@ -114,9 +115,10 @@ proc len*(n: PRstNode): int = result = len(n.sons) proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], - anchor = "", loc: PRstLocation = nil): PRstNode = + anchor = "", li: ref TLineInfo = nil): PRstNode = result = PRstNode(kind: kind, sons: sons) - if loc != nil: result.loc = loc + result.anchor = anchor + if li != nil: result.li = li proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = assert kind in {rnLeaf, rnSmiley} diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 0ed31a876b580..b8ef4523a09b9 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils + algorithm, parseutils, ../../../compiler/lineinfos import ../../std/private/since @@ -72,11 +72,11 @@ type tocPart*: seq[TocEntry] hasToc*: bool theIndex: string # Contents of the index file to be dumped at the end. - options*: RstParseOptions findFile*: FindFileHandler msgHandler*: MsgHandler outDir*: string ## output directory, initialized by docgen.nim destFile*: string ## output (HTML) file, initialized by docgen.nim + files*: seq[string] filename*: string ## source Nim or Rst file meta*: array[MetaEnum, string] currentSection: string ## \ @@ -112,9 +112,9 @@ proc init(p: var CodeBlockParams) = proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, config: StringTableRef, filename: string, - options: RstParseOptions, findFile: FindFileHandler = nil, - msgHandler: MsgHandler = nil) = + msgHandler: MsgHandler = nil, + files: seq[string] = @[]) = ## Initializes a ``RstGenerator``. ## ## You need to call this before using a ``RstGenerator`` with any other @@ -160,9 +160,9 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, g.target = target g.tocPart = @[] g.filename = filename + g.files = files g.splitAfter = 20 g.theIndex = "" - g.options = options g.findFile = findFile g.currentSection = "" g.id = 0 @@ -909,7 +909,7 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = template err(msg: string) = - d.msgHandler(n.loc.filename, n.loc.line, n.loc.col, meInvalidRstField, msg) + rstMessage(d.files, d.msgHandler, n.li, meInvalidRstField, msg) let value = n.getFieldValue var number: int let nChars = parseInt(value, number) @@ -957,8 +957,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage else: - d.msgHandler(n.loc.filename, n.loc.line, n.loc.col, - mwUnsupportedField, n.getArgument) + rstMessage(d.files, d.msgHandler, n.li, mwUnsupportedField, n.getArgument) proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = ## Iterates over all code block fields and returns processed params. @@ -1069,8 +1068,7 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) = dispA(d.target, result, blockStart, blockStart, []) if params.lang == langNone: if len(params.langStr) > 0: - d.msgHandler(n.loc.filename, n.loc.line, n.loc.col, - mwUnsupportedLanguage, params.langStr) + rstMessage(d.files, d.msgHandler, n.li, mwUnsupportedLanguage, params.langStr) for letter in m.text: escChar(d.target, result, letter, emText) else: renderCodeLang(result, params.lang, m.text, d.target) @@ -1565,11 +1563,11 @@ proc rstToHtml*(s: string, options: RstParseOptions, result = "" const filen = "input" - var d: RstGenerator - initRstGenerator(d, outHtml, config, filen, options, myFindFile, msgHandler) var dummyHasToc = false - var rst = rstParse(s, filen, line=LineRstInit, column=ColRstInit, - dummyHasToc, options, myFindFile, msgHandler) + var (rst, files) = rstParse(s, filen, line=LineRstInit, column=ColRstInit, + dummyHasToc, options, myFindFile, msgHandler) + var d: RstGenerator + initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, files) result = "" renderRstToOut(d, rst, result) @@ -1578,10 +1576,9 @@ proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, ## Convenience proc for `renderRstToOut` and `initRstGenerator`. runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}""" if rstSource.len == 0: return - var option: bool + var dummyHasToc: bool + let (rst, files) = rstParse(rstSource, "", line=LineRstInit, column=ColRstInit, + dummyHasToc, options) var rstGenera: RstGenerator - rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", options) - rstGenera.renderRstToOut( - rstParse(rstSource, "", line=LineRstInit, column=ColRstInit, - option, options), - result) + rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", files=files) + rstGenera.renderRstToOut(rst, result) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 65725138af165..9acdd11cec5c7 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -54,8 +54,8 @@ proc toAst(input: string, result = "" var dummyHasToc = false - var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit, - dummyHasToc, rstOptions, myFindFile, testMsgHandler) + var (rst, files) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, + dummyHasToc, rstOptions, myFindFile, testMsgHandler) result = renderRstToStr(rst) except EParseError as e: if e.msg != "": diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index fba977f9b4514..997ac27b446bb 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -420,9 +420,10 @@ Some chapter "the following intermediate section level(s) are missing on " & "lines 12..15: underline -----)") + test "RST sections overline": # the same as input9good but with overline headings # first overline heading has a special meaning: document title - let input10 = dedent """ + let input = dedent """ ====== Title0 ====== @@ -456,20 +457,22 @@ Some chapter """ var option: bool var rstGenera: RstGenerator - var output10: string - rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {}) - rstGenera.renderRstToOut(rstParse(input10, "", 1, 1, option, {}), output10) + var output: string + let (rst, files) = rstParse(input, "", 1, 1, option, {}) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" doAssert rstGenera.meta[metaSubTitle] == "SubTitle0" - doAssert "

Level1

" in output10 - doAssert "

Level2

" in output10 - doAssert "

Level3

" in output10 - doAssert "

L1

" in output10 - doAssert "

Another2

" in output10 - doAssert "

More3

" in output10 - + doAssert "

Level1

" in output + doAssert "

Level2

" in output + doAssert "

Level3

" in output + doAssert "

L1

" in output + doAssert "

Another2

" in output + doAssert "

More3

" in output + + test "RST sections overline 2": # check that a paragraph prevents interpreting overlines as document titles - let input11 = dedent """ + let input = dedent """ Paragraph ====== @@ -480,18 +483,20 @@ Some chapter SubTitle0 +++++++++ """ - var option11: bool - var rstGenera11: RstGenerator - var output11: string - rstGenera11.initRstGenerator(outHtml, defaultConfig(), "input", {}) - rstGenera11.renderRstToOut(rstParse(input11, "", 1, 1, option11, {}), output11) - doAssert rstGenera11.meta[metaTitle] == "" - doAssert rstGenera11.meta[metaSubTitle] == "" - doAssert "

Title0

" in output11 - doAssert "

SubTitle0

" in output11 - + var option: bool + var rstGenera: RstGenerator + var output: string + let (rst, files) = rstParse(input, "", 1, 1, option, {}) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.renderRstToOut(rst, output) + doAssert rstGenera.meta[metaTitle] == "" + doAssert rstGenera.meta[metaSubTitle] == "" + doAssert "

Title0

" in output + doAssert "

SubTitle0

" in output + + test "RST+Markdown sections": # check that RST and Markdown headings don't interfere - let input12 = dedent """ + let input = dedent """ ====== Title0 ====== @@ -509,14 +514,15 @@ Some chapter MySection2a ----------- """ - var option12: bool - var rstGenera12: RstGenerator - var output12: string - rstGenera12.initRstGenerator(outHtml, defaultConfig(), "input", {}) - rstGenera12.renderRstToOut(rstParse(input12, "", 1, 1, option12, {roSupportMarkdown}), output12) - doAssert rstGenera12.meta[metaTitle] == "Title0" - doAssert rstGenera12.meta[metaSubTitle] == "" - doAssert output12 == + var option: bool + var rstGenera: RstGenerator + var output: string + let (rst, files) = rstParse(input, "", 1, 1, option, {roSupportMarkdown}) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.renderRstToOut(rst, output) + doAssert rstGenera.meta[metaTitle] == "Title0" + doAssert rstGenera.meta[metaSubTitle] == "" + doAssert output == "\n

MySection1a

" & # RST "\n

MySection1b

" & # Markdown "\n

MySection1c

" & # RST @@ -653,8 +659,8 @@ Check that comment disappears: let output1 = input1.toHtml doAssert output1 == "Check that comment disappears:" - test "RST line blocks": - let input1 = """ + test "RST line blocks + headings": + let input = """ ===== Test1 ===== @@ -667,17 +673,19 @@ Test1 """ var option: bool var rstGenera: RstGenerator - var output1: string - rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {}) - rstGenera.renderRstToOut(rstParse(input1, "", 1, 1, option, {}), output1) + var output: string + let (rst, files) = rstParse(input, "", 1, 1, option, {}) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Test1" # check that title was not overwritten to '|' - doAssert output1 == "



line block
other line

" - let output1l = rstToLatex(input1, {}) + doAssert output == "



line block
other line

" + let output1l = rstToLatex(input, {}) doAssert "line block\n\n" in output1l doAssert "other line\n\n" in output1l doAssert output1l.count("\\vspace") == 2 + 2 # +2 surrounding paddings + test "RST line blocks": let input2 = dedent""" Paragraph1 From 1b31d616fb3167b5f3f5208b23bb9da4e4be1d17 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 3 Jul 2021 00:49:39 +0300 Subject: [PATCH 04/29] fix blank lines in include file (rm harmful strip) --- lib/packages/docutils/rst.nim | 2 +- tests/stdlib/trst.nim | 31 +++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 4f8022f4c7dd8..bdc762c7c3880 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2634,7 +2634,7 @@ proc dirInclude(p: var RstParser): PRstNode = let saveFileIdx = p.s.currFileIdx setCurrFilename(p.s, path) getTokens( - inputString[startPosition..endPosition].strip(), + inputString[startPosition..endPosition], q.tok) # workaround a GCC bug; more like the interior pointer bug? #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 9acdd11cec5c7..15fba7675cbed 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -382,6 +382,29 @@ suite "Warnings": "input(9, 6) Warning: broken link 'shortdotlink'" ]) + test "With include directive and blank lines at the beginning": + "other.rst".writeFile(dedent""" + + + firstParagraph + + here brokenLink_""") + let input = ".. include:: other.rst" + var warnings = new seq[string] + let output = input.toAst(warnings=warnings) + check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"] + check(output == dedent""" + rnInner + rnParagraph + rnLeaf 'firstParagraph' + rnParagraph + rnLeaf 'here' + rnLeaf ' ' + rnRef + rnLeaf 'brokenLink' + """) + removeFile("other.rst") + suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") @@ -400,7 +423,7 @@ OtherStart .. include:: other.rst :start-after: OtherStart """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + check "Visible " == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") test "Include everything before": @@ -414,7 +437,7 @@ And this should **NOT** be visible in `docs.html` .. include:: other.rst :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible " == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") @@ -432,7 +455,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + check "Visible " == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") @@ -452,7 +475,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible " == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") suite "RST escaping": From 80cafde68e7206491606292b86eb2f20d5c95172 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 3 Jul 2021 18:04:28 +0300 Subject: [PATCH 05/29] don't use ref TLineInfo --- lib/packages/docutils/rst.nim | 33 +++++++++++++++----------------- lib/packages/docutils/rstast.nim | 12 ++++++++---- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index bdc762c7c3880..1dc143dbbc104 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -628,8 +628,7 @@ proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) proc rstMessage*(files: seq[string], f: MsgHandler, - li: ref TLineInfo | TLineInfo, - msgKind: MsgKind, arg: string) = + li: TLineInfo, msgKind: MsgKind, arg: string) = ## Print warnings using `li: TLineInfo`, i.e. in 2nd-pass warnings for ## footnotes/substitutions/references or from ``rstgen.nim``. let file = getFilename(files, li.fileIndex) @@ -961,10 +960,9 @@ proc getAutoSymbol(s: PRstSharedState, order: int): string = if fnote.autoSymIdx == order: return fnote.label -proc newRstNodeA(p: var RstParser, kind: RstNodeKind, - li: ref TLineInfo = nil): PRstNode = +proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode = ## create node and consume the current anchor - result = newRstNode(kind, li=li) + result = newRstNode(kind) if p.curAnchor != "": result.anchor = p.curAnchor p.curAnchor = "" @@ -1182,14 +1180,12 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind = if result == rnUnknownRole: rstMessage(p, mwUnsupportedLanguage, sym) -proc newLocation(p: RstParser, iTok: int): ref TLineInfo = - new result +proc lineInfo(p: RstParser, iTok: int): TLineInfo = result.col = int16(p.col + p.tok[iTok].col) result.line = uint16(p.line + p.tok[iTok].line) result.fileIndex = p.s.currFileIdx -proc newLocation(p: RstParser): ref TLineInfo = - newLocation(p, p.idx) +proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) proc toInlineCode(n: PRstNode, language: string): PRstNode = ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). @@ -1247,7 +1243,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef - result = newRstNode(newKind, newSons, li=n.li) + result = newRstNode(newKind, sons=newSons, li=n.li) elif match(p, p.idx, ":w:"): # a role: let (roleName, lastIdx) = getRefname(p, p.idx+1) @@ -1340,7 +1336,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) = while currentTok(p).kind in {tkWord, tkPunct}: if currentTok(p).kind == tkPunct: if isInlineMarkupEnd(p, "_", exact=true): - reference = newRstNode(rnRef, li=newLocation(p, saveIdx)) + reference = newRstNode(rnRef, li=lineInfo(p, saveIdx)) break if not validRefnamePunct(currentTok(p).symbol): break @@ -1421,7 +1417,8 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, else: rstMessage(p, meExpected, postfix, line, col) proc parseMarkdownCodeblock(p: var RstParser): PRstNode = - result = newRstNodeA(p, rnCodeBlock, li=newLocation(p)) + result = newRstNodeA(p, rnCodeBlock) + result.li = lineInfo(p) var args = newRstNode(rnDirArg) if currentTok(p).kind == tkWord: args.add(newLeaf(p)) @@ -1572,12 +1569,12 @@ proc parseInline(p: var RstParser, father: PRstNode) = n = n.toOtherRole(k, roleName) father.add(n) elif isInlineMarkupStart(p, "`"): - var n = newRstNode(rnInterpretedText, li=newLocation(p, p.idx+1)) + var n = newRstNode(rnInterpretedText, li=lineInfo(p, p.idx+1)) parseUntil(p, n, "`", false) # bug #17260 n = parsePostfix(p, n) father.add(n) elif isInlineMarkupStart(p, "|"): - var n = newRstNode(rnSubstitutionReferences, li=newLocation(p, p.idx+1)) + var n = newRstNode(rnSubstitutionReferences, li=lineInfo(p, p.idx+1)) parseUntil(p, n, "|", false) father.add(n) elif roSupportMarkdown in p.s.options and @@ -1588,7 +1585,7 @@ proc parseInline(p: var RstParser, father: PRstNode) = (n = parseFootnoteName(p, reference=true); n != nil): var nn = newRstNode(rnFootnoteRef) nn.loc = new PFootnoteRefInfo - nn.loc.li = newLocation(p, saveIdx+1)[] + nn.loc.li = lineInfo(p, saveIdx+1) nn.add n let (fnType, _) = getFootnoteType(n) case fnType @@ -1730,7 +1727,7 @@ proc parseField(p: var RstParser): PRstNode = ## Returns a parsed rnField node. ## ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody. - result = newRstNode(rnField, li=newLocation(p)) + result = newRstNode(rnField, li=lineInfo(p)) var col = currentTok(p).col var fieldname = newRstNode(rnFieldName) parseUntil(p, fieldname, ":", false) @@ -2506,7 +2503,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to ## check them before accessing. result = newRstNodeA(p, k) - if k == rnCodeBlock: result.li = newLocation(p) + if k == rnCodeBlock: result.li = lineInfo(p) var args: PRstNode = nil var options: PRstNode = nil if hasArg in flags: @@ -2674,7 +2671,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) assert result.sons[1].kind == rnFieldList # Hook the extra field and specify the Nim language as value. - var extraNode = newRstNode(rnField, li=newLocation(p)) + var extraNode = newRstNode(rnField, li=lineInfo(p)) extraNode.add(newRstNode(rnFieldName)) extraNode.add(newRstNode(rnFieldBody)) extraNode.sons[0].add newLeaf("default-language") diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 8027564a0d049..662a0fbaceae3 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -101,10 +101,10 @@ type ## auto-numbered ones without a label) of rnRef, rnSubstitutionReferences, rnInterpretedText, rnField, rnInlineCode, rnCodeBlock: - li*: ref TLineInfo ## To have line/column info for warnings at + li*: TLineInfo ## To have line/column info for warnings at ## nodes that are post-processed after parsing of rnFootnoteRef: - loc*: PFootnoteRefInfo # almost the same + loc*: PFootnoteRefInfo ## almost the same else: discard anchor*: string ## anchor, internal link target @@ -115,10 +115,14 @@ proc len*(n: PRstNode): int = result = len(n.sons) proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], - anchor = "", li: ref TLineInfo = nil): PRstNode = + anchor = ""): PRstNode = result = PRstNode(kind: kind, sons: sons) result.anchor = anchor - if li != nil: result.li = li + +proc newRstNode*(kind: RstNodeKind, li: TLineInfo, + sons: seq[PRstNode] = @[]): PRstNode = + result = PRstNode(kind: kind, sons: sons) + result.li = li proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = assert kind in {rnLeaf, rnSmiley} From ce5de16612993082088e887e4b617559a2a69686 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 6 Jul 2021 23:50:05 +0300 Subject: [PATCH 06/29] return `hasToc` as output parameter for uniformity --- lib/packages/docutils/rst.nim | 11 ++++++----- lib/packages/docutils/rstgen.nim | 10 ++++------ tests/stdlib/trst.nim | 5 ++--- tests/stdlib/trstgen.nim | 12 ++++-------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 1dc143dbbc104..0b3a88e4fa649 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -3010,10 +3010,11 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = result.sons = newSons proc rstParse*(text, filename: string, - line, column: int, hasToc: var bool, + line, column: int, options: RstParseOptions, findFile: FindFileHandler = nil, - msgHandler: MsgHandler = nil): (PRstNode, seq[string]) = + msgHandler: MsgHandler = nil): + tuple[node: PRstNode, files: seq[string], hasToc: bool] = ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, ## note that 2nd tuple element should be fed to `initRstGenerator` ## argument `files` (it is being filled here at least with `filename` @@ -3021,6 +3022,6 @@ proc rstParse*(text, filename: string, var sharedState = newRstSharedState(options, filename, findFile, msgHandler) let unresolved = rstParsePass1(text, filename, line, column, sharedState) preparePass2(sharedState, unresolved) - result[0] = resolveSubs(sharedState, unresolved) - result[1] = sharedState.files - hasToc = sharedState.hasToc + result.node = resolveSubs(sharedState, unresolved) + result.files = sharedState.files + result.hasToc = sharedState.hasToc diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index b8ef4523a09b9..143305b35056b 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1563,9 +1563,8 @@ proc rstToHtml*(s: string, options: RstParseOptions, result = "" const filen = "input" - var dummyHasToc = false - var (rst, files) = rstParse(s, filen, line=LineRstInit, column=ColRstInit, - dummyHasToc, options, myFindFile, msgHandler) + let (rst, files, _) = rstParse(s, filen, line=LineRstInit, column=ColRstInit, + options, myFindFile, msgHandler) var d: RstGenerator initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, files) result = "" @@ -1576,9 +1575,8 @@ proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, ## Convenience proc for `renderRstToOut` and `initRstGenerator`. runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}""" if rstSource.len == 0: return - var dummyHasToc: bool - let (rst, files) = rstParse(rstSource, "", line=LineRstInit, column=ColRstInit, - dummyHasToc, options) + let (rst, files, _) = rstParse(rstSource, "", + line=LineRstInit, column=ColRstInit, options) var rstGenera: RstGenerator rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", files=files) rstGenera.renderRstToOut(rst, result) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 15fba7675cbed..3e6bf114560e1 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -53,9 +53,8 @@ proc toAst(input: string, # we don't find any files in online mode: result = "" - var dummyHasToc = false - var (rst, files) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, - dummyHasToc, rstOptions, myFindFile, testMsgHandler) + var (rst, files, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, + rstOptions, myFindFile, testMsgHandler) result = renderRstToStr(rst) except EParseError as e: if e.msg != "": diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 997ac27b446bb..f3d61e2179ecb 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -455,10 +455,9 @@ Some chapter ~~~~~ """ - var option: bool var rstGenera: RstGenerator var output: string - let (rst, files) = rstParse(input, "", 1, 1, option, {}) + let (rst, files, _) = rstParse(input, "", 1, 1, {}) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" @@ -483,10 +482,9 @@ Some chapter SubTitle0 +++++++++ """ - var option: bool var rstGenera: RstGenerator var output: string - let (rst, files) = rstParse(input, "", 1, 1, option, {}) + let (rst, files, _) = rstParse(input, "", 1, 1, {}) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "" @@ -514,10 +512,9 @@ Some chapter MySection2a ----------- """ - var option: bool var rstGenera: RstGenerator var output: string - let (rst, files) = rstParse(input, "", 1, 1, option, {roSupportMarkdown}) + let (rst, files, _) = rstParse(input, "", 1, 1, {roSupportMarkdown}) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" @@ -671,10 +668,9 @@ Test1 | other line """ - var option: bool var rstGenera: RstGenerator var output: string - let (rst, files) = rstParse(input, "", 1, 1, option, {}) + let (rst, files, _) = rstParse(input, "", 1, 1, {}) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Test1" From ed47dbd291bf08afacbec8e813a0ea423ba7aa61 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 7 Jul 2021 11:18:10 +0300 Subject: [PATCH 07/29] Update compiler/docgen.nim Co-authored-by: Timothee Cour --- compiler/docgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index adb6b1089f226..501f70c065397 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -202,7 +202,7 @@ proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex} proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil, - isPureRst=false): PDoc = + isPureRst = false): PDoc = declareClosures() new(result) result.module = module From cf27ed9694459f198d433aa5fadae72922d04651 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 7 Jul 2021 11:18:19 +0300 Subject: [PATCH 08/29] Update compiler/docgen.nim Co-authored-by: Timothee Cour --- compiler/docgen.nim | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 501f70c065397..44dc098535774 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -210,12 +210,8 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.cache = cache result.outDir = conf.outDir.string result.isPureRst = isPureRst - var options: RstParseOptions = {} - if isPureRst: - options = {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown} - else: - options = {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, - roNimFile} + var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown} + if not isPureRst: options.incl roNimFile result.sharedState = newRstSharedState( options, filename.string, docgenFindFile, compilerMsgHandler) From 19fa19e7a5f1b7d49c0328863621d039d92af0dd Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 7 Jul 2021 11:20:56 +0300 Subject: [PATCH 09/29] Update lib/packages/docutils/rst.nim Co-authored-by: Timothee Cour --- lib/packages/docutils/rst.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 0b3a88e4fa649..1e44984a2bbb7 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -510,7 +510,7 @@ type # number, order of occurrence msgHandler: MsgHandler # How to handle errors. findFile: FindFileHandler # How to find files. - files*: seq[string] # map FileIndex -> file name (for storing + filenames*: seq[string] # map FileIndex -> file name (for storing # file names for warnings after 1st stage) currFileIdx: FileIndex # current index in `files` hasToc*: bool From b361cd32ecf240aec1ffad8034f4c11195d3e910 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 7 Jul 2021 11:34:50 +0300 Subject: [PATCH 10/29] address review - stylistic things --- compiler/docgen.nim | 4 +-- lib/packages/docutils/rst.nim | 61 ++++++++++++++++---------------- lib/packages/docutils/rstast.nim | 8 ++--- lib/packages/docutils/rstgen.nim | 28 ++++++++------- tests/stdlib/trst.nim | 4 +-- tests/stdlib/trstgen.nim | 8 ++--- 6 files changed, 59 insertions(+), 54 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 44dc098535774..9466d55fa252d 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1171,8 +1171,8 @@ proc finishGenerateDoc*(d: var PDoc) = d.jEntriesFinal.add entry.json d.jEntriesPre[i].rst = nil - # pass file map `files` to ``rstgen.nim`` for its warnings - d.files = move(d.sharedState.files) + # pass file map `filenames` to ``rstgen.nim`` for its warnings + d.filenames = move(d.sharedState.filenames) proc add(d: PDoc; j: JsonItem) = if j.json != nil or j.rst != nil: d.jEntriesPre.add j diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 1e44984a2bbb7..afab0f04d3ed3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -510,9 +510,9 @@ type # number, order of occurrence msgHandler: MsgHandler # How to handle errors. findFile: FindFileHandler # How to find files. - filenames*: seq[string] # map FileIndex -> file name (for storing + filenames*: seq[string] # map FileIndex -> file name (for storing # file names for warnings after 1st stage) - currFileIdx: FileIndex # current index in `files` + currFileIdx: FileIndex # current index in `filesnames` hasToc*: bool PRstSharedState* = ref RstSharedState @@ -583,21 +583,21 @@ proc whichRoleAux(sym: string): RstNodeKind = result = rnUnknownRole proc setCurrFilename(s: PRstSharedState, file1: string) = - for i, file2 in s.files: + for i, file2 in s.filenames: if file1 == file2: s.currFileIdx = i.FileIndex return - s.files.add file1 - s.currFileIdx = (s.files.len - 1).FileIndex + s.filenames.add file1 + s.currFileIdx = (s.filenames.len - 1).FileIndex -proc getFilename(files: seq[string], fid: FileIndex): string = - if fid.int < files.len: - result = files[fid.int] +proc getFilename(filenames: seq[string], fid: FileIndex): string = + if fid.int < filenames.len: + result = filenames[fid.int] else: result = "input" proc currFilename(s: PRstSharedState): string = - getFilename(s.files, s.currFileIdx) + getFilename(s.filenames, s.currFileIdx) proc newRstSharedState*(options: RstParseOptions, filename: string, @@ -627,12 +627,12 @@ proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) -proc rstMessage*(files: seq[string], f: MsgHandler, - li: TLineInfo, msgKind: MsgKind, arg: string) = - ## Print warnings using `li: TLineInfo`, i.e. in 2nd-pass warnings for +proc rstMessage*(filenames: seq[string], f: MsgHandler, + info: TLineInfo, msgKind: MsgKind, arg: string) = + ## Print warnings using `info: TLineInfo`, i.e. in 2nd-pass warnings for ## footnotes/substitutions/references or from ``rstgen.nim``. - let file = getFilename(files, li.fileIndex) - f(file, li.line.int, li.col.int, msgKind, arg) + let file = getFilename(filenames, info.fileIndex) + f(file, info.line.int, info.col.int, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = p.s.msgHandler(p.s.currFilename, p.line + line, @@ -1189,7 +1189,7 @@ proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) proc toInlineCode(n: PRstNode, language: string): PRstNode = ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). - result = newRstNode(rnInlineCode, li=n.li) + result = newRstNode(rnInlineCode, info=n.info) let args = newRstNode(rnDirArg) var lang = language if language == "cpp": lang = "c++" @@ -1243,7 +1243,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef - result = newRstNode(newKind, sons=newSons, li=n.li) + result = newRstNode(newKind, sons=newSons, info=n.info) elif match(p, p.idx, ":w:"): # a role: let (roleName, lastIdx) = getRefname(p, p.idx+1) @@ -1336,7 +1336,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) = while currentTok(p).kind in {tkWord, tkPunct}: if currentTok(p).kind == tkPunct: if isInlineMarkupEnd(p, "_", exact=true): - reference = newRstNode(rnRef, li=lineInfo(p, saveIdx)) + reference = newRstNode(rnRef, info=lineInfo(p, saveIdx)) break if not validRefnamePunct(currentTok(p).symbol): break @@ -1418,7 +1418,7 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, proc parseMarkdownCodeblock(p: var RstParser): PRstNode = result = newRstNodeA(p, rnCodeBlock) - result.li = lineInfo(p) + result.info = lineInfo(p) var args = newRstNode(rnDirArg) if currentTok(p).kind == tkWord: args.add(newLeaf(p)) @@ -1569,12 +1569,12 @@ proc parseInline(p: var RstParser, father: PRstNode) = n = n.toOtherRole(k, roleName) father.add(n) elif isInlineMarkupStart(p, "`"): - var n = newRstNode(rnInterpretedText, li=lineInfo(p, p.idx+1)) + var n = newRstNode(rnInterpretedText, info=lineInfo(p, p.idx+1)) parseUntil(p, n, "`", false) # bug #17260 n = parsePostfix(p, n) father.add(n) elif isInlineMarkupStart(p, "|"): - var n = newRstNode(rnSubstitutionReferences, li=lineInfo(p, p.idx+1)) + var n = newRstNode(rnSubstitutionReferences, info=lineInfo(p, p.idx+1)) parseUntil(p, n, "|", false) father.add(n) elif roSupportMarkdown in p.s.options and @@ -1585,7 +1585,7 @@ proc parseInline(p: var RstParser, father: PRstNode) = (n = parseFootnoteName(p, reference=true); n != nil): var nn = newRstNode(rnFootnoteRef) nn.loc = new PFootnoteRefInfo - nn.loc.li = lineInfo(p, saveIdx+1) + nn.loc.info = lineInfo(p, saveIdx+1) nn.add n let (fnType, _) = getFootnoteType(n) case fnType @@ -1727,7 +1727,7 @@ proc parseField(p: var RstParser): PRstNode = ## Returns a parsed rnField node. ## ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody. - result = newRstNode(rnField, li=lineInfo(p)) + result = newRstNode(rnField, info=lineInfo(p)) var col = currentTok(p).col var fieldname = newRstNode(rnFieldName) parseUntil(p, fieldname, ":", false) @@ -2503,7 +2503,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to ## check them before accessing. result = newRstNodeA(p, k) - if k == rnCodeBlock: result.li = lineInfo(p) + if k == rnCodeBlock: result.info = lineInfo(p) var args: PRstNode = nil var options: PRstNode = nil if hasArg in flags: @@ -2671,7 +2671,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) assert result.sons[1].kind == rnFieldList # Hook the extra field and specify the Nim language as value. - var extraNode = newRstNode(rnField, li=lineInfo(p)) + var extraNode = newRstNode(rnField, info=lineInfo(p)) extraNode.add(newRstNode(rnFieldName)) extraNode.add(newRstNode(rnFieldBody)) extraNode.sons[0].add newLeaf("default-language") @@ -2908,7 +2908,8 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = var key = addNodes(n) var e = getEnv(key) if e != "": result = newLeaf(e) - else: rstMessage(s.files, s.msgHandler, n.li, mwUnknownSubstitution, key) + else: rstMessage(s.filenames, s.msgHandler, n.info, + mwUnknownSubstitution, key) of rnHeadline, rnOverline: # fix up section levels depending on presence of a title and subtitle if s.hTitleCnt == 2: @@ -2933,7 +2934,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = result.sons = @[text, # visible text of reference newLeaf(anchor)] # link itself else: - rstMessage(s.files, s.msgHandler, n.li, mwBrokenLink, refn) + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) of rnFootnote: var (fnType, num) = getFootnoteType(n.sons[0]) case fnType @@ -2982,7 +2983,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = if anch != "": result.add newLeaf(anch) # add link else: - rstMessage(s.files, s.msgHandler, n.loc.li, mwBrokenLink, refn) + rstMessage(s.filenames, s.msgHandler, n.loc.info, mwBrokenLink, refn) result.add newLeaf(refn) # add link of rnLeaf: discard @@ -3014,14 +3015,14 @@ proc rstParse*(text, filename: string, options: RstParseOptions, findFile: FindFileHandler = nil, msgHandler: MsgHandler = nil): - tuple[node: PRstNode, files: seq[string], hasToc: bool] = + tuple[node: PRstNode, filenames: seq[string], hasToc: bool] = ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, ## note that 2nd tuple element should be fed to `initRstGenerator` - ## argument `files` (it is being filled here at least with `filename` + ## argument `filenames` (it is being filled here at least with `filename` ## and possibly with other files from RST ``.. include::`` statement). var sharedState = newRstSharedState(options, filename, findFile, msgHandler) let unresolved = rstParsePass1(text, filename, line, column, sharedState) preparePass2(sharedState, unresolved) result.node = resolveSubs(sharedState, unresolved) - result.files = sharedState.files + result.filenames = sharedState.filenames result.hasToc = sharedState.hasToc diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 662a0fbaceae3..9a5547956be29 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -79,7 +79,7 @@ type RstNodeSeq* = seq[PRstNode] PFootnoteRefInfo* = ref object ## location inside a file, only for ## `rnFootnoteRef` (packed for memory saving) - li*: TLineInfo + info*: TLineInfo order*: int RstNode* {.acyclic, final.} = object ## AST node (result of RST parsing) case kind*: RstNodeKind ## the node's kind @@ -101,7 +101,7 @@ type ## auto-numbered ones without a label) of rnRef, rnSubstitutionReferences, rnInterpretedText, rnField, rnInlineCode, rnCodeBlock: - li*: TLineInfo ## To have line/column info for warnings at + info*: TLineInfo ## To have line/column info for warnings at ## nodes that are post-processed after parsing of rnFootnoteRef: loc*: PFootnoteRefInfo ## almost the same @@ -119,10 +119,10 @@ proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], result = PRstNode(kind: kind, sons: sons) result.anchor = anchor -proc newRstNode*(kind: RstNodeKind, li: TLineInfo, +proc newRstNode*(kind: RstNodeKind, info: TLineInfo, sons: seq[PRstNode] = @[]): PRstNode = result = PRstNode(kind: kind, sons: sons) - result.li = li + result.info = info proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = assert kind in {rnLeaf, rnSmiley} diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 143305b35056b..881191b9215d9 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -76,7 +76,7 @@ type msgHandler*: MsgHandler outDir*: string ## output directory, initialized by docgen.nim destFile*: string ## output (HTML) file, initialized by docgen.nim - files*: seq[string] + filenames*: seq[string] filename*: string ## source Nim or Rst file meta*: array[MetaEnum, string] currentSection: string ## \ @@ -114,7 +114,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, config: StringTableRef, filename: string, findFile: FindFileHandler = nil, msgHandler: MsgHandler = nil, - files: seq[string] = @[]) = + filenames: seq[string] = @[]) = ## Initializes a ``RstGenerator``. ## ## You need to call this before using a ``RstGenerator`` with any other @@ -160,7 +160,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, g.target = target g.tocPart = @[] g.filename = filename - g.files = files + g.filenames = filenames g.splitAfter = 20 g.theIndex = "" g.findFile = findFile @@ -909,7 +909,7 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = template err(msg: string) = - rstMessage(d.files, d.msgHandler, n.li, meInvalidRstField, msg) + rstMessage(d.filenames, d.msgHandler, n.info, meInvalidRstField, msg) let value = n.getFieldValue var number: int let nChars = parseInt(value, number) @@ -957,7 +957,8 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage else: - rstMessage(d.files, d.msgHandler, n.li, mwUnsupportedField, n.getArgument) + rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedField, + n.getArgument) proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = ## Iterates over all code block fields and returns processed params. @@ -1068,7 +1069,8 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) = dispA(d.target, result, blockStart, blockStart, []) if params.lang == langNone: if len(params.langStr) > 0: - rstMessage(d.files, d.msgHandler, n.li, mwUnsupportedLanguage, params.langStr) + rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedLanguage, + params.langStr) for letter in m.text: escChar(d.target, result, letter, emText) else: renderCodeLang(result, params.lang, m.text, d.target) @@ -1563,10 +1565,11 @@ proc rstToHtml*(s: string, options: RstParseOptions, result = "" const filen = "input" - let (rst, files, _) = rstParse(s, filen, line=LineRstInit, column=ColRstInit, - options, myFindFile, msgHandler) + let (rst, filenames, _) = rstParse(s, filen, + line=LineRstInit, column=ColRstInit, + options, myFindFile, msgHandler) var d: RstGenerator - initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, files) + initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, filenames) result = "" renderRstToOut(d, rst, result) @@ -1575,8 +1578,9 @@ proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, ## Convenience proc for `renderRstToOut` and `initRstGenerator`. runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}""" if rstSource.len == 0: return - let (rst, files, _) = rstParse(rstSource, "", - line=LineRstInit, column=ColRstInit, options) + let (rst, filenames, _) = rstParse(rstSource, "", + line=LineRstInit, column=ColRstInit, + options) var rstGenera: RstGenerator - rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", files=files) + rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", filenames=filenames) rstGenera.renderRstToOut(rst, result) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 3e6bf114560e1..d792496602926 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -53,8 +53,8 @@ proc toAst(input: string, # we don't find any files in online mode: result = "" - var (rst, files, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, - rstOptions, myFindFile, testMsgHandler) + var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, + rstOptions, myFindFile, testMsgHandler) result = renderRstToStr(rst) except EParseError as e: if e.msg != "": diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index f3d61e2179ecb..a677a28124604 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -458,7 +458,7 @@ Some chapter var rstGenera: RstGenerator var output: string let (rst, files, _) = rstParse(input, "", 1, 1, {}) - rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames = files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" doAssert rstGenera.meta[metaSubTitle] == "SubTitle0" @@ -485,7 +485,7 @@ Some chapter var rstGenera: RstGenerator var output: string let (rst, files, _) = rstParse(input, "", 1, 1, {}) - rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "" doAssert rstGenera.meta[metaSubTitle] == "" @@ -515,7 +515,7 @@ Some chapter var rstGenera: RstGenerator var output: string let (rst, files, _) = rstParse(input, "", 1, 1, {roSupportMarkdown}) - rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" doAssert rstGenera.meta[metaSubTitle] == "" @@ -671,7 +671,7 @@ Test1 var rstGenera: RstGenerator var output: string let (rst, files, _) = rstParse(input, "", 1, 1, {}) - rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", files=files) + rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Test1" # check that title was not overwritten to '|' From 4d20a5fa308d0eca298b7c62db26bf686e38c2a4 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 7 Jul 2021 22:09:33 +0300 Subject: [PATCH 11/29] Update compiler/docgen.nim Co-authored-by: Timothee Cour --- compiler/docgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 9466d55fa252d..eb7881ed57ef0 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1429,7 +1429,7 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) = proc commandRstAux(cache: IdentCache, conf: ConfigRef; filename: AbsoluteFile, outExt: string) = var filen = addFileExt(filename, "txt") - var d = newDocumentor(filen, cache, conf, outExt, isPureRst=true) + let d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) let rst = parseRst(readFile(filen.string), filen.string, line=LineRstInit, column=ColRstInit, conf, d.sharedState) From d13c14df00cb2ae1ffa9fdf6c09cd7c622846445 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 7 Jul 2021 22:38:18 +0300 Subject: [PATCH 12/29] unify RST warnings/errors names --- compiler/docgen.nim | 24 +++++++-------- compiler/lineinfos.nim | 52 +++++++++++++++++--------------- compiler/main.nim | 4 +-- lib/packages/docutils/rst.nim | 3 +- lib/packages/docutils/rstgen.nim | 2 +- 5 files changed, 44 insertions(+), 41 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index eb7881ed57ef0..8e26b0e5f5613 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -159,18 +159,18 @@ template declareClosures = case msgKind of meCannotOpenFile: k = errCannotOpenFile of meExpected: k = errXExpected - of meGridTableNotImplemented: k = errGridTableNotImplemented + of meGridTableNotImplemented: k = errRstGridTableNotImplemented of meMarkdownIllformedTable: k = errMarkdownIllformedTable - of meNewSectionExpected: k = errNewSectionExpected - of meGeneralParseError: k = errGeneralParseError - of meInvalidDirective: k = errInvalidDirectiveX - of meInvalidRstField: k = errInvalidRstField - of meFootnoteMismatch: k = errFootnoteMismatch - of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel - of mwUnknownSubstitution: k = warnUnknownSubstitutionX - of mwBrokenLink: k = warnBrokenLink - of mwUnsupportedLanguage: k = warnLanguageXNotSupported - of mwUnsupportedField: k = warnFieldXNotSupported + of meNewSectionExpected: k = errRstNewSectionExpected + of meGeneralParseError: k = errRstGeneralParseError + of meInvalidDirective: k = errRstInvalidDirectiveX + of meInvalidField: k = errRstInvalidField + of meFootnoteMismatch: k = errRstFootnoteMismatch + of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel + of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX + of mwBrokenLink: k = warnRstBrokenLink + of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported + of mwUnsupportedField: k = warnRstFieldXNotSupported of mwRstStyle: k = warnRstStyle {.gcsafe.}: globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg) @@ -1429,7 +1429,7 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) = proc commandRstAux(cache: IdentCache, conf: ConfigRef; filename: AbsoluteFile, outExt: string) = var filen = addFileExt(filename, "txt") - let d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) + var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) let rst = parseRst(readFile(filen.string), filen.string, line=LineRstInit, column=ColRstInit, conf, d.sharedState) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 26cefd127885f..1127048251449 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -32,13 +32,13 @@ type # non-fatal errors errIllFormedAstX, errCannotOpenFile, errXExpected, - errGridTableNotImplemented, + errRstGridTableNotImplemented, errMarkdownIllformedTable, - errGeneralParseError, - errNewSectionExpected, - errInvalidDirectiveX, - errInvalidRstField, - errFootnoteMismatch, + errRstNewSectionExpected, + errRstGeneralParseError, + errRstInvalidDirectiveX, + errRstInvalidField, + errRstFootnoteMismatch, errProveInit, # deadcode errGenerated, errUser, @@ -47,11 +47,13 @@ type warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit", warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated", warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", - warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX", - warnBrokenLink = "BrokenLink", - warnLanguageXNotSupported = "LanguageXNotSupported", - warnFieldXNotSupported = "FieldXNotSupported", - warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored", + warnRstRedefinitionOfLabel = "RedefinitionOfLabel", + warnRstUnknownSubstitutionX = "UnknownSubstitutionX", + warnRstBrokenLink = "BrokenLink", + warnRstLanguageXNotSupported = "LanguageXNotSupported", + warnRstFieldXNotSupported = "FieldXNotSupported", + warnRstStyle = "warnRstStyle", + warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam", warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap", warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport", @@ -94,13 +96,13 @@ const errIllFormedAstX: "illformed AST: $1", errCannotOpenFile: "cannot open '$1'", errXExpected: "'$1' expected", - errGridTableNotImplemented: "grid table is not implemented", + errRstGridTableNotImplemented: "grid table is not implemented", errMarkdownIllformedTable: "illformed delimiter row of a markdown table", - errGeneralParseError: "general parse error", - errNewSectionExpected: "new section expected $1", - errInvalidDirectiveX: "invalid directive: '$1'", - errInvalidRstField: "invalid field: $1", - errFootnoteMismatch: "number of footnotes and their references don't match: $1", + errRstNewSectionExpected: "new section expected $1", + errRstGeneralParseError: "general parse error", + errRstInvalidDirectiveX: "invalid directive: '$1'", + errRstInvalidField: "invalid field: $1", + errRstFootnoteMismatch: "number of footnotes and their references don't match: $1", errProveInit: "Cannot prove that '$1' is initialized.", # deadcode errGenerated: "$1", errUser: "$1", @@ -112,11 +114,11 @@ const warnConfigDeprecated: "config file '$1' is deprecated", warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)", warnUnknownMagic: "unknown magic '$1' might crash the compiler", - warnRedefinitionOfLabel: "redefinition of label '$1'", - warnUnknownSubstitutionX: "unknown substitution '$1'", - warnBrokenLink: "broken link '$1'", - warnLanguageXNotSupported: "language '$1' not supported", - warnFieldXNotSupported: "field '$1' not supported", + warnRstRedefinitionOfLabel: "redefinition of label '$1'", + warnRstUnknownSubstitutionX: "unknown substitution '$1'", + warnRstBrokenLink: "broken link '$1'", + warnRstLanguageXNotSupported: "language '$1' not supported", + warnRstFieldXNotSupported: "field '$1' not supported", warnRstStyle: "RST style: $1", warnCommentXIgnored: "comment '$1' ignored", warnTypelessParam: "", # deadcode @@ -198,9 +200,9 @@ const warnMax* = pred(hintSuccess) hintMin* = hintSuccess hintMax* = high(TMsgKind) - rstWarnings* = {warnRedefinitionOfLabel, warnUnknownSubstitutionX, - warnBrokenLink, warnLanguageXNotSupported, - warnFieldXNotSupported, warnRstStyle} + rstWarnings* = {warnRstRedefinitionOfLabel, warnRstUnknownSubstitutionX, + warnRstBrokenLink, warnRstLanguageXNotSupported, + warnRstFieldXNotSupported, warnRstStyle} type TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints diff --git a/compiler/main.nim b/compiler/main.nim index 2f29e6b27eff4..e4ec4d72931a1 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -285,7 +285,7 @@ proc mainCommand*(graph: ModuleGraph) = of cmdDoc: docLikeCmd(): conf.setNoteDefaults(warnLockLevel, false) # issue #13218 - conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218 + conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # issue #13218 # because currently generates lots of false positives due to conflation # of labels links in doc comments, e.g. for random.rand: # ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer @@ -297,7 +297,7 @@ proc mainCommand*(graph: ModuleGraph) = # XXX: why are warnings disabled by default for rst2html and rst2tex? for warn in rstWarnings: conf.setNoteDefaults(warn, true) - conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218 + conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # similar to issue #13218 when defined(leanCompiler): conf.quitOrRaise "compiler wasn't built with documentation generator" else: diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index afab0f04d3ed3..af40b6f627899 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -217,6 +217,7 @@ type mcWarning = "Warning", mcError = "Error" + # keep the order in sync with compiler/docgen.nim and compiler/lineinfos.nim: MsgKind* = enum ## the possible messages meCannotOpenFile = "cannot open '$1'", meExpected = "'$1' expected", @@ -225,7 +226,7 @@ type meNewSectionExpected = "new section expected $1", meGeneralParseError = "general parse error", meInvalidDirective = "invalid directive: '$1'", - meInvalidRstField = "invalid field: $1", + meInvalidField = "invalid field: $1", meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", mwRedefinitionOfLabel = "redefinition of label '$1'", mwUnknownSubstitution = "unknown substitution '$1'", diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 881191b9215d9..c07d520dad635 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -909,7 +909,7 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = template err(msg: string) = - rstMessage(d.filenames, d.msgHandler, n.info, meInvalidRstField, msg) + rstMessage(d.filenames, d.msgHandler, n.info, meInvalidField, msg) let value = n.getFieldValue var number: int let nChars = parseInt(value, number) From 98b4d0deaca5b8aa390b7e894ffb41d92922e615 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 8 Jul 2021 23:46:33 +0300 Subject: [PATCH 13/29] doAssert + minor name change --- compiler/docgen.nim | 2 +- compiler/lineinfos.nim | 4 ++-- lib/packages/docutils/rst.nim | 7 +++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8e26b0e5f5613..8a44e365675c3 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -160,7 +160,7 @@ template declareClosures = of meCannotOpenFile: k = errCannotOpenFile of meExpected: k = errXExpected of meGridTableNotImplemented: k = errRstGridTableNotImplemented - of meMarkdownIllformedTable: k = errMarkdownIllformedTable + of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable of meNewSectionExpected: k = errRstNewSectionExpected of meGeneralParseError: k = errRstGeneralParseError of meInvalidDirective: k = errRstInvalidDirectiveX diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 1127048251449..ad2f90f301616 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -33,7 +33,7 @@ type errIllFormedAstX, errCannotOpenFile, errXExpected, errRstGridTableNotImplemented, - errMarkdownIllformedTable, + errRstMarkdownIllformedTable, errRstNewSectionExpected, errRstGeneralParseError, errRstInvalidDirectiveX, @@ -97,7 +97,7 @@ const errCannotOpenFile: "cannot open '$1'", errXExpected: "'$1' expected", errRstGridTableNotImplemented: "grid table is not implemented", - errMarkdownIllformedTable: "illformed delimiter row of a markdown table", + errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table", errRstNewSectionExpected: "new section expected $1", errRstGeneralParseError: "general parse error", errRstInvalidDirectiveX: "invalid directive: '$1'", diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index af40b6f627899..54a1d490038f3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -592,10 +592,9 @@ proc setCurrFilename(s: PRstSharedState, file1: string) = s.currFileIdx = (s.filenames.len - 1).FileIndex proc getFilename(filenames: seq[string], fid: FileIndex): string = - if fid.int < filenames.len: - result = filenames[fid.int] - else: - result = "input" + doAssert(0 <= fid.int and fid.int < filenames.len, + "incorrect FileIndex $1 (range 0..$2)" % [$fid.int, $(filenames.len-1)]) + result = filenames[fid.int] proc currFilename(s: PRstSharedState): string = getFilename(s.filenames, s.currFileIdx) From 1e633aaf6e0c4b4947816cd655f8bc1ee239a825 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 9 Jul 2021 00:19:46 +0300 Subject: [PATCH 14/29] fix a bug caught by doAssert --- compiler/docgen.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8a44e365675c3..747c4f5cc93da 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1132,6 +1132,9 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = proc finishGenerateDoc*(d: var PDoc) = ## Perform 2nd RST pass for resolution of links/footnotes/headings... + # copy file map `filenames` to ``rstgen.nim`` for its warnings + d.filenames = d.sharedState.filenames + # Main title/subtitle are allowed only in the first RST fragment of document var firstRst = PRstNode(nil) for fragment in d.modDescPre: @@ -1171,9 +1174,6 @@ proc finishGenerateDoc*(d: var PDoc) = d.jEntriesFinal.add entry.json d.jEntriesPre[i].rst = nil - # pass file map `filenames` to ``rstgen.nim`` for its warnings - d.filenames = move(d.sharedState.filenames) - proc add(d: PDoc; j: JsonItem) = if j.json != nil or j.rst != nil: d.jEntriesPre.add j From f7bb08cb1ba5a7a0e91c7f7cd2b38be2d097072f Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 9 Jul 2021 00:21:30 +0300 Subject: [PATCH 15/29] apply strbasics.strip to final HTML/Latex --- lib/packages/docutils/rstgen.nim | 4 +++- tests/stdlib/trst.nim | 8 ++++---- tests/stdlib/trstgen.nim | 15 +++++++-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index c07d520dad635..22c55d31718bf 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils, ../../../compiler/lineinfos + algorithm, parseutils, ../../../compiler/lineinfos, std/strbasics import ../../std/private/since @@ -1572,6 +1572,7 @@ proc rstToHtml*(s: string, options: RstParseOptions, initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, filenames) result = "" renderRstToOut(d, rst, result) + strbasics.strip(result) proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} = @@ -1584,3 +1585,4 @@ proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, var rstGenera: RstGenerator rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", filenames=filenames) rstGenera.renderRstToOut(rst, result) + strbasics.strip(result) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index d792496602926..fc3ccbf4e88c4 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -422,7 +422,7 @@ OtherStart .. include:: other.rst :start-after: OtherStart """ - check "Visible " == rstTohtml(input, {}, defaultConfig()) + check "Visible" == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") test "Include everything before": @@ -436,7 +436,7 @@ And this should **NOT** be visible in `docs.html` .. include:: other.rst :end-before: OtherEnd """ - doAssert "Visible " == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") @@ -454,7 +454,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - check "Visible " == rstTohtml(input, {}, defaultConfig()) + check "Visible" == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") @@ -474,7 +474,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - doAssert "Visible " == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) removeFile("other.rst") suite "RST escaping": diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index a677a28124604..8766da547a238 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -237,8 +237,7 @@ not in table""" D1 """ & id"code" & " " & op"\|" & """D2 E1 | text F2 without pipe -

not in table

-""") +

not in table

""") let input2 = """ | A1 header | A2 | | --- | --- |""" @@ -631,7 +630,7 @@ let x = 1 let p2 = """

Par2 value2.

""" let p3 = """

Par3 """ & id"value3" & ".

" let p4 = """

Par4 value4.

""" - let expected = p1 & p2 & "\n" & p3 & "\n" & p4 & "\n" + let expected = p1 & p2 & "\n" & p3 & "\n" & p4 check(input.toHtml == expected) test "role directive": @@ -690,7 +689,7 @@ Test1 Paragraph2""" let output2 = input2.toHtml - doAssert "Paragraph1


Paragraph2

\n" == output2 + doAssert "Paragraph1


Paragraph2

" == output2 let input3 = dedent""" | xxx @@ -857,7 +856,7 @@ Test1 check("input(6, 1) Warning: RST style: \n" & "not enough indentation on line 6" in warnings8[0]) doAssert output8 == "Paragraph.
    " & - "
  1. stringA
    2. \n
  2. stringB
    3. \n
\n

C. string1 string2

\n" + "
  • stringA
    • \n
  • stringB
    • \n\n

    C. string1 string2

    " test "Markdown enumerated lists": let input1 = dedent """ @@ -930,7 +929,7 @@ Test1 Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_. """ let output2 = input2.toHtml - doAssert output2 == "Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_. " + doAssert output2 == "Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_." # check that auto-symbol footnotes work: let input3 = dedent """ @@ -1171,7 +1170,7 @@ Test1 "input(8, 4) Warning: language 'anotherLang' not supported" ]) check(output == "
    \nanything
    " & - "

    \nsomeCode\n

    \n") + "

    \nsomeCode\n

    ") test "RST admonitions": # check that all admonitions are implemented @@ -1502,7 +1501,7 @@ Test1 check "(3, 15) Warning: " in warnings[1] check "language 'py:class' not supported" in warnings[1] check("""

    See function spam.

    """ & "\n" & - """

    See also egg.

    """ & "\n" == + """

    See also egg.

    """ == output) test "(not) Roles: check escaping 1": From ae2083b5291acf72b6073edbdf7acb04e724fa44 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 11 Jul 2021 00:59:34 +0300 Subject: [PATCH 16/29] rm redundant filename --- compiler/docgen.nim | 2 +- lib/packages/docutils/rst.nim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 747c4f5cc93da..e45ac398cadcf 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -185,7 +185,7 @@ proc parseRst(text, filename: string, line, column: int, conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() - result = rstParsePass1(text, filename, line, column, sharedState) + result = rstParsePass1(text, line, column, sharedState) proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, guessTarget: bool): AbsoluteFile = diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 54a1d490038f3..450e63b5145d3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2874,7 +2874,7 @@ proc parseDotDot(p: var RstParser): PRstNode = else: result = parseComment(p, col) -proc rstParsePass1*(fragment, filename: string, +proc rstParsePass1*(fragment: string, line, column: int, sharedState: PRstSharedState): PRstNode = ## Parses an RST `fragment`. @@ -3021,7 +3021,7 @@ proc rstParse*(text, filename: string, ## argument `filenames` (it is being filled here at least with `filename` ## and possibly with other files from RST ``.. include::`` statement). var sharedState = newRstSharedState(options, filename, findFile, msgHandler) - let unresolved = rstParsePass1(text, filename, line, column, sharedState) + let unresolved = rstParsePass1(text, line, column, sharedState) preparePass2(sharedState, unresolved) result.node = resolveSubs(sharedState, unresolved) result.filenames = sharedState.filenames From e8ede97a753dd149493c867623f3e68261fe5e02 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 11 Jul 2021 15:46:50 +0300 Subject: [PATCH 17/29] fix test after rebase --- tests/stdlib/trstgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 8766da547a238..8676774043429 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1169,7 +1169,7 @@ Test1 "input(4, 11) Warning: language 'unsupportedLang' not supported", "input(8, 4) Warning: language 'anotherLang' not supported" ]) - check(output == "
    \nanything
    " & + check(output == "
    anything
    " & "

    \nsomeCode\n

    ") test "RST admonitions": From f78422251036ceb711d2dabbca463a4c2e1d6017 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 15 Jul 2021 23:13:55 +0300 Subject: [PATCH 18/29] delete `order` from rnFootnoteRef, also display errors/warnings properly when footnote references are from different files --- lib/packages/docutils/rst.nim | 62 ++++++++++++++++++-------------- lib/packages/docutils/rstast.nim | 10 +----- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 450e63b5145d3..13c999a6b84ca 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -503,10 +503,12 @@ type subs: seq[Substitution] # substitutions refs*: seq[Substitution] # references anchors*: seq[AnchorSubst] # internal target substitutions - lineFootnoteNum: seq[int] # footnote line, auto numbers .. [#] - lineFootnoteNumRef: seq[int] # footnote line, their reference [#]_ - lineFootnoteSym: seq[int] # footnote line, auto symbols .. [*] - lineFootnoteSymRef: seq[int] # footnote line, their reference [*]_ + lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#] + lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_ + currFootnoteNumRef: int # ... their counter for `resolveSubs` + lineFootnoteSym: seq[TLineInfo] # footnote line, auto symbols .. [*] + lineFootnoteSymRef: seq[TLineInfo] # footnote line, their reference [*]_ + currFootnoteSymRef: int # ... their counter for `resolveSubs` footnotes: seq[FootnoteSubst] # correspondence b/w footnote label, # number, order of occurrence msgHandler: MsgHandler # How to handle errors. @@ -852,11 +854,18 @@ proc addFootnoteNumManual(p: var RstParser, num: int) = return p.s.footnotes.add((fnManualNumber, num, -1, -1, $num)) +proc lineInfo(p: RstParser, iTok: int): TLineInfo = + result.col = int16(p.col + p.tok[iTok].col) + result.line = uint16(p.line + p.tok[iTok].line) + result.fileIndex = p.s.currFileIdx + +proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) + proc addFootnoteNumAuto(p: var RstParser, label: string) = ## add auto-numbered footnote. ## Empty label [#] means it'll be resolved by the occurrence. if label == "": # simple auto-numbered [#] - p.s.lineFootnoteNum.add curLine(p) + p.s.lineFootnoteNum.add lineInfo(p) p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label)) else: # auto-numbered with label [#label] for fnote in p.s.footnotes: @@ -866,7 +875,7 @@ proc addFootnoteNumAuto(p: var RstParser, label: string) = p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label)) proc addFootnoteSymAuto(p: var RstParser) = - p.s.lineFootnoteSym.add curLine(p) + p.s.lineFootnoteSym.add lineInfo(p) p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) proc orderFootnotes(s: PRstSharedState) = @@ -875,7 +884,15 @@ proc orderFootnotes(s: PRstSharedState) = ## Save the result back to `s.footnotes`. # Report an error if found any mismatch in number of automatic footnotes - proc listFootnotes(lines: seq[int]): string = + proc listFootnotes(locations: seq[TLineInfo]): string = + var lines: seq[string] + for info in locations: + if s.filenames.len > 1: + let file = getFilename(s.filenames, info.fileIndex) + lines.add file & ":" + else: # no need to add file name here if there is only 1 + lines.add "" + lines[^1].add $info.line result.add $lines.len & " (lines " & join(lines, ", ") & ")" if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len: rstMessage(s, meFootnoteMismatch, @@ -1180,13 +1197,6 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind = if result == rnUnknownRole: rstMessage(p, mwUnsupportedLanguage, sym) -proc lineInfo(p: RstParser, iTok: int): TLineInfo = - result.col = int16(p.col + p.tok[iTok].col) - result.line = uint16(p.line + p.tok[iTok].line) - result.fileIndex = p.s.currFileIdx - -proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) - proc toInlineCode(n: PRstNode, language: string): PRstNode = ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). result = newRstNode(rnInlineCode, info=n.info) @@ -1584,17 +1594,14 @@ proc parseInline(p: var RstParser, father: PRstNode) = elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and (n = parseFootnoteName(p, reference=true); n != nil): var nn = newRstNode(rnFootnoteRef) - nn.loc = new PFootnoteRefInfo - nn.loc.info = lineInfo(p, saveIdx+1) + nn.info = lineInfo(p, saveIdx+1) nn.add n let (fnType, _) = getFootnoteType(n) case fnType of fnAutoSymbol: - p.s.lineFootnoteSymRef.add curLine(p) - nn.loc.order = p.s.lineFootnoteSymRef.len + p.s.lineFootnoteSymRef.add lineInfo(p) of fnAutoNumber: - p.s.lineFootnoteNumRef.add curLine(p) - nn.loc.order = p.s.lineFootnoteNumRef.len + p.s.lineFootnoteNumRef.add lineInfo(p) else: discard father.add(nn) else: @@ -2961,21 +2968,22 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = result.add(nn) var refn = fnType.prefix # create new rnFootnoteRef, add final label, and finalize target refn: - result = newRstNode(rnFootnoteRef) - result.loc = n.loc + result = newRstNode(rnFootnoteRef, info = n.info) case fnType of fnManualNumber: addLabel num refn.add $num of fnAutoNumber: - addLabel getFootnoteNum(s, n.loc.order) - refn.add $n.loc.order + inc s.currFootnoteNumRef + addLabel getFootnoteNum(s, s.currFootnoteNumRef) + refn.add $s.currFootnoteNumRef of fnAutoNumberLabel: addLabel getFootnoteNum(s, rstnodeToRefname(n)) refn.add rstnodeToRefname(n) of fnAutoSymbol: - addLabel getAutoSymbol(s, n.loc.order) - refn.add $n.loc.order + inc s.currFootnoteSymRef + addLabel getAutoSymbol(s, s.currFootnoteSymRef) + refn.add $s.currFootnoteSymRef of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) @@ -2983,7 +2991,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = if anch != "": result.add newLeaf(anch) # add link else: - rstMessage(s.filenames, s.msgHandler, n.loc.info, mwBrokenLink, refn) + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) result.add newLeaf(refn) # add link of rnLeaf: discard diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 9a5547956be29..6be217aea04da 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -77,10 +77,6 @@ type PRstNode* = ref RstNode ## an RST node RstNodeSeq* = seq[PRstNode] - PFootnoteRefInfo* = ref object ## location inside a file, only for - ## `rnFootnoteRef` (packed for memory saving) - info*: TLineInfo - order*: int RstNode* {.acyclic, final.} = object ## AST node (result of RST parsing) case kind*: RstNodeKind ## the node's kind of rnLeaf, rnSmiley: @@ -100,11 +96,9 @@ type order*: int ## footnote order (for auto-symbol footnotes and ## auto-numbered ones without a label) of rnRef, rnSubstitutionReferences, - rnInterpretedText, rnField, rnInlineCode, rnCodeBlock: + rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: info*: TLineInfo ## To have line/column info for warnings at ## nodes that are post-processed after parsing - of rnFootnoteRef: - loc*: PFootnoteRefInfo ## almost the same else: discard anchor*: string ## anchor, internal link target @@ -406,8 +400,6 @@ proc renderRstToStr*(node: PRstNode, indent=0): string = result.add " level=" & $node.level of rnFootnote, rnCitation, rnOptionListItem: result.add (if node.order == 0: "" else: " order=" & $node.order) - of rnFootnoteRef: - result.add (if node.loc.order == 0: "" else: " order=" & $node.loc.order) else: discard result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") From b56d6c0d070537a073e37f1a59a96a6af3736cc0 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 00:49:47 +0300 Subject: [PATCH 19/29] Update compiler/lineinfos.nim Co-authored-by: Timothee Cour --- compiler/lineinfos.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index ad2f90f301616..8ab52a452b23d 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -200,9 +200,7 @@ const warnMax* = pred(hintSuccess) hintMin* = hintSuccess hintMax* = high(TMsgKind) - rstWarnings* = {warnRstRedefinitionOfLabel, warnRstUnknownSubstitutionX, - warnRstBrokenLink, warnRstLanguageXNotSupported, - warnRstFieldXNotSupported, warnRstStyle} + rstWarnings* = {warnRstRedefinitionOfLabel..warnRstStyle} type TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints From 22c254d8cbaba3605cac58bda96522aca4061da4 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 00:52:41 +0300 Subject: [PATCH 20/29] Update lib/packages/docutils/rstast.nim Co-authored-by: Timothee Cour --- lib/packages/docutils/rstast.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 6be217aea04da..fd16086178030 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -115,8 +115,7 @@ proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], proc newRstNode*(kind: RstNodeKind, info: TLineInfo, sons: seq[PRstNode] = @[]): PRstNode = - result = PRstNode(kind: kind, sons: sons) - result.info = info + result = PRstNode(kind: kind, sons: sons, info: info) proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = assert kind in {rnLeaf, rnSmiley} From a0a9ae155496cb9568ada2ac29f387c8e76395eb Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 00:52:50 +0300 Subject: [PATCH 21/29] Update lib/packages/docutils/rstast.nim Co-authored-by: Timothee Cour --- lib/packages/docutils/rstast.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index fd16086178030..88a9b47d15734 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -110,8 +110,7 @@ proc len*(n: PRstNode): int = proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], anchor = ""): PRstNode = - result = PRstNode(kind: kind, sons: sons) - result.anchor = anchor + result = PRstNode(kind: kind, sons: sons, anchor: anchor) proc newRstNode*(kind: RstNodeKind, info: TLineInfo, sons: seq[PRstNode] = @[]): PRstNode = From d52ce1f3a64716e5db6c62052e7b2398d1efce27 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 00:54:43 +0300 Subject: [PATCH 22/29] Update lib/packages/docutils/rstast.nim Co-authored-by: Timothee Cour --- lib/packages/docutils/rstast.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 88a9b47d15734..e0175b9840371 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -9,7 +9,7 @@ ## This module implements an AST for the `reStructuredText`:idx: parser. -import strutils, json, ../../../compiler/lineinfos +import strutils, json, "$lib/../compiler/lineinfos" type RstNodeKind* = enum ## the possible node kinds of an PRstNode From 12dde3d70d83bb7a499738fbd41a12d2136c74fe Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 00:57:14 +0300 Subject: [PATCH 23/29] revert because of error: Error: cannot prove that it's safe to initialize 'info' with the runtime value for the discriminator 'kind' --- lib/packages/docutils/rstast.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index e0175b9840371..ab7dbe7ed2398 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -114,7 +114,8 @@ proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], proc newRstNode*(kind: RstNodeKind, info: TLineInfo, sons: seq[PRstNode] = @[]): PRstNode = - result = PRstNode(kind: kind, sons: sons, info: info) + result = PRstNode(kind: kind, sons: sons) + result.info = info proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = assert kind in {rnLeaf, rnSmiley} From 769d36776b90859298e037e67716c7af5d3889c2 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 00:59:51 +0300 Subject: [PATCH 24/29] Update lib/packages/docutils/rstgen.nim Co-authored-by: Timothee Cour --- lib/packages/docutils/rstgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 22c55d31718bf..cd1b83523c763 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils, ../../../compiler/lineinfos, std/strbasics + algorithm, parseutils, "$lib/../compiler/lineinfos", std/strbasics import ../../std/private/since From 4c5dc1c7ef190d7821b2295b7aabb602ddf6fb7a Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 01:00:33 +0300 Subject: [PATCH 25/29] apply suggestion --- lib/packages/docutils/rst.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 13c999a6b84ca..88b514933f08a 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -196,7 +196,7 @@ import os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, ../../../compiler/lineinfos + std/private/miscdollars, "$lib/../compiler/lineinfos" from highlite import SourceLanguage, getSourceLanguage type From 209c72ec4a067eb5609b3bd02692f0bf7c31e922 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 16 Jul 2021 01:01:13 +0300 Subject: [PATCH 26/29] Update lib/packages/docutils/rst.nim Co-authored-by: Timothee Cour --- lib/packages/docutils/rst.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 88b514933f08a..6e3fc2411c390 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -631,7 +631,7 @@ proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = proc rstMessage*(filenames: seq[string], f: MsgHandler, info: TLineInfo, msgKind: MsgKind, arg: string) = - ## Print warnings using `info: TLineInfo`, i.e. in 2nd-pass warnings for + ## Print warnings using `info`, i.e. in 2nd-pass warnings for ## footnotes/substitutions/references or from ``rstgen.nim``. let file = getFilename(filenames, info.fileIndex) f(file, info.line.int, info.col.int, msgKind, arg) From 3d5f9031a84b63e20245e7b5fb3448bc187d7299 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 17 Jul 2021 13:57:24 +0300 Subject: [PATCH 27/29] add Table for string->file name mapping --- lib/packages/docutils/rst.nim | 35 ++++++++++++++++++-------------- lib/packages/docutils/rstgen.nim | 4 ++-- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 6e3fc2411c390..d14b0ffa7d52e 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -196,7 +196,7 @@ import os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, "$lib/../compiler/lineinfos" + std/private/miscdollars, "$lib/../compiler/lineinfos", tables from highlite import SourceLanguage, getSourceLanguage type @@ -491,7 +491,9 @@ type autoNumIdx: int # order of occurence: fnAutoNumber, fnAutoNumberLabel autoSymIdx: int # order of occurence: fnAutoSymbol label: string # valid for fnAutoNumberLabel - + RstFileTable* = object + filenameToIdx*: Table[string, FileIndex] + idxToFilename*: seq[string] RstSharedState = object options: RstParseOptions # parsing options hLevels: LevelMap # hierarchy of heading styles @@ -513,7 +515,7 @@ type # number, order of occurrence msgHandler: MsgHandler # How to handle errors. findFile: FindFileHandler # How to find files. - filenames*: seq[string] # map FileIndex -> file name (for storing + filenames*: RstFileTable # map file name <-> FileIndex (for storing # file names for warnings after 1st stage) currFileIdx: FileIndex # current index in `filesnames` hasToc*: bool @@ -585,18 +587,21 @@ proc whichRoleAux(sym: string): RstNodeKind = else: # unknown role result = rnUnknownRole -proc setCurrFilename(s: PRstSharedState, file1: string) = - for i, file2 in s.filenames: - if file1 == file2: - s.currFileIdx = i.FileIndex - return - s.filenames.add file1 - s.currFileIdx = (s.filenames.len - 1).FileIndex +proc len(filenames: RstFileTable): int = filenames.idxToFilename.len -proc getFilename(filenames: seq[string], fid: FileIndex): string = +proc setCurrFilename(s: PRstSharedState, file1: string) = + let nextIdx = s.filenames.len.FileIndex + let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) + if v == nextIdx: + s.filenames.filenameToIdx[file1] = v + s.filenames.idxToFilename.add file1 + s.currFileIdx = v + +proc getFilename(filenames: RstFileTable, fid: FileIndex): string = doAssert(0 <= fid.int and fid.int < filenames.len, - "incorrect FileIndex $1 (range 0..$2)" % [$fid.int, $(filenames.len-1)]) - result = filenames[fid.int] + "incorrect FileIndex $1 (range 0..$2)" % [ + $fid.int, $(filenames.len - 1)]) + result = filenames.idxToFilename[fid.int] proc currFilename(s: PRstSharedState): string = getFilename(s.filenames, s.currFileIdx) @@ -629,7 +634,7 @@ proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) -proc rstMessage*(filenames: seq[string], f: MsgHandler, +proc rstMessage*(filenames: RstFileTable, f: MsgHandler, info: TLineInfo, msgKind: MsgKind, arg: string) = ## Print warnings using `info`, i.e. in 2nd-pass warnings for ## footnotes/substitutions/references or from ``rstgen.nim``. @@ -3023,7 +3028,7 @@ proc rstParse*(text, filename: string, options: RstParseOptions, findFile: FindFileHandler = nil, msgHandler: MsgHandler = nil): - tuple[node: PRstNode, filenames: seq[string], hasToc: bool] = + tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] = ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, ## note that 2nd tuple element should be fed to `initRstGenerator` ## argument `filenames` (it is being filled here at least with `filename` diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index cd1b83523c763..ceb0ca5ba8e3c 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -76,7 +76,7 @@ type msgHandler*: MsgHandler outDir*: string ## output directory, initialized by docgen.nim destFile*: string ## output (HTML) file, initialized by docgen.nim - filenames*: seq[string] + filenames*: RstFileTable filename*: string ## source Nim or Rst file meta*: array[MetaEnum, string] currentSection: string ## \ @@ -114,7 +114,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, config: StringTableRef, filename: string, findFile: FindFileHandler = nil, msgHandler: MsgHandler = nil, - filenames: seq[string] = @[]) = + filenames = default(RstFileTable)) = ## Initializes a ``RstGenerator``. ## ## You need to call this before using a ``RstGenerator`` with any other From cc0df0baf10dab0aac3dae09c4822bd48be392f0 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 19 Jul 2021 17:01:35 +0200 Subject: [PATCH 28/29] do not import compiler/lineinfos --- lib/packages/docutils/rst.nim | 2 +- lib/packages/docutils/rstast.nim | 9 ++++++++- lib/packages/docutils/rstgen.nim | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index d14b0ffa7d52e..abea026dd85f3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -196,7 +196,7 @@ import os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, "$lib/../compiler/lineinfos", tables + std/private/miscdollars, tables from highlite import SourceLanguage, getSourceLanguage type diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index ab7dbe7ed2398..fa0620f44c74f 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -9,7 +9,7 @@ ## This module implements an AST for the `reStructuredText`:idx: parser. -import strutils, json, "$lib/../compiler/lineinfos" +import strutils, json type RstNodeKind* = enum ## the possible node kinds of an PRstNode @@ -74,6 +74,11 @@ type rnLeaf # a leaf; the node's text field contains the # leaf val + FileIndex* = distinct int32 + TLineInfo* = object + line*: uint16 + col*: int16 + fileIndex*: FileIndex PRstNode* = ref RstNode ## an RST node RstNodeSeq* = seq[PRstNode] @@ -105,6 +110,8 @@ type ## (aka HTML id tag, aka Latex label/hypertarget) sons*: RstNodeSeq ## the node's sons +proc `==`*(a, b: FileIndex): bool {.borrow.} + proc len*(n: PRstNode): int = result = len(n.sons) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index ceb0ca5ba8e3c..10a2294799092 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils, "$lib/../compiler/lineinfos", std/strbasics + algorithm, parseutils, std/strbasics import ../../std/private/since From 9c709be352d3fb245de0e02cc8b36d11775de720 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 19 Jul 2021 17:18:05 +0200 Subject: [PATCH 29/29] fix ambiguous calls --- compiler/docgen.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index e45ac398cadcf..ba078b95ee2f6 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -12,11 +12,12 @@ import ast, strutils, strtabs, options, msgs, os, idents, - wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast, + wordrecg, syntaxes, renderer, lexer, packages/docutils/rst, packages/docutils/rstgen, json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, pathutils, tables, nimpaths, renderverbatim, osproc +import packages/docutils/rstast except FileIndex, TLineInfo from uri import encodeUrl from std/private/globs import nativeToUnixPath