From ca503c91138a4a401902c053fd0bfb9bc0740ff3 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 23 Mar 2021 11:11:37 +0100 Subject: [PATCH 01/11] [wip] add fuse-based search --- pdoc/__init__.py | 9 ++- pdoc/render.py | 47 ++++++++++++- pdoc/render_helpers.py | 4 +- pdoc/templates/default/module.html.jinja2 | 80 ++++++++++++++++++++++- pdoc/templates/fuse.basic.min.js | 9 +++ test/test_render_helpers.py | 5 +- 6 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 pdoc/templates/fuse.basic.min.js diff --git a/pdoc/__init__.py b/pdoc/__init__.py index 71616e70..1f115b84 100644 --- a/pdoc/__init__.py +++ b/pdoc/__init__.py @@ -381,7 +381,7 @@ def write(mod: doc.Module): def write(mod: doc.Module): retval.write(r(mod)) - all_modules = extract.parse_specs(modules) + all_modules: dict[str, Optional[doc.Module]] = extract.parse_specs(modules) if format == "html": @@ -405,7 +405,8 @@ def r(mod: doc.Module) -> str: f"Error importing {mod}:\n{traceback.format_exc()}", RuntimeWarning ) else: - write(doc.Module(m)) + all_modules[mod] = doc.Module(m) + write(all_modules[mod]) if not output_directory: return retval.getvalue() @@ -417,4 +418,8 @@ def r(mod: doc.Module) -> str: if index: (output_directory / "index.html").write_bytes(index.encode()) + search = render.search_index(all_modules) + if search: + (output_directory / "search.json").write_bytes(search.encode()) + return retval.getvalue() diff --git a/pdoc/render.py b/pdoc/render.py index 3fdc727c..250347de 100644 --- a/pdoc/render.py +++ b/pdoc/render.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os from pathlib import Path from typing import Collection, Mapping, Optional @@ -16,7 +17,7 @@ link, linkify, minify_css, - render_docstring, + render_docstring, render_markdown, ) @@ -98,7 +99,48 @@ def html_error(error: str, details: str = "") -> str: ) -@defuse_unsafe_reprs() +def search_index(all_modules: dict[str, Optional[pdoc.doc.Module]]) -> str: + """Renders the search index.""" + items = [] + with defuse_unsafe_reprs(): + for modname, mod in all_modules.items(): + + def make_item(doc: pdoc.doc.Doc, **kwargs) -> dict[str, str]: + # noinspection PyTypeChecker + return { + "modulename": doc.modulename, + "qualname": doc.qualname, + "type": doc.type, + "doc": render_docstring( + {"module": mod, "docformat": env.globals["docformat"]}, # type: ignore + doc.docstring + ), + **kwargs + } + + def make_index(mod: pdoc.doc.Namespace): + yield make_item(mod) + for m in mod.own_members: + if isinstance(m, pdoc.doc.Variable): + yield make_item( + m, + default_value=m.default_value_str, + annotation=m.annotation_str, + ) + elif isinstance(m, pdoc.doc.Function): + yield make_item( + m, + signature=str(m.signature), + ) + elif isinstance(m, pdoc.doc.Class): + yield from make_index(m) + else: + pass + + items.extend(make_index(mod)) + return json.dumps(items) + + def repr_module(module: pdoc.doc.Module) -> str: """Renders `repr(pdoc.doc.Module)`, primarily used for tests and debugging.""" with defuse_unsafe_reprs(): @@ -123,6 +165,7 @@ def repr_module(module: pdoc.doc.Module) -> str: Examples can be found in this module's source code. """ env.filters["render_docstring"] = render_docstring +env.filters["render_markdown"] = render_markdown env.filters["highlight"] = highlight env.filters["linkify"] = linkify env.filters["link"] = link diff --git a/pdoc/render_helpers.py b/pdoc/render_helpers.py index 8c6090cc..1e930f28 100644 --- a/pdoc/render_helpers.py +++ b/pdoc/render_helpers.py @@ -52,7 +52,7 @@ def highlight(code: str) -> str: @cache -def _markdown(docstring: str) -> str: +def render_markdown(docstring: str) -> str: """ Convert `docstring` from Markdown to HTML. """ @@ -70,7 +70,7 @@ def render_docstring(context: Context, docstring: str) -> str: module: pdoc.doc.Module = context["module"] docformat = getattr(module.obj, "__docformat__", context["docformat"]) or "" docstring = docstrings.convert(docstring, docformat, module.source_file) - return _markdown(docstring) + return docstring def split_identifier(all_modules: Container[str], fullname: str) -> tuple[str, str]: diff --git a/pdoc/templates/default/module.html.jinja2 b/pdoc/templates/default/module.html.jinja2 index bd7d911e..a64c4bf8 100644 --- a/pdoc/templates/default/module.html.jinja2 +++ b/pdoc/templates/default/module.html.jinja2 @@ -282,6 +282,10 @@ padding: 0; } + .pdoc input[type=search] { + outline-offset: 0; + } + /* Page Heading */ .pdoc .modulename { margin-top: 0; @@ -567,7 +571,7 @@ {% enddefaultmacro %} {% defaultmacro docstring(var) %} {% if var.docstring %} -
{{ var.docstring | render_docstring | linkify(namespace=var.qualname) }}
+
{{ var.docstring | render_docstring | render_markdown | linkify(namespace=var.qualname) }}
{% endif %} {% enddefaultmacro %} {% defaultmacro nav_members(members) %} @@ -676,9 +680,13 @@ {% endif %} {% endblock %} + {% block search %} + + {% endblock %} + {% block nav_title %}{% endblock %} - {% set index = module.docstring | render_docstring | attr("toc_html") %} + {% set index = module.docstring | render_docstring | render_markdown | attr("toc_html") %} {% if index %}

Contents

{{ index | safe }} @@ -762,4 +770,72 @@ }, 1000); {% endif %} + {% block search_js %} + + + {% endblock %} {% endblock %} diff --git a/pdoc/templates/fuse.basic.min.js b/pdoc/templates/fuse.basic.min.js new file mode 100644 index 00000000..3e303ef3 --- /dev/null +++ b/pdoc/templates/fuse.basic.min.js @@ -0,0 +1,9 @@ +/** + * Fuse.js v6.4.6 - Lightweight fuzzy-search (http://fusejs.io) + * + * Copyright (c) 2021 Kiro Risk (http://kiro.me) + * All Rights Reserved. Apache Software License 2.0 + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +var e,t;e=this,t=function(){"use strict";function e(t){return(e="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e})(t)}function t(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function n(e,t){for(var n=0;ne.length)&&(t=e.length);for(var n=0,r=new Array(t);n0&&void 0!==arguments[0]?arguments[0]:3,t=new Map,n=Math.pow(10,e);return{get:function(e){var r=e.match(w).length;if(t.has(r))return t.get(r);var i=1/Math.sqrt(r),o=parseFloat(Math.round(i*n)/n);return t.set(r,o),o},clear:function(){t.clear()}}}var _=function(){function e(){var n=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},r=n.getFn,i=void 0===r?x.getFn:r;t(this,e),this.norm=L(3),this.getFn=i,this.isCreated=!1,this.setIndexRecords()}return r(e,[{key:"setSources",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];this.docs=e}},{key:"setIndexRecords",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];this.records=e}},{key:"setKeys",value:function(){var e=this,t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];this.keys=t,this._keysMap={},t.forEach((function(t,n){e._keysMap[t.id]=n}))}},{key:"create",value:function(){var e=this;!this.isCreated&&this.docs.length&&(this.isCreated=!0,u(this.docs[0])?this.docs.forEach((function(t,n){e._addString(t,n)})):this.docs.forEach((function(t,n){e._addObject(t,n)})),this.norm.clear())}},{key:"add",value:function(e){var t=this.size();u(e)?this._addString(e,t):this._addObject(e,t)}},{key:"removeAt",value:function(e){this.records.splice(e,1);for(var t=e,n=this.size();t2&&void 0!==arguments[2]?arguments[2]:{},r=n.getFn,i=void 0===r?x.getFn:r,o=new _({getFn:i});return o.setKeys(e.map(b)),o.setSources(t),o.create(),o}function O(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=t.errors,r=void 0===n?0:n,i=t.currentLocation,o=void 0===i?0:i,a=t.expectedLocation,c=void 0===a?0:a,s=t.distance,h=void 0===s?x.distance:s,u=t.ignoreLocation,l=void 0===u?x.ignoreLocation:u,d=r/e.length;if(l)return d;var f=Math.abs(c-o);return h?d+f/h:f?1:d}function A(){for(var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[],t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:x.minMatchCharLength,n=[],r=-1,i=-1,o=0,a=e.length;o=t&&n.push([r,i]),r=-1)}return e[o-1]&&o-r>=t&&n.push([r,o-1]),n}function j(e){for(var t={},n=0,r=e.length;n1&&void 0!==arguments[1]?arguments[1]:{},o=i.location,a=void 0===o?x.location:o,c=i.threshold,s=void 0===c?x.threshold:c,h=i.distance,u=void 0===h?x.distance:h,l=i.includeMatches,d=void 0===l?x.includeMatches:l,f=i.findAllMatches,v=void 0===f?x.findAllMatches:f,y=i.minMatchCharLength,g=void 0===y?x.minMatchCharLength:y,p=i.isCaseSensitive,m=void 0===p?x.isCaseSensitive:p,b=i.ignoreLocation,k=void 0===b?x.ignoreLocation:b;if(t(this,e),this.options={location:a,threshold:s,distance:u,includeMatches:d,findAllMatches:v,minMatchCharLength:g,isCaseSensitive:m,ignoreLocation:k},this.pattern=m?n:n.toLowerCase(),this.chunks=[],this.pattern.length){var M=function(e,t){r.chunks.push({pattern:e,alphabet:j(e),startIndex:t})},w=this.pattern.length;if(w>32){for(var L=0,_=w%32,S=w-_;L3&&void 0!==arguments[3]?arguments[3]:{},i=r.location,o=void 0===i?x.location:i,a=r.distance,c=void 0===a?x.distance:a,s=r.threshold,h=void 0===s?x.threshold:s,u=r.findAllMatches,l=void 0===u?x.findAllMatches:u,d=r.minMatchCharLength,f=void 0===d?x.minMatchCharLength:d,v=r.includeMatches,y=void 0===v?x.includeMatches:v,p=r.ignoreLocation,m=void 0===p?x.ignoreLocation:p;if(t.length>32)throw new Error(g(32));for(var b,k=t.length,M=e.length,w=Math.max(0,Math.min(o,M)),L=h,_=w,S=f>1||y,j=S?Array(M):[];(b=e.indexOf(t,_))>-1;){var E=O(t,{currentLocation:b,expectedLocation:w,distance:c,ignoreLocation:m});if(L=Math.min(E,L),_=b+k,S)for(var I=0;I=J;U-=1){var q=U-1,B=n[e.charAt(q)];if(S&&(j[q]=+!!B),T[U]=(T[U+1]<<1|1)&B,$&&(T[U]|=(C[U+1]|C[U])<<1|1|C[U+1]),T[U]&N&&(F=O(t,{errors:$,currentLocation:q,expectedLocation:w,distance:c,ignoreLocation:m}))<=L){if(L=F,(_=q)<=w)break;J=Math.max(1,2*w-_)}}var V=O(t,{errors:$+1,currentLocation:w,expectedLocation:w,distance:c,ignoreLocation:m});if(V>L)break;C=T}var G={isMatch:_>=0,score:Math.max(.001,F)};if(S){var H=A(j,f);H.length?y&&(G.indices=H):G.isMatch=!1}return G}(e,n,i,{location:a+o,distance:s,threshold:h,findAllMatches:u,minMatchCharLength:l,includeMatches:r,ignoreLocation:d}),m=p.isMatch,b=p.score,k=p.indices;m&&(y=!0),v+=b,m&&k&&(f=[].concat(c(f),c(k)))}));var p={isMatch:y,score:y?v/this.chunks.length:1};return y&&r&&(p.indices=f),p}}]),e}(),I=[];function C(e,t){for(var n=0,r=I.length;n-1&&(n.refIndex=e.idx),t.matches.push(n)}}))}function N(e,t){t.score=e.score}function $(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},r=n.includeMatches,i=void 0===r?x.includeMatches:r,o=n.includeScore,a=void 0===o?x.includeScore:o,c=[];return i&&c.push(P),a&&c.push(N),e.map((function(e){var n=e.idx,r={item:t[n],refIndex:n};return c.length&&c.forEach((function(t){t(e,r)})),r}))}var D=function(){function e(n){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},i=arguments.length>2?arguments[2]:void 0;if(t(this,e),this.options=a({},x,{},r),this.options.useExtendedSearch)throw new Error("Extended search is not available");this._keyStore=new m(this.options.keys),this.setCollection(n,i)}return r(e,[{key:"setCollection",value:function(e,t){if(this._docs=e,t&&!(t instanceof _))throw new Error("Incorrect 'index' type");this._myIndex=t||S(this.options.keys,this._docs,{getFn:this.options.getFn})}},{key:"add",value:function(e){f(e)&&(this._docs.push(e),this._myIndex.add(e))}},{key:"remove",value:function(){for(var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(){return!1},t=[],n=0,r=this._docs.length;n1&&void 0!==arguments[1]?arguments[1]:{},n=t.limit,r=void 0===n?-1:n,i=this.options,o=i.includeMatches,a=i.includeScore,c=i.shouldSort,s=i.sortFn,h=i.ignoreFieldNorm,d=u(e)?u(this._docs[0])?this._searchStringList(e):this._searchObjectList(e):this._searchLogical(e);return F(d,{ignoreFieldNorm:h}),c&&d.sort(s),l(r)&&r>-1&&(d=d.slice(0,r)),$(d,this._docs,{includeMatches:o,includeScore:a})}},{key:"_searchStringList",value:function(e){var t=C(e,this.options),n=this._myIndex.records,r=[];return n.forEach((function(e){var n=e.v,i=e.i,o=e.n;if(f(n)){var a=t.searchIn(n),c=a.isMatch,s=a.score,h=a.indices;c&&r.push({item:n,idx:i,matches:[{score:s,value:n,norm:o,indices:h}]})}})),r}},{key:"_searchLogical",value:function(e){throw new Error("Logical search is not available")}},{key:"_searchObjectList",value:function(e){var t=this,n=C(e,this.options),r=this._myIndex,i=r.keys,o=r.records,a=[];return o.forEach((function(e){var r=e.$,o=e.i;if(f(r)){var s=[];i.forEach((function(e,i){s.push.apply(s,c(t._findMatches({key:e,value:r[i],searcher:n})))})),s.length&&a.push({idx:o,item:r,matches:s})}})),a}},{key:"_findMatches",value:function(e){var t=e.key,n=e.value,r=e.searcher;if(!f(n))return[];var i=[];if(h(n))n.forEach((function(e){var n=e.v,o=e.i,a=e.n;if(f(n)){var c=r.searchIn(n),s=c.isMatch,h=c.score,u=c.indices;s&&i.push({score:h,key:t,value:n,idx:o,norm:a,indices:u})}}));else{var o=n.v,a=n.n,c=r.searchIn(o),s=c.isMatch,u=c.score,l=c.indices;s&&i.push({score:u,key:t,value:o,norm:a,indices:l})}return i}}]),e}();return D.version="6.4.6",D.createIndex=S,D.parseIndex=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=t.getFn,r=void 0===n?x.getFn:n,i=e.keys,o=e.records,a=new _({getFn:r});return a.setKeys(i),a.setIndexRecords(o),a},D.config=x,D},"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):(e=e||self).Fuse=t(); diff --git a/test/test_render_helpers.py b/test/test_render_helpers.py index 21314e1b..543e400c 100644 --- a/test/test_render_helpers.py +++ b/test/test_render_helpers.py @@ -6,7 +6,8 @@ import pytest -from pdoc.render_helpers import (edit_url, qualname_candidates, relative_link, render_docstring, split_identifier) +from pdoc.render_helpers import (edit_url, qualname_candidates, relative_link, render_docstring, render_markdown, + split_identifier) @pytest.mark.parametrize( @@ -74,4 +75,4 @@ def test_markdown_toc(): It's easy to introduce a `.strip()` in there and this gets washed away, so let's test that it works properly. """ - assert render_docstring(mock.MagicMock(), "#foo\n#bar").toc_html # type: ignore + assert render_markdown(mock.MagicMock(), "#foo\n#bar").toc_html # type: ignore From e4002e848ed729d3c05cc36b576044fdddef0585 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 7 May 2021 17:20:31 +0200 Subject: [PATCH 02/11] remove render_markdown again --- pdoc/render.py | 11 ++++------- pdoc/render_helpers.py | 4 ++-- pdoc/templates/default/module.html.jinja2 | 4 ++-- test/test_render_helpers.py | 4 ++-- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/pdoc/render.py b/pdoc/render.py index 250347de..e6683250 100644 --- a/pdoc/render.py +++ b/pdoc/render.py @@ -7,6 +7,7 @@ from jinja2 import Environment, FileSystemLoader +import pdoc.docstrings import pdoc.doc from pdoc._compat import Literal from pdoc.render_helpers import ( @@ -17,7 +18,7 @@ link, linkify, minify_css, - render_docstring, render_markdown, + render_docstring, ) @@ -106,15 +107,12 @@ def search_index(all_modules: dict[str, Optional[pdoc.doc.Module]]) -> str: for modname, mod in all_modules.items(): def make_item(doc: pdoc.doc.Doc, **kwargs) -> dict[str, str]: - # noinspection PyTypeChecker + docformat = getattr(mod.obj, "__docformat__", env.globals["docformat"]) or "" return { "modulename": doc.modulename, "qualname": doc.qualname, "type": doc.type, - "doc": render_docstring( - {"module": mod, "docformat": env.globals["docformat"]}, # type: ignore - doc.docstring - ), + "doc": pdoc.docstrings.convert(doc.docstring, docformat, mod.source_file), **kwargs } @@ -165,7 +163,6 @@ def repr_module(module: pdoc.doc.Module) -> str: Examples can be found in this module's source code. """ env.filters["render_docstring"] = render_docstring -env.filters["render_markdown"] = render_markdown env.filters["highlight"] = highlight env.filters["linkify"] = linkify env.filters["link"] = link diff --git a/pdoc/render_helpers.py b/pdoc/render_helpers.py index 1e930f28..8c6090cc 100644 --- a/pdoc/render_helpers.py +++ b/pdoc/render_helpers.py @@ -52,7 +52,7 @@ def highlight(code: str) -> str: @cache -def render_markdown(docstring: str) -> str: +def _markdown(docstring: str) -> str: """ Convert `docstring` from Markdown to HTML. """ @@ -70,7 +70,7 @@ def render_docstring(context: Context, docstring: str) -> str: module: pdoc.doc.Module = context["module"] docformat = getattr(module.obj, "__docformat__", context["docformat"]) or "" docstring = docstrings.convert(docstring, docformat, module.source_file) - return docstring + return _markdown(docstring) def split_identifier(all_modules: Container[str], fullname: str) -> tuple[str, str]: diff --git a/pdoc/templates/default/module.html.jinja2 b/pdoc/templates/default/module.html.jinja2 index a64c4bf8..5c0f4c06 100644 --- a/pdoc/templates/default/module.html.jinja2 +++ b/pdoc/templates/default/module.html.jinja2 @@ -571,7 +571,7 @@ {% enddefaultmacro %} {% defaultmacro docstring(var) %} {% if var.docstring %} -
{{ var.docstring | render_docstring | render_markdown | linkify(namespace=var.qualname) }}
+
{{ var.docstring | render_docstring | linkify(namespace=var.qualname) }}
{% endif %} {% enddefaultmacro %} {% defaultmacro nav_members(members) %} @@ -686,7 +686,7 @@ {% block nav_title %}{% endblock %} - {% set index = module.docstring | render_docstring | render_markdown | attr("toc_html") %} + {% set index = module.docstring | render_docstring | attr("toc_html") %} {% if index %}

Contents

{{ index | safe }} diff --git a/test/test_render_helpers.py b/test/test_render_helpers.py index 543e400c..f9540486 100644 --- a/test/test_render_helpers.py +++ b/test/test_render_helpers.py @@ -6,7 +6,7 @@ import pytest -from pdoc.render_helpers import (edit_url, qualname_candidates, relative_link, render_docstring, render_markdown, +from pdoc.render_helpers import (edit_url, qualname_candidates, relative_link, render_docstring, split_identifier) @@ -75,4 +75,4 @@ def test_markdown_toc(): It's easy to introduce a `.strip()` in there and this gets washed away, so let's test that it works properly. """ - assert render_markdown(mock.MagicMock(), "#foo\n#bar").toc_html # type: ignore + assert render_docstring(mock.MagicMock(), "#foo\n#bar").toc_html # type: ignore From eb2b7dc1af56ede0c988ef15302de99a870a2de0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 7 May 2021 21:35:30 +0200 Subject: [PATCH 03/11] add search prototype based on elasticlunr --- pdoc/render.py | 23 ++- pdoc/templates/default/module.html.jinja2 | 178 +++++++++++++++------- pdoc/templates/elasticlunr.min.js | 10 ++ pdoc/templates/fuse.basic.min.js | 9 -- 4 files changed, 148 insertions(+), 72 deletions(-) create mode 100644 pdoc/templates/elasticlunr.min.js delete mode 100644 pdoc/templates/fuse.basic.min.js diff --git a/pdoc/render.py b/pdoc/render.py index e6683250..f61fa046 100644 --- a/pdoc/render.py +++ b/pdoc/render.py @@ -102,17 +102,19 @@ def html_error(error: str, details: str = "") -> str: def search_index(all_modules: dict[str, Optional[pdoc.doc.Module]]) -> str: """Renders the search index.""" - items = [] + search_index = {} with defuse_unsafe_reprs(): for modname, mod in all_modules.items(): + docformat = getattr(mod.obj, "__docformat__", env.globals["docformat"]) or "" def make_item(doc: pdoc.doc.Doc, **kwargs) -> dict[str, str]: - docformat = getattr(mod.obj, "__docformat__", env.globals["docformat"]) or "" + docstr = pdoc.docstrings.convert(doc.docstring, docformat, mod.source_file) + if docstr: + docstr = _markdown(docstr) return { - "modulename": doc.modulename, "qualname": doc.qualname, "type": doc.type, - "doc": pdoc.docstrings.convert(doc.docstring, docformat, mod.source_file), + "doc": docstr, **kwargs } @@ -120,23 +122,20 @@ def make_index(mod: pdoc.doc.Namespace): yield make_item(mod) for m in mod.own_members: if isinstance(m, pdoc.doc.Variable): - yield make_item( - m, - default_value=m.default_value_str, - annotation=m.annotation_str, - ) + yield make_item(m) elif isinstance(m, pdoc.doc.Function): yield make_item( m, - signature=str(m.signature), + parameters=list(m.signature.parameters), + funcdef=m.funcdef, ) elif isinstance(m, pdoc.doc.Class): yield from make_index(m) else: pass - items.extend(make_index(mod)) - return json.dumps(items) + search_index[modname] = list(make_index(mod)) + return json.dumps(search_index) def repr_module(module: pdoc.doc.Module) -> str: diff --git a/pdoc/templates/default/module.html.jinja2 b/pdoc/templates/default/module.html.jinja2 index 5c0f4c06..278c737a 100644 --- a/pdoc/templates/default/module.html.jinja2 +++ b/pdoc/templates/default/module.html.jinja2 @@ -771,71 +771,147 @@ {% endif %} {% block search_js %} - + {% endblock %} {% endblock %} diff --git a/pdoc/templates/elasticlunr.min.js b/pdoc/templates/elasticlunr.min.js new file mode 100644 index 00000000..06cc9b32 --- /dev/null +++ b/pdoc/templates/elasticlunr.min.js @@ -0,0 +1,10 @@ +/** + * elasticlunr - http://weixsong.github.io + * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.9.5 + * + * Copyright (C) 2017 Oliver Nightingale + * Copyright (C) 2017 Wei Song + * MIT Licensed + * @license + */ +!function(){function e(e){if(null===e||"object"!=typeof e)return e;var t=e.constructor();for(var n in e)e.hasOwnProperty(n)&&(t[n]=e[n]);return t}var t=function(e){var n=new t.Index;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),e&&e.call(n,n),n};t.version="0.9.5",lunr=t,t.utils={},t.utils.warn=function(e){return function(t){e.console&&console.warn&&console.warn(t)}}(this),t.utils.toString=function(e){return void 0===e||null===e?"":e.toString()},t.EventEmitter=function(){this.events={}},t.EventEmitter.prototype.addListener=function(){var e=Array.prototype.slice.call(arguments),t=e.pop(),n=e;if("function"!=typeof t)throw new TypeError("last argument must be a function");n.forEach(function(e){this.hasHandler(e)||(this.events[e]=[]),this.events[e].push(t)},this)},t.EventEmitter.prototype.removeListener=function(e,t){if(this.hasHandler(e)){var n=this.events[e].indexOf(t);-1!==n&&(this.events[e].splice(n,1),0==this.events[e].length&&delete this.events[e])}},t.EventEmitter.prototype.emit=function(e){if(this.hasHandler(e)){var t=Array.prototype.slice.call(arguments,1);this.events[e].forEach(function(e){e.apply(void 0,t)},this)}},t.EventEmitter.prototype.hasHandler=function(e){return e in this.events},t.tokenizer=function(e){if(!arguments.length||null===e||void 0===e)return[];if(Array.isArray(e)){var n=e.filter(function(e){return null===e||void 0===e?!1:!0});n=n.map(function(e){return t.utils.toString(e).toLowerCase()});var i=[];return n.forEach(function(e){var n=e.split(t.tokenizer.seperator);i=i.concat(n)},this),i}return e.toString().trim().toLowerCase().split(t.tokenizer.seperator)},t.tokenizer.defaultSeperator=/[\s\-]+/,t.tokenizer.seperator=t.tokenizer.defaultSeperator,t.tokenizer.setSeperator=function(e){null!==e&&void 0!==e&&"object"==typeof e&&(t.tokenizer.seperator=e)},t.tokenizer.resetSeperator=function(){t.tokenizer.seperator=t.tokenizer.defaultSeperator},t.tokenizer.getSeperator=function(){return t.tokenizer.seperator},t.Pipeline=function(){this._queue=[]},t.Pipeline.registeredFunctions={},t.Pipeline.registerFunction=function(e,n){n in t.Pipeline.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+n),e.label=n,t.Pipeline.registeredFunctions[n]=e},t.Pipeline.getRegisteredFunction=function(e){return e in t.Pipeline.registeredFunctions!=!0?null:t.Pipeline.registeredFunctions[e]},t.Pipeline.warnIfFunctionNotRegistered=function(e){var n=e.label&&e.label in this.registeredFunctions;n||t.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},t.Pipeline.load=function(e){var n=new t.Pipeline;return e.forEach(function(e){var i=t.Pipeline.getRegisteredFunction(e);if(!i)throw new Error("Cannot load un-registered function: "+e);n.add(i)}),n},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(e){t.Pipeline.warnIfFunctionNotRegistered(e),this._queue.push(e)},this)},t.Pipeline.prototype.after=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i+1,0,n)},t.Pipeline.prototype.before=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i,0,n)},t.Pipeline.prototype.remove=function(e){var t=this._queue.indexOf(e);-1!==t&&this._queue.splice(t,1)},t.Pipeline.prototype.run=function(e){for(var t=[],n=e.length,i=this._queue.length,o=0;n>o;o++){for(var r=e[o],s=0;i>s&&(r=this._queue[s](r,o,e),void 0!==r&&null!==r);s++);void 0!==r&&null!==r&&t.push(r)}return t},t.Pipeline.prototype.reset=function(){this._queue=[]},t.Pipeline.prototype.get=function(){return this._queue},t.Pipeline.prototype.toJSON=function(){return this._queue.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.DocumentStore,this.index={},this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var e=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,e)},t.Index.prototype.off=function(e,t){return this.eventEmitter.removeListener(e,t)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;n._fields=e.fields,n._ref=e.ref,n.documentStore=t.DocumentStore.load(e.documentStore),n.pipeline=t.Pipeline.load(e.pipeline),n.index={};for(var i in e.index)n.index[i]=t.InvertedIndex.load(e.index[i]);return n},t.Index.prototype.addField=function(e){return this._fields.push(e),this.index[e]=new t.InvertedIndex,this},t.Index.prototype.setRef=function(e){return this._ref=e,this},t.Index.prototype.saveDocument=function(e){return this.documentStore=new t.DocumentStore(e),this},t.Index.prototype.addDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.addDoc(i,e),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));this.documentStore.addFieldLength(i,n,o.length);var r={};o.forEach(function(e){e in r?r[e]+=1:r[e]=1},this);for(var s in r){var u=r[s];u=Math.sqrt(u),this.index[n].addToken(s,{ref:i,tf:u})}},this),n&&this.eventEmitter.emit("add",e,this)}},t.Index.prototype.removeDocByRef=function(e){if(e&&this.documentStore.isDocStored()!==!1&&this.documentStore.hasDoc(e)){var t=this.documentStore.getDoc(e);this.removeDoc(t,!1)}},t.Index.prototype.removeDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.hasDoc(i)&&(this.documentStore.removeDoc(i),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));o.forEach(function(e){this.index[n].removeToken(e,i)},this)},this),n&&this.eventEmitter.emit("remove",e,this))}},t.Index.prototype.updateDoc=function(e,t){var t=void 0===t?!0:t;this.removeDocByRef(e[this._ref],!1),this.addDoc(e,!1),t&&this.eventEmitter.emit("update",e,this)},t.Index.prototype.idf=function(e,t){var n="@"+t+"/"+e;if(Object.prototype.hasOwnProperty.call(this._idfCache,n))return this._idfCache[n];var i=this.index[t].getDocFreq(e),o=1+Math.log(this.documentStore.length/(i+1));return this._idfCache[n]=o,o},t.Index.prototype.getFields=function(){return this._fields.slice()},t.Index.prototype.search=function(e,n){if(!e)return[];e="string"==typeof e?{any:e}:JSON.parse(JSON.stringify(e));var i=null;null!=n&&(i=JSON.stringify(n));for(var o=new t.Configuration(i,this.getFields()).get(),r={},s=Object.keys(e),u=0;u0&&t.push(e);for(var i in n)"docs"!==i&&"df"!==i&&this.expandToken(e+i,t,n[i]);return t},t.InvertedIndex.prototype.toJSON=function(){return{root:this.root}},t.Configuration=function(e,n){var e=e||"";if(void 0==n||null==n)throw new Error("fields should not be null");this.config={};var i;try{i=JSON.parse(e),this.buildUserConfig(i,n)}catch(o){t.utils.warn("user configuration parse failed, will use default configuration"),this.buildDefaultConfig(n)}},t.Configuration.prototype.buildDefaultConfig=function(e){this.reset(),e.forEach(function(e){this.config[e]={boost:1,bool:"OR",expand:!1}},this)},t.Configuration.prototype.buildUserConfig=function(e,n){var i="OR",o=!1;if(this.reset(),"bool"in e&&(i=e.bool||i),"expand"in e&&(o=e.expand||o),"fields"in e)for(var r in e.fields)if(n.indexOf(r)>-1){var s=e.fields[r],u=o;void 0!=s.expand&&(u=s.expand),this.config[r]={boost:s.boost||0===s.boost?s.boost:1,bool:s.bool||i,expand:u}}else t.utils.warn("field name in user configuration not found in index instance fields");else this.addAllFields2UserConfig(i,o,n)},t.Configuration.prototype.addAllFields2UserConfig=function(e,t,n){n.forEach(function(n){this.config[n]={boost:1,bool:e,expand:t}},this)},t.Configuration.prototype.get=function(){return this.config},t.Configuration.prototype.reset=function(){this.config={}},lunr.SortedSet=function(){this.length=0,this.elements=[]},lunr.SortedSet.load=function(e){var t=new this;return t.elements=e,t.length=e.length,t},lunr.SortedSet.prototype.add=function(){var e,t;for(e=0;e1;){if(r===e)return o;e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o]}return r===e?o:-1},lunr.SortedSet.prototype.locationFor=function(e){for(var t=0,n=this.elements.length,i=n-t,o=t+Math.floor(i/2),r=this.elements[o];i>1;)e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o];return r>e?o:e>r?o+1:void 0},lunr.SortedSet.prototype.intersect=function(e){for(var t=new lunr.SortedSet,n=0,i=0,o=this.length,r=e.length,s=this.elements,u=e.elements;;){if(n>o-1||i>r-1)break;s[n]!==u[i]?s[n]u[i]&&i++:(t.add(s[n]),n++,i++)}return t},lunr.SortedSet.prototype.clone=function(){var e=new lunr.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},lunr.SortedSet.prototype.union=function(e){var t,n,i;this.length>=e.length?(t=this,n=e):(t=e,n=this),i=t.clone();for(var o=0,r=n.toArray();oe.length)&&(t=e.length);for(var n=0,r=new Array(t);n0&&void 0!==arguments[0]?arguments[0]:3,t=new Map,n=Math.pow(10,e);return{get:function(e){var r=e.match(w).length;if(t.has(r))return t.get(r);var i=1/Math.sqrt(r),o=parseFloat(Math.round(i*n)/n);return t.set(r,o),o},clear:function(){t.clear()}}}var _=function(){function e(){var n=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},r=n.getFn,i=void 0===r?x.getFn:r;t(this,e),this.norm=L(3),this.getFn=i,this.isCreated=!1,this.setIndexRecords()}return r(e,[{key:"setSources",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];this.docs=e}},{key:"setIndexRecords",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];this.records=e}},{key:"setKeys",value:function(){var e=this,t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];this.keys=t,this._keysMap={},t.forEach((function(t,n){e._keysMap[t.id]=n}))}},{key:"create",value:function(){var e=this;!this.isCreated&&this.docs.length&&(this.isCreated=!0,u(this.docs[0])?this.docs.forEach((function(t,n){e._addString(t,n)})):this.docs.forEach((function(t,n){e._addObject(t,n)})),this.norm.clear())}},{key:"add",value:function(e){var t=this.size();u(e)?this._addString(e,t):this._addObject(e,t)}},{key:"removeAt",value:function(e){this.records.splice(e,1);for(var t=e,n=this.size();t2&&void 0!==arguments[2]?arguments[2]:{},r=n.getFn,i=void 0===r?x.getFn:r,o=new _({getFn:i});return o.setKeys(e.map(b)),o.setSources(t),o.create(),o}function O(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=t.errors,r=void 0===n?0:n,i=t.currentLocation,o=void 0===i?0:i,a=t.expectedLocation,c=void 0===a?0:a,s=t.distance,h=void 0===s?x.distance:s,u=t.ignoreLocation,l=void 0===u?x.ignoreLocation:u,d=r/e.length;if(l)return d;var f=Math.abs(c-o);return h?d+f/h:f?1:d}function A(){for(var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[],t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:x.minMatchCharLength,n=[],r=-1,i=-1,o=0,a=e.length;o=t&&n.push([r,i]),r=-1)}return e[o-1]&&o-r>=t&&n.push([r,o-1]),n}function j(e){for(var t={},n=0,r=e.length;n1&&void 0!==arguments[1]?arguments[1]:{},o=i.location,a=void 0===o?x.location:o,c=i.threshold,s=void 0===c?x.threshold:c,h=i.distance,u=void 0===h?x.distance:h,l=i.includeMatches,d=void 0===l?x.includeMatches:l,f=i.findAllMatches,v=void 0===f?x.findAllMatches:f,y=i.minMatchCharLength,g=void 0===y?x.minMatchCharLength:y,p=i.isCaseSensitive,m=void 0===p?x.isCaseSensitive:p,b=i.ignoreLocation,k=void 0===b?x.ignoreLocation:b;if(t(this,e),this.options={location:a,threshold:s,distance:u,includeMatches:d,findAllMatches:v,minMatchCharLength:g,isCaseSensitive:m,ignoreLocation:k},this.pattern=m?n:n.toLowerCase(),this.chunks=[],this.pattern.length){var M=function(e,t){r.chunks.push({pattern:e,alphabet:j(e),startIndex:t})},w=this.pattern.length;if(w>32){for(var L=0,_=w%32,S=w-_;L3&&void 0!==arguments[3]?arguments[3]:{},i=r.location,o=void 0===i?x.location:i,a=r.distance,c=void 0===a?x.distance:a,s=r.threshold,h=void 0===s?x.threshold:s,u=r.findAllMatches,l=void 0===u?x.findAllMatches:u,d=r.minMatchCharLength,f=void 0===d?x.minMatchCharLength:d,v=r.includeMatches,y=void 0===v?x.includeMatches:v,p=r.ignoreLocation,m=void 0===p?x.ignoreLocation:p;if(t.length>32)throw new Error(g(32));for(var b,k=t.length,M=e.length,w=Math.max(0,Math.min(o,M)),L=h,_=w,S=f>1||y,j=S?Array(M):[];(b=e.indexOf(t,_))>-1;){var E=O(t,{currentLocation:b,expectedLocation:w,distance:c,ignoreLocation:m});if(L=Math.min(E,L),_=b+k,S)for(var I=0;I=J;U-=1){var q=U-1,B=n[e.charAt(q)];if(S&&(j[q]=+!!B),T[U]=(T[U+1]<<1|1)&B,$&&(T[U]|=(C[U+1]|C[U])<<1|1|C[U+1]),T[U]&N&&(F=O(t,{errors:$,currentLocation:q,expectedLocation:w,distance:c,ignoreLocation:m}))<=L){if(L=F,(_=q)<=w)break;J=Math.max(1,2*w-_)}}var V=O(t,{errors:$+1,currentLocation:w,expectedLocation:w,distance:c,ignoreLocation:m});if(V>L)break;C=T}var G={isMatch:_>=0,score:Math.max(.001,F)};if(S){var H=A(j,f);H.length?y&&(G.indices=H):G.isMatch=!1}return G}(e,n,i,{location:a+o,distance:s,threshold:h,findAllMatches:u,minMatchCharLength:l,includeMatches:r,ignoreLocation:d}),m=p.isMatch,b=p.score,k=p.indices;m&&(y=!0),v+=b,m&&k&&(f=[].concat(c(f),c(k)))}));var p={isMatch:y,score:y?v/this.chunks.length:1};return y&&r&&(p.indices=f),p}}]),e}(),I=[];function C(e,t){for(var n=0,r=I.length;n-1&&(n.refIndex=e.idx),t.matches.push(n)}}))}function N(e,t){t.score=e.score}function $(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},r=n.includeMatches,i=void 0===r?x.includeMatches:r,o=n.includeScore,a=void 0===o?x.includeScore:o,c=[];return i&&c.push(P),a&&c.push(N),e.map((function(e){var n=e.idx,r={item:t[n],refIndex:n};return c.length&&c.forEach((function(t){t(e,r)})),r}))}var D=function(){function e(n){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},i=arguments.length>2?arguments[2]:void 0;if(t(this,e),this.options=a({},x,{},r),this.options.useExtendedSearch)throw new Error("Extended search is not available");this._keyStore=new m(this.options.keys),this.setCollection(n,i)}return r(e,[{key:"setCollection",value:function(e,t){if(this._docs=e,t&&!(t instanceof _))throw new Error("Incorrect 'index' type");this._myIndex=t||S(this.options.keys,this._docs,{getFn:this.options.getFn})}},{key:"add",value:function(e){f(e)&&(this._docs.push(e),this._myIndex.add(e))}},{key:"remove",value:function(){for(var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(){return!1},t=[],n=0,r=this._docs.length;n1&&void 0!==arguments[1]?arguments[1]:{},n=t.limit,r=void 0===n?-1:n,i=this.options,o=i.includeMatches,a=i.includeScore,c=i.shouldSort,s=i.sortFn,h=i.ignoreFieldNorm,d=u(e)?u(this._docs[0])?this._searchStringList(e):this._searchObjectList(e):this._searchLogical(e);return F(d,{ignoreFieldNorm:h}),c&&d.sort(s),l(r)&&r>-1&&(d=d.slice(0,r)),$(d,this._docs,{includeMatches:o,includeScore:a})}},{key:"_searchStringList",value:function(e){var t=C(e,this.options),n=this._myIndex.records,r=[];return n.forEach((function(e){var n=e.v,i=e.i,o=e.n;if(f(n)){var a=t.searchIn(n),c=a.isMatch,s=a.score,h=a.indices;c&&r.push({item:n,idx:i,matches:[{score:s,value:n,norm:o,indices:h}]})}})),r}},{key:"_searchLogical",value:function(e){throw new Error("Logical search is not available")}},{key:"_searchObjectList",value:function(e){var t=this,n=C(e,this.options),r=this._myIndex,i=r.keys,o=r.records,a=[];return o.forEach((function(e){var r=e.$,o=e.i;if(f(r)){var s=[];i.forEach((function(e,i){s.push.apply(s,c(t._findMatches({key:e,value:r[i],searcher:n})))})),s.length&&a.push({idx:o,item:r,matches:s})}})),a}},{key:"_findMatches",value:function(e){var t=e.key,n=e.value,r=e.searcher;if(!f(n))return[];var i=[];if(h(n))n.forEach((function(e){var n=e.v,o=e.i,a=e.n;if(f(n)){var c=r.searchIn(n),s=c.isMatch,h=c.score,u=c.indices;s&&i.push({score:h,key:t,value:n,idx:o,norm:a,indices:u})}}));else{var o=n.v,a=n.n,c=r.searchIn(o),s=c.isMatch,u=c.score,l=c.indices;s&&i.push({score:u,key:t,value:o,norm:a,indices:l})}return i}}]),e}();return D.version="6.4.6",D.createIndex=S,D.parseIndex=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=t.getFn,r=void 0===n?x.getFn:n,i=e.keys,o=e.records,a=new _({getFn:r});return a.setKeys(i),a.setIndexRecords(o),a},D.config=x,D},"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):(e=e||self).Fuse=t(); From c1d2fce5481c3650a21825efce6560c491904d1b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 May 2021 02:17:11 +0200 Subject: [PATCH 04/11] do not fail if inspect.getdoc raises --- CHANGELOG.md | 1 + pdoc/doc.py | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 155bc2ee..d1f15d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ # Unreleased: pdoc next - Update Bootstrap to v5.0.0. + - Do not fail if `inspect.getdoc()` raises. # 2021-04-30: pdoc 6.6.0 diff --git a/pdoc/doc.py b/pdoc/doc.py index 32d4b20a..56a80c6a 100644 --- a/pdoc/doc.py +++ b/pdoc/doc.py @@ -131,8 +131,7 @@ def docstring(self) -> str: If no docstring can be found, an empty string is returned. """ - doc = inspect.getdoc(self.obj) or "" - return doc.strip() + return _safe_getdoc(self.obj) @cached_property def source(self) -> str: @@ -744,8 +743,7 @@ def docstring(self) -> str: cls = sys.modules.get(_safe_getattr(self.obj, "__module__", None), None) for name in _safe_getattr(self.obj, "__qualname__", "").split(".")[:-1]: cls = _safe_getattr(cls, name, None) - doc = inspect.getdoc(_safe_getattr(cls, self.name, None)) or "" - doc = doc.strip() + doc = _safe_getdoc(_safe_getattr(cls, self.name, None)) if doc == object.__init__.__doc__: # inspect.getdoc(Foo.__init__) returns the docstring, for object.__init__ if left undefined... @@ -1040,3 +1038,17 @@ def _safe_getattr(obj, attr, default): RuntimeWarning, ) return default + + +def _safe_getdoc(obj: Any) -> str: + """Like `inspect.getdoc()`, but never raises. Always returns a stripped string.""" + try: + doc = inspect.getdoc(obj) or "" + except Exception as e: + warnings.warn( + f"inspect.getdoc({obj!r}) raised an exception: {e}", + RuntimeWarning, + ) + return "" + else: + return doc.strip() From 25307e30de5458d4fdcacef0c4b325f51780325e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 May 2021 02:18:49 +0200 Subject: [PATCH 05/11] extend elasticlunr search prototype --- CHANGELOG.md | 1 + pdoc/render.py | 90 ++++---- pdoc/render_helpers.py | 11 +- pdoc/templates/build-search-index.js | 24 +++ pdoc/templates/default/module.html.jinja2 | 239 ++++++++++++---------- pdoc/web.py | 21 ++ test/modd.conf | 2 +- 7 files changed, 239 insertions(+), 149 deletions(-) create mode 100644 pdoc/templates/build-search-index.js diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f15d36..78870b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ # Unreleased: pdoc next + - Add search functionality. - Update Bootstrap to v5.0.0. - Do not fail if `inspect.getdoc()` raises. diff --git a/pdoc/render.py b/pdoc/render.py index f61fa046..9a42fc6e 100644 --- a/pdoc/render.py +++ b/pdoc/render.py @@ -2,13 +2,16 @@ import json import os +import subprocess +import tempfile +import warnings from pathlib import Path from typing import Collection, Mapping, Optional from jinja2 import Environment, FileSystemLoader -import pdoc.docstrings import pdoc.doc +import pdoc.docstrings from pdoc._compat import Literal from pdoc.render_helpers import ( DefaultMacroExtension, @@ -19,6 +22,7 @@ linkify, minify_css, render_docstring, + render_docstring_with_context, ) @@ -100,42 +104,54 @@ def html_error(error: str, details: str = "") -> str: ) -def search_index(all_modules: dict[str, Optional[pdoc.doc.Module]]) -> str: +def search_index(all_modules: dict[str, pdoc.doc.Module]) -> str: """Renders the search index.""" - search_index = {} - with defuse_unsafe_reprs(): - for modname, mod in all_modules.items(): - docformat = getattr(mod.obj, "__docformat__", env.globals["docformat"]) or "" - - def make_item(doc: pdoc.doc.Doc, **kwargs) -> dict[str, str]: - docstr = pdoc.docstrings.convert(doc.docstring, docformat, mod.source_file) - if docstr: - docstr = _markdown(docstr) - return { - "qualname": doc.qualname, - "type": doc.type, - "doc": docstr, - **kwargs - } - - def make_index(mod: pdoc.doc.Namespace): - yield make_item(mod) - for m in mod.own_members: - if isinstance(m, pdoc.doc.Variable): - yield make_item(m) - elif isinstance(m, pdoc.doc.Function): - yield make_item( - m, - parameters=list(m.signature.parameters), - funcdef=m.funcdef, - ) - elif isinstance(m, pdoc.doc.Class): - yield from make_index(m) - else: - pass - - search_index[modname] = list(make_index(mod)) - return json.dumps(search_index) + documents = [] + for modname, mod in all_modules.items(): + docformat = getattr(mod.obj, "__docformat__", env.globals["docformat"]) or "" + + def make_item(doc: pdoc.doc.Doc, **kwargs) -> dict[str, str]: + return { + "fullname": doc.fullname, + "modulename": doc.modulename, + "qualname": doc.qualname, + "type": doc.type, + "doc": render_docstring(doc.docstring, mod, docformat), + **kwargs, + } + + def make_index(mod: pdoc.doc.Namespace): + yield make_item(mod) + for m in mod.own_members: + if isinstance(m, pdoc.doc.Variable): + yield make_item(m) + elif isinstance(m, pdoc.doc.Function): + yield make_item( + m, + parameters=list(m.signature.parameters), + funcdef=m.funcdef, + ) + elif isinstance(m, pdoc.doc.Class): + yield from make_index(m) + else: + pass + + documents.extend(make_index(mod)) + + raw = json.dumps(documents) + try: + out = subprocess.check_output( + ["node", env.get_template("build-search-index.js").filename], + input=raw.encode(), + cwd=Path(__file__).parent / "templates", + ) + index = json.loads(out) + index["_isPrebuiltIndex"] = True + except Exception as e: + warnings.warn(f"Error precompiling search index: {e}", UserWarning) + return raw + else: + return json.dumps(index) def repr_module(module: pdoc.doc.Module) -> str: @@ -161,7 +177,7 @@ def repr_module(module: pdoc.doc.Module) -> str: You can modify this object to add custom filters and globals. Examples can be found in this module's source code. """ -env.filters["render_docstring"] = render_docstring +env.filters["render_docstring"] = render_docstring_with_context env.filters["highlight"] = highlight env.filters["linkify"] = linkify env.filters["link"] = link diff --git a/pdoc/render_helpers.py b/pdoc/render_helpers.py index 8c6090cc..eae0c4d5 100644 --- a/pdoc/render_helpers.py +++ b/pdoc/render_helpers.py @@ -63,12 +63,19 @@ def _markdown(docstring: str) -> str: @contextfilter -def render_docstring(context: Context, docstring: str) -> str: +def render_docstring_with_context(context: Context, docstring: str) -> str: """ Converts `docstring` from a custom docformat to Markdown (if necessary), and then from Markdown to HTML. """ module: pdoc.doc.Module = context["module"] - docformat = getattr(module.obj, "__docformat__", context["docformat"]) or "" + docformat: Optional[str] = context["docformat"] + return render_docstring(docstring, module, docformat) + + +def render_docstring( + docstring: str, module: pdoc.doc.Module, default_docformat: str +) -> str: + docformat = getattr(module.obj, "__docformat__", default_docformat) or "" docstring = docstrings.convert(docstring, docformat, module.source_file) return _markdown(docstring) diff --git a/pdoc/templates/build-search-index.js b/pdoc/templates/build-search-index.js new file mode 100644 index 00000000..bd6e8b43 --- /dev/null +++ b/pdoc/templates/build-search-index.js @@ -0,0 +1,24 @@ +/** + * This script is invoked by pdoc to precompile the search index. + * Precompiling the search index increases file size, but skips the CPU-heavy index building in the browser. + */ +let elasticlunr = require('./elasticlunr.min'); + +let fs = require('fs'); +let docs = JSON.parse(fs.readFileSync(0, 'utf-8')); + +/* mirrored in module.html.jinja2 (part 1) */ +elasticlunr.tokenizer.setSeperator(/[\s\-.;&]+|<[^>]*>/); + +/* mirrored in module.html.jinja2 (part 2) */ +searchIndex = elasticlunr(function () { + this.addField('qualname'); + this.addField('fullname'); + this.addField('doc'); + this.setRef('fullname'); +}); +for (let doc of docs) { + searchIndex.addDoc(doc); +} + +process.stdout.write(JSON.stringify(searchIndex.toJSON())); diff --git a/pdoc/templates/default/module.html.jinja2 b/pdoc/templates/default/module.html.jinja2 index 278c737a..4152f44f 100644 --- a/pdoc/templates/default/module.html.jinja2 +++ b/pdoc/templates/default/module.html.jinja2 @@ -283,6 +283,7 @@ } .pdoc input[type=search] { + display: block; outline-offset: 0; } @@ -409,6 +410,7 @@ /* Attributes */ .pdoc .attr { + display: block; color: var(--text); margin: 1rem 0 .5rem; /* @@ -468,6 +470,12 @@ content: "()"; } + /* Search results */ + .pdoc .search-result .docstring { + overflow: auto; + max-height: 25vh; + } + /* "built with pdoc" attribution */ .pdoc .attribution { margin-top: 2rem; @@ -758,10 +766,10 @@ {% endblock %} diff --git a/pdoc/web.py b/pdoc/web.py index d24a6d54..d8f46187 100644 --- a/pdoc/web.py +++ b/pdoc/web.py @@ -13,6 +13,7 @@ import pkgutil import sysconfig import traceback +import warnings import webbrowser from typing import Collection, Optional, Union @@ -44,6 +45,11 @@ def handle_request(self) -> Optional[str]: if path == "/": out = render.html_index(self.server.all_modules) + elif path == "/search.json": + self.send_response(200) + self.send_header("content-type", "application/json") + self.end_headers() + return self.server.render_search_index() else: module = removesuffix(path.lstrip("/"), ".html").replace("/", ".") if module not in self.server.all_modules: @@ -104,6 +110,21 @@ def __init__( super().__init__(addr, DocHandler) self.all_modules = all_modules + @cache + def render_search_index(self) -> str: + """Render the search index. For performance reasons this is always cached.""" + all_mods = {} + for mod in self.all_modules: + try: + m = extract.load_module(mod) + except RuntimeError: + warnings.warn( + f"Error importing {mod}:\n{traceback.format_exc()}", RuntimeWarning + ) + else: + all_mods[mod] = doc.Module(m) + return render.search_index(all_mods) + # https://github.com/mitmproxy/mitmproxy/blob/af3dfac85541ce06c0e3302a4ba495fe3c77b18a/mitmproxy/tools/web/webaddons.py#L35-L61 def open_browser(url: str) -> bool: # pragma: no cover diff --git a/test/modd.conf b/test/modd.conf index 0cd14613..f1bf83c4 100644 --- a/test/modd.conf +++ b/test/modd.conf @@ -2,6 +2,6 @@ # cd .. && modd -f test/modd.conf pdoc/** test/** { - prep: pdoc -o .render test/testdata/flavors_rst.py + prep: pdoc -o .render pdoc daemon: devd -m -p 9000 .render } From 06768cce8226e23e5693970cbafb439b0f69492f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 May 2021 03:10:36 +0200 Subject: [PATCH 06/11] search index: hide non-public members --- pdoc/render.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pdoc/render.py b/pdoc/render.py index 9a42fc6e..f89dc117 100644 --- a/pdoc/render.py +++ b/pdoc/render.py @@ -1,13 +1,15 @@ from __future__ import annotations +import collections import json import os import subprocess -import tempfile +import types import warnings from pathlib import Path from typing import Collection, Mapping, Optional +import jinja2 from jinja2 import Environment, FileSystemLoader import pdoc.doc @@ -106,6 +108,15 @@ def html_error(error: str, details: str = "") -> str: def search_index(all_modules: dict[str, pdoc.doc.Module]) -> str: """Renders the search index.""" + # This is a rather terrible hack to determine if a given object is public and should be included in the index. + module_template: jinja2.Template = env.select_template(["module.html.jinja2", "default/module.html.jinja2"]) + ctx: jinja2.runtime.Context = module_template.new_context({ + "module": pdoc.doc.Module(types.ModuleType("")), + "all_modules": {} + }) + for _ in module_template.root_render_func(ctx): # type: ignore + pass + documents = [] for modname, mod in all_modules.items(): docformat = getattr(mod.obj, "__docformat__", env.globals["docformat"]) or "" @@ -121,11 +132,13 @@ def make_item(doc: pdoc.doc.Doc, **kwargs) -> dict[str, str]: } def make_index(mod: pdoc.doc.Namespace): + if not ctx["is_public"](mod): + return yield make_item(mod) for m in mod.own_members: - if isinstance(m, pdoc.doc.Variable): + if isinstance(m, pdoc.doc.Variable) and ctx["is_public"](m): yield make_item(m) - elif isinstance(m, pdoc.doc.Function): + elif isinstance(m, pdoc.doc.Function) and ctx["is_public"](m): yield make_item( m, parameters=list(m.signature.parameters), From df9cffbda670b2b9b943452acc03023422a309f1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 May 2021 17:07:13 +0200 Subject: [PATCH 07/11] simplify default template fallback --- pdoc/render.py | 13 ++++--------- pdoc/templates/{ => default}/frame.html.jinja2 | 0 2 files changed, 4 insertions(+), 9 deletions(-) rename pdoc/templates/{ => default}/frame.html.jinja2 (100%) diff --git a/pdoc/render.py b/pdoc/render.py index f89dc117..a719ce1b 100644 --- a/pdoc/render.py +++ b/pdoc/render.py @@ -75,9 +75,7 @@ def html_module( This is only passed by `pdoc.web`. """ with defuse_unsafe_reprs(): - return env.select_template( - ["module.html.jinja2", "default/module.html.jinja2"] - ).render( + return env.get_template("module.html.jinja2").render( module=module, all_modules=all_modules, mtime=mtime, @@ -89,18 +87,14 @@ def html_module( def html_index(all_modules: Collection[str]) -> str: """Renders the module index.""" - return env.select_template( - ["index.html.jinja2", "default/index.html.jinja2"] - ).render( + return env.get_template("index.html.jinja2").render( all_modules=[m for m in all_modules if "._" not in m], ) def html_error(error: str, details: str = "") -> str: """Renders an error message.""" - return env.select_template( - ["index.html.jinja2", "default/error.html.jinja2"] - ).render( + return env.get_template("error.html.jinja2").render( error=error, details=details, ) @@ -176,6 +170,7 @@ def repr_module(module: pdoc.doc.Module) -> str: _default_searchpath = [ Path(os.environ.get("XDG_CONFIG_HOME", "~/.config")).expanduser() / "pdoc", Path(__file__).parent / "templates", + Path(__file__).parent / "templates" / "default", ] env = Environment( diff --git a/pdoc/templates/frame.html.jinja2 b/pdoc/templates/default/frame.html.jinja2 similarity index 100% rename from pdoc/templates/frame.html.jinja2 rename to pdoc/templates/default/frame.html.jinja2 From 0f4a893bb500d7dd05aa952715043d926e08fe9b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 May 2021 17:08:44 +0200 Subject: [PATCH 08/11] search: add keyboard result selection --- pdoc/templates/default/module.html.jinja2 | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/pdoc/templates/default/module.html.jinja2 b/pdoc/templates/default/module.html.jinja2 index 4152f44f..7a1281f4 100644 --- a/pdoc/templates/default/module.html.jinja2 +++ b/pdoc/templates/default/module.html.jinja2 @@ -933,6 +933,36 @@ } else { searchBox.addEventListener("focus", initialize, {once: true}); } + + {# keyboard navigation for results #} + searchBox.addEventListener("keydown", e => { + if (["ArrowDown", "ArrowUp", "Enter"].includes(e.key)) { + let focused = currentContent.querySelector(".search-result.focused"); + if (!focused) { + currentContent.querySelector(".search-result").classList.add("focused"); + } else if ( + e.key === "ArrowDown" + && focused.nextElementSibling + && focused.nextElementSibling.classList.contains("search-result") + ) { + focused.classList.remove("focused"); + focused.nextElementSibling.classList.add("focused"); + focused.nextElementSibling.scrollIntoView({behavior: "smooth", block: "nearest", inline: "nearest"}); + } else if ( + e.key === "ArrowUp" + && focused.previousElementSibling + && focused.previousElementSibling.classList.contains("search-result") + ) { + focused.classList.remove("focused"); + focused.previousElementSibling.classList.add("focused"); + focused.previousElementSibling.scrollIntoView({behavior: "smooth", block: "nearest", inline: "nearest"}); + } else if ( + e.key === "Enter" + ) { + focused.querySelector("a").click(); + } + } + }); {% endblock %} {% endblock %} From 1da0cf3a0cc759412d7d1a75d33fb0374a5b4fe5 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 May 2021 17:08:57 +0200 Subject: [PATCH 09/11] search: revise module index --- pdoc/render.py | 7 +- pdoc/templates/default/error.html.jinja2 | 2 +- pdoc/templates/default/index.html.jinja2 | 168 ++++++---------------- pdoc/templates/default/module.html.jinja2 | 37 +++-- 4 files changed, 69 insertions(+), 145 deletions(-) diff --git a/pdoc/render.py b/pdoc/render.py index a719ce1b..2cc79359 100644 --- a/pdoc/render.py +++ b/pdoc/render.py @@ -1,6 +1,5 @@ from __future__ import annotations -import collections import json import os import subprocess @@ -126,13 +125,13 @@ def make_item(doc: pdoc.doc.Doc, **kwargs) -> dict[str, str]: } def make_index(mod: pdoc.doc.Namespace): - if not ctx["is_public"](mod): + if not ctx["is_public"](mod).strip(): return yield make_item(mod) for m in mod.own_members: - if isinstance(m, pdoc.doc.Variable) and ctx["is_public"](m): + if isinstance(m, pdoc.doc.Variable) and ctx["is_public"](m).strip(): yield make_item(m) - elif isinstance(m, pdoc.doc.Function) and ctx["is_public"](m): + elif isinstance(m, pdoc.doc.Function) and ctx["is_public"](m).strip(): yield make_item( m, parameters=list(m.signature.parameters), diff --git a/pdoc/templates/default/error.html.jinja2 b/pdoc/templates/default/error.html.jinja2 index 87f40429..bc969ecd 100644 --- a/pdoc/templates/default/error.html.jinja2 +++ b/pdoc/templates/default/error.html.jinja2 @@ -2,7 +2,7 @@ {% block title %}{{ error }}{% endblock %} {% block style %} {{ super() | safe }} - {% endblock %} -{% block body %} -
- {% block logo %} - - {% endblock %} - -
Available Modules
-
- {% for module in all_modules %} - {{ module }} - {% endfor %} -
- -
+{% block nav %} + +
+ {% block logo %} + + pdoc logo + + {% endblock %} + +
{% endblock %} +{% block module_info %}{% endblock %} +{% block module_contents %}{% endblock %} diff --git a/pdoc/templates/default/module.html.jinja2 b/pdoc/templates/default/module.html.jinja2 index 7a1281f4..0977acc1 100644 --- a/pdoc/templates/default/module.html.jinja2 +++ b/pdoc/templates/default/module.html.jinja2 @@ -3,9 +3,9 @@ {% block style %} {% filter minify_css %} {{ super() }} - + {% block style_pdoc %} - - - + +