From 1f5af7955ce9e15cf2f97f53e7175608c7e17b8c Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Sun, 19 Dec 2021 13:21:46 +0900 Subject: [PATCH 1/7] generate html in parallel, but it's very slow --- .github/workflows/generate.yml | 2 +- Makefile | 26 +++++++++++++ tools/build_html.vim | 68 ++++++++++++++++++++++++++++++++++ tools/build_tag.vim | 40 ++++++++++++++++++++ 4 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 tools/build_html.vim create mode 100644 tools/build_tag.vim diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index 5f743f2a0..feb00e8d1 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -81,7 +81,7 @@ jobs: run: | cd work # Create html documents - make html + make -j4 html2 cd .. - name: Lock mutex diff --git a/Makefile b/Makefile index cf726c639..823424165 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,10 @@ +SRC_JAX := $(wildcard doc/*.jax vim_faq/*.jax) +OUT_JAX := $(addprefix target/html/doc/,$(notdir $(SRC_JAX))) +TAGS_JAX := target/html/doc/tags.jax +HTML := $(OUT_JAX:.jax=.html) $(TAGS_JAX:.jax=.html) + .PHONY: all check replace html clean +.PHONY: html2 tags all: @@ -15,5 +21,25 @@ html: cp doc/*.jax vim_faq/*.jax target/html/doc -cd target/html/doc ; vim -eu ../../../tools/buildhtml.vim -c "qall!" +html2: $(HTML) + +tags: $(TAGS_JAX) + +target/html/doc/%.jax: doc/%.jax + -@mkdir -p target/html/doc + cp $< $@ + +target/html/doc/%.jax: vim_faq/%.jax + -@mkdir -p target/html/doc + cp $< $@ + +$(TAGS_JAX): $(OUT_JAX) + rm -f $@ + -cd target/html/doc ; vim -eu ../../../tools/build_tag.vim -c "qall!" + +target/html/doc/%.html: target/html/doc/%.jax $(TAGS_JAX) + -@mkdir -p target/html/doc + -cd target/html/doc ; vim -eu ../../../tools/build_html.vim $(notdir $(:h/tag_aliases.vim +source :h/untranslated.vim +source :h/makehtml.vim + +let s:tools_dir = expand(':p:h') +let s:proj_dir = expand(':p:h:h') + +function! s:main() + " for the lastest help syntax + let &runtimepath = s:tools_dir . ',' . &runtimepath + " for ja custom syntax + let &runtimepath .= ',' . s:proj_dir + if argc() != 2 + echoerr "require two arguments" + finish + endif + call s:BuildHtml(argv(0), argv(1)) +endfunction + +function! s:BuildHtml(src, dst) + call MakeHtml2(a:src, a:dst, 1) + call s:ToJekyll(a:dst) +endfunction + +function! s:ToJekyll(file) + set nomodeline + execute "edit! " . a:file + set fileformat=unix + + let helpname = expand('%:t:r') + if helpname == 'index' + let helpname = 'help' + endif + + " remove header + silent 1,/^
/delete _ + " remove footer + silent /^
/,$delete _ + + " escape jekyll tags + silent %s/{\{2,}\|{%/{{ "\0" }}/ge + + " YAML front matter + call append(0, [ + \ '---', + \ 'layout: vimdoc', + \ printf("helpname: '%s'", helpname), + \ '---', + \ ]) + + update! +endfunction + +call s:main() diff --git a/tools/build_tag.vim b/tools/build_tag.vim new file mode 100644 index 000000000..bd4fc201a --- /dev/null +++ b/tools/build_tag.vim @@ -0,0 +1,40 @@ +set nocompatible +set nomore +set encoding=utf-8 +set fileencodings=utf-8 +syntax on +colorscheme delek +let g:html_no_progress = 1 + +if has('windows') && !has('gui_running') + set t_ti= +endif + +enew! + +source :h/tag_aliases.vim +source :h/untranslated.vim +source :h/makehtml.vim + +let s:tools_dir = expand(':p:h') +let s:proj_dir = expand(':p:h:h') + +function! s:main() + " for the lastest help syntax + let &runtimepath = s:tools_dir . ',' . &runtimepath + " for ja custom syntax + let &runtimepath .= ',' . s:proj_dir + call s:BuildTag() +endfunction + +function! s:BuildTag + " generate tags + try + helptags . + catch + echo v:exception + endtry + call MakeTagsFile() +endfunction + +call s:main() From 2f1fbdb0b2b4e4c94b4067226ea343615b4c80b2 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Sun, 19 Dec 2021 18:43:43 +0900 Subject: [PATCH 2/7] fix func definision --- tools/build_tag.vim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_tag.vim b/tools/build_tag.vim index bd4fc201a..2559d0561 100644 --- a/tools/build_tag.vim +++ b/tools/build_tag.vim @@ -27,7 +27,7 @@ function! s:main() call s:BuildTag() endfunction -function! s:BuildTag +function! s:BuildTag() " generate tags try helptags . From 566876ba251e35cdfc84392b3d11bd0b33b2626b Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Mon, 20 Dec 2021 01:59:48 +0900 Subject: [PATCH 3/7] cache tags-xx as JSON file for speed --- tools/makehtml.vim | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/tools/makehtml.vim b/tools/makehtml.vim index 2d3f8fb9e..8c394dc27 100644 --- a/tools/makehtml.vim +++ b/tools/makehtml.vim @@ -38,6 +38,10 @@ function! MakeHtmlAll(...) endif endfunction +function! s:JsonName(lang) + return a:lang == "" ? "tags.json" : printf("tags-%s.json", a:lang) +endfunction + function! MakeTagsFile() let files = split(glob('tags'), '\n') let files += split(glob('tags-??'), '\n') @@ -50,7 +54,7 @@ function! MakeTagsFile() endif silent new `=fname` silent %delete _ - let tags = s:GetTags(lang) + let tags = s:LoadTags(lang) for tagname in sort(keys(tags)) if tagname == "help-tags" continue @@ -60,6 +64,8 @@ function! MakeTagsFile() call append('$', ' vim:ft=help:') silent 1delete _ silent wq! + " save JSON tag cache + call writefile([js_encode(tags)], s:JsonName(lang), "s") endfor endfunction @@ -216,26 +222,36 @@ endfunction function! s:GetTags(lang) if !exists("s:tags_" . a:lang) - let &l:tags = (a:lang == "") ? "./tags" : "./tags-" . a:lang - let tags = {} - for item in taglist(".*") - let item["html"] = s:HtmlName(item["filename"]) - let tags[item["name"]] = item - endfor - " for help-tags - let item = {} - let item["name"] = "help-tags" - if s:IsSingleMode() - let item["html"] = "tags.html" + " try to load JSON tag cache + let jsonname = s:JsonName(a:lang) + if filereadable(jsonname) + let s:tags_{a:lang} = js_decode(readfile(jsonname)[0]) else - let item["html"] = printf("tags%s.html", (a:lang == "") ? "" : "." . a:lang) + let s:tags_{a:lang} = s:LoadTags(a:lang) endif - let tags[item["name"]] = item - let s:tags_{a:lang} = tags endif return s:tags_{a:lang} endfunction +func s:LoadTags(lang) + let &l:tags = (a:lang == "") ? "./tags" : "./tags-" . a:lang + let tags = {} + for item in taglist(".*") + let item["html"] = s:HtmlName(item["filename"]) + let tags[item["name"]] = item + endfor + " for help-tags + let item = {} + let item["name"] = "help-tags" + if s:IsSingleMode() + let item["html"] = "tags.html" + else + let item["html"] = printf("tags%s.html", (a:lang == "") ? "" : "." . a:lang) + endif + let tags[item["name"]] = item + return tags +endfunction + function! s:IsSingleMode() " if there is one language files, do not append language identifier " (use "help.html" instead of "help.ab.html"). From 5efb601abfbd9656a2afa718e3a67547bfccb186 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Mon, 27 Dec 2021 16:48:53 +0900 Subject: [PATCH 4/7] trial with GitHub Actions --- .github/workflows/parallel-build.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/parallel-build.yml diff --git a/.github/workflows/parallel-build.yml b/.github/workflows/parallel-build.yml new file mode 100644 index 000000000..e0c1b36a1 --- /dev/null +++ b/.github/workflows/parallel-build.yml @@ -0,0 +1,18 @@ +name: Parallel build + +on: [push, pull_request] + +jobs: + build-in-parallel: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: Setup Vim + uses: thinca/action-setup-vim@v1 + with: + vim_version: 'v8.2.0020' + vim_type: 'Vim' + - name: Generate new document + run: | + make -j4 html2 From 1eb9bfe6596d2640afd95823c5f06e85ffbd1190 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Mon, 27 Dec 2021 16:57:37 +0900 Subject: [PATCH 5/7] increase parallelism to 6 --- .github/workflows/parallel-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/parallel-build.yml b/.github/workflows/parallel-build.yml index e0c1b36a1..1ce692dd0 100644 --- a/.github/workflows/parallel-build.yml +++ b/.github/workflows/parallel-build.yml @@ -1,6 +1,6 @@ name: Parallel build -on: [push, pull_request] +on: [push] jobs: build-in-parallel: @@ -15,4 +15,4 @@ jobs: vim_type: 'Vim' - name: Generate new document run: | - make -j4 html2 + make -j6 html2 From a8a9b8c00e4ff41a30fb6ae74932bcd16d3cabc1 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Mon, 27 Dec 2021 17:25:51 +0900 Subject: [PATCH 6/7] reduce parallelism to 2 --- .github/workflows/parallel-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/parallel-build.yml b/.github/workflows/parallel-build.yml index 1ce692dd0..acaf30040 100644 --- a/.github/workflows/parallel-build.yml +++ b/.github/workflows/parallel-build.yml @@ -15,4 +15,4 @@ jobs: vim_type: 'Vim' - name: Generate new document run: | - make -j6 html2 + make -j2 html2 From 12c027dca71199dc7926366ced7345e8f5c3fbb9 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Mon, 27 Dec 2021 17:34:19 +0900 Subject: [PATCH 7/7] remove experimental workflow --- .github/workflows/parallel-build.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 .github/workflows/parallel-build.yml diff --git a/.github/workflows/parallel-build.yml b/.github/workflows/parallel-build.yml deleted file mode 100644 index acaf30040..000000000 --- a/.github/workflows/parallel-build.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Parallel build - -on: [push] - -jobs: - build-in-parallel: - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v2 - - name: Setup Vim - uses: thinca/action-setup-vim@v1 - with: - vim_version: 'v8.2.0020' - vim_type: 'Vim' - - name: Generate new document - run: | - make -j2 html2