View source for Blog:Statically hosting a wiki/Fetcher

Note that unlike the prior version, I didn't provide any customization here. Everything relating to the wiki is hard-baked in. You will need to get <code>gumbo</code> from Luarocks and <code>json</code> from https://github.com/rxi/json.lua. You'll also need to paste [[MediaWiki:Common.js]] to <code>common.js</code> manually.

By the way, make sure that you're using a file system which allows for case-sensitivity, otherwise you'll have issues.

<syntaxhighlight lang="lua">
local gumbo = require("gumbo")
local json = require("json")

function urlencode(str)
    if str then
        str = string.gsub(str, "\n", "\r\n")
        str = string.gsub(str, "([^%w%-_%.~])", function(c)
            return string.format("%%%02X", string.byte(c))
        end)
        str = string.gsub(str, " ", "+")
    end
    return str
end
function params(data)
    local raw = ""
    for i,v in pairs(data) do
        raw = raw..urlencode(v)..(i%2==1 and "=" or "&")
    end
    return raw:sub(1, -2)
end

local nhbase = [[<!DOCTYPE html><html><head>
    <meta charset="UTF-8">
    <title>{TITLE} - aadenboy wiki</title>
    <link rel="stylesheet" href="/wiki/main.css">
    <script src="https://code.jquery.com/jquery-4.0.0.min.js"></script>
    <meta name="ResourceLoaderDynamicStyles" content="">
    <link rel="stylesheet" href="/wiki/sitestyles.css">
    <meta name="generator" content="MediaWiki 1.43.0">
    <meta name="robots" content="max-image-preview:standard">
    <meta name="format-detection" content="telephone=no">
    <link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/">
</head><body></body></html>]]

function meta(document, name, property, content)
    local m = document:createElement("meta")
    if name then m:setAttribute("name", name) end
    m:setAttribute("property", property)
    m:setAttribute("content", content)
    document:getElementsByTagName("head")[1]:appendChild(m)
end
function text(node)
    if node.nodeType == 3 or node.data then return node.data end
    local t = ""
    for i,v in ipairs(node.childNodes) do
        t = t..text(v)
    end
    return t
end

function fix(title, mode)
    local file = io.open("localhost:8888/mediawiki/index.php/"..title..(mode or "")..".html", "r")
    local page = file:read("*a")
    file:close()

    --[[
        Changes:
            * Replace the head
                * Also the title
                * But carry over any custom meta tags
            * Change "/mediawiki/", "/mediawiki/index.php/", and "/mediawiki/index.php?title=" in URLs to "/wiki/"
                * If the link contains "&amp;action=history", replace it with the repository link
            * Remove element with class "printfooter"
            * Remove element with ID "p-personal"
            * Remove element with ID "left-navigation"
            * Change element with ID "right-navigation" to have the id "left-navigation" instead
            * Change first child of element with ID "ca-viewsource" to have the title "View the source of this page for editing."
            * Remove element with ID "p-cactions"
            * Remove element with ID "p-search"
            * Remove element with ID "p-tb"
            * Replace the final script element with `<script async="" src="/common.js"></script>`
            * Generate metadata
    ]]

    local document = gumbo.parse(page)
    local oldhead = document:getElementsByTagName("head")[1]
    local newhead = gumbo.parse(nhbase):getElementsByTagName("head")[1]
    oldhead.parentNode:replaceChild(newhead, oldhead)
    local telem = oldhead:getElementsByTagName("title")[1]
    newhead:replaceChild(telem, newhead:getElementsByTagName("title")[1]) -- lazy
    
    local metas = document:getElementsByTagName("meta")
    for i,v in ipairs(metas) do
        if v:getAttribute("name") == "page-no-include" and v:getAttribute("content") == "yes" then
            return false
        end
    end

    -- no querySelector :(
    local anchors = document:getElementsByTagName("a")
    local images = document:getElementsByTagName("img")
    for i,v in ipairs(anchors) do
        v:setAttribute("href", (v:getAttribute("href") or "")
            :gsub("^/mediawiki/index%.php%?title=", "/wiki/")
            :gsub("^/mediawiki/index%.php/", "/wiki/")
            :gsub("^/mediawiki/", "/wiki/")
            :gsub("&action=edit", "%%3Faction=edit")
            :gsub("^.*&action=history.*$", "https://github.com/aadenboy/wiki/commits/main/"..title..".html")
        )
    end
    for i,v in ipairs(images) do
        v:setAttribute("src", (v:getAttribute("src") or "")
            :gsub("^/mediawiki/index%.php%?title=", "/wiki/")
            :gsub("^/mediawiki/index%.php/", "/wiki/")
            :gsub("^/mediawiki/", "/wiki/")
        )
        v:setAttribute("srcset", (v:getAttribute("srcset") or "")
            :gsub("/mediawiki/index%.php%?title=", "/wiki/")
            :gsub("/mediawiki/index%.php/", "/wiki/")
            :gsub("/mediawiki/", "/wiki/")
        )
    end

    document:getElementsByClassName("printfooter")[1]:remove()
    document:getElementById("p-personal"):remove()
    document:getElementById("left-navigation"):remove()
    document:getElementById("right-navigation"):setAttribute("id", "left-navigation")
    document:getElementById("ca-viewsource").childNodes[1]:setAttribute("title", "View the source of this page for editing.")
    document:getElementById("p-search"):remove()
    document:getElementById("p-cactions"):remove()
    document:getElementById("p-tb"):remove()
    
    local scripts = document:getElementsByTagName("script")
    local commonscript = document:createElement("script")
    commonscript:setAttribute("async", "")
    commonscript:setAttribute("src", "/wiki/common.js")
    if scripts[#scripts].parentNode.tagName == "BODY" then
        scripts[#scripts].parentNode:replaceChild(commonscript, scripts[#scripts])
    else
        document.body:appendChild(commonscript)
    end

    --[[
        reference
        <meta property="og:site_name" content="Tecknar" />
        <meta property="og:url" content="https://aadenboy.github.io/tecknar/" />
        <meta property="og:type" content="website" />
        <meta name="title" property="og:title" content="Tecknar" />
        <meta name="image" property="og:image" content="https://aadenboy.github.io/tecknar/tecknar.png"/>
        <meta property="og:image:type" content="image/png">
        <meta property="og:image:width" content="906">
        <meta property="og:image:height" content="413">
        <meta name="description" property="og:description" content="A lightweight, embeddable drawing pad for the web." />
        <meta name="twitter:card" content="summary_large_image" />
        <meta name="twitter:title" content="Tecknar" />
        <meta name="twitter:description" content="A lightweight, embeddable drawing pad for the web." />
        <meta name="twitter:image:src" content="https://aadenboy.github.io/tecknar/tecknar.png">
        <meta name="twitter:image" content="https://aadenboy.github.io/tecknar/tecknar.png">
        <meta name="twitter:domain" content="https://aadenboy.github.io/tecknar/">
    ]]
    meta(document, nil, "og:site_name", "Aadenboy Wiki")
    meta(document, nil, "og:url", "https://aadenboy.github.io/wiki/"..urlencode(title))
    meta(document, nil, "og:type", "website")
    meta(document, "title", "og:title", telem.innerHTML:gsub(" %- .+$", ""))
    local firstimage = document:getElementsByTagName("img")[1]
    if firstimage then meta(document, "image", "og:image", "https://aadenboy.github.io"..firstimage:getAttribute("src")) end

    local paragraphs = document:getElementsByTagName("p")
    local description = ""
    for i,v in pairs(paragraphs) do
        if type(v) == "table" and not (v:getAttribute("class") or ""):match("mw%-empty%-elt") and v.parentNode.parentNode:getAttribute("id") == "mw-content-text" then
            description = text(v)
            break
        end
    end
    description = description:gsub("%s*\n%s*", " "):gsub("^%s*(.-)%s*$", "%1")

    meta(document, "description", "description", description)
    meta(document, nil, "twitter:card", "summary_large_image")
    meta(document, nil, "twitter:title", telem.innerHTML:gsub(" %- .+$", ""))
    meta(document, nil, "twitter:description", description)
    if firstimage then
        meta(document, nil, "twitter:image:src", "https://aadenboy.github.io"..firstimage:getAttribute("src"))
        meta(document, nil, "twitter:image", "https://aadenboy.github.io"..firstimage:getAttribute("src"))
    end
    meta(document, nil, "twitter:domain", "https://aadenboy.github.io/wiki/")

    local final = document:serialize():gsub("<!%-%-.-%-%->", "") -- shrug

    if title:match("/") then
        os.execute("mkdir -p localhost:8888/mediawiki/"..title:gsub("/[^/]+$", "/"))
    end
    local outfile = io.open("localhost:8888/mediawiki/"..title..".html", "w")
    outfile:write(final)
    outfile:close()

    return true
end

function apijson(action)
    local call = io.popen('wget -q -o /dev/null --no-cache -O - "http://localhost:8888/mediawiki/api.php?'..params(action)..'"')
    local data = json.decode(call:read("*a"))
    call:close()
    return data
end

local nsdata = apijson{
    "action","query";
    "format","json";
    "meta","siteinfo";
    "siprop","namespaces";
}

local sorted = {}
for i,v in pairs(nsdata.query.namespaces) do
    if i+0 > -1 and (i+0)%2==0 and i ~= "2" and i ~= "3" then table.insert(sorted, tonumber(i)) end
end
table.sort(sorted)
for i,v in ipairs(sorted) do
    print("Starting namespace "..nsdata.query.namespaces[v..""]["*"]:gsub("^$", "Main").." ("..i.." of "..#sorted..")")

    local pages = apijson{
        "action","query";
        "format","json";
        "list","allpages",
        "aplimit",500;
        "apnamespace",v;
    }
    print("\tQueried "..#pages.query.allpages.." pages")
    local totalpages = pages.query.allpages
    while pages.continue do
        pages = apijson{
            "action","query";
            "format","json";
            "list","allpages",
            "aplimit",500;
            "apnamespace",v;
            "apcontinue",pages.continue.apcontinue;
            "continue",pages.continue.continue;
        }
        for i,v in ipairs(pages.query.allpages) do
            totalpages[#totalpages+1] = v
        end
        print("\tQueried "..#pages.query.allpages.." additional pages (now "..#totalpages.." total pages)")
    end

    for i,v in ipairs(totalpages) do
        print("\tGenerating page "..v.title)
        v.title = v.title:gsub(" ", "_")
        local title = urlencode(v.title):gsub("%%2F", "/")
        local call = io.popen('wget -E -H -p --no-cache --content-on-error -o - "http://localhost:8888/mediawiki/index.php/'..title..'?redirect=no" | grep "Saving to"')
        --[[for f in call:read("*a"):gmatch('‘([^\n]+%?[0-9a-zA-Z]+)’') do
            print("\t\tGenerated "..f:gsub("^.+/", "")..", moving")
            os.execute("mv "..f.." "..f:gsub("%?[0-9a-zA-Z]+$", ""))
        end]]
        call:close()
        local included = fix(v.title, "?redirect=no")
        if not included then
            print("\t\tPage skipped, not included")
        else
            print("\t\tBase article finished")
            local call = io.popen('wget -E -H -p --no-cache --content-on-error -o - "http://localhost:8888/mediawiki/index.php/'..title..'?action=edit" | grep "Saving to"')
            --[[for f in call:read("*a"):gmatch('‘([^\n]+%?[0-9a-zA-Z]+)’') do
                print("\t\tGenerated "..f..", moving")
                os.execute("mv "..f.." "..f:gsub("%?[0-9a-zA-Z]+$"))
            end]]
            call:close()
            fix(v.title.."?action=edit")
            print("\t\tEdit page finished")
        end
    end
end

-- http://localhost:8888/mediawiki/api.php?action=query&format=json&list=allpages&aplimit=10&apnamespace=6&apcontinue=CatAlignmentChart.jpeg&continue=-||

--[[local pages = {}
for page in pagesstr:gmatch("[^\n]+") do
    local code = os.execute('wget -E -H -p --content-on-error "http://localhost:8888/mediawiki/index.php/'..urlencode(page)..'?redirect=no"')
    fix(page, "?redirect=no")
    local code = os.execute('wget -E -H -p --content-on-error "http://localhost:8888/mediawiki/index.php/'..urlencode(page)..'?action=edit"')
    fix(page.."?action=edit")
end]]
</syntaxhighlight>

I also use this Shell file, though your mileage may vary.
<syntaxhighlight lang="bash">
#!/bin/bash
export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"

cd ~/THE/FILE/PATH # Obviously not what it is called; this is just an example.
printf "\n\n%s\n" "== Automatic update: $(date -u +'%H:%M:%S, %-d %B %Y (UTC)') ==" > console.out
lua fetcher.lua

cd ~/THE/FILE/PATH/localhost:8888/mediawiki
rm -rf index.php
cp Main_Page.html index.html
shopt -s globstar
for i in **/*/*.*\?*; do
    mv "$i" "${i%%\?*}"
    echo "Moved $i to ${i%%\?*}"
end
: > main.css
for i in load.php*.css; do
    sed -E 's|http://localhost:8888/mediawiki/|/wiki/|g; s|/mediawiki/|/wiki/|g; s|\?[0-9a-zA-Z]+||g' $i >> main.css
done

if [[ -n $(git status -s) ]]; then
    echo "Committing"
    git add .
    git commit -m "Automatic update: $(date -u +'%H:%M:%S, %-d %B %Y (UTC)')"
    git push -u origin main
    echo "Pushed"
fi
</syntaxhighlight>

[[Category:Blogs]]