Blog:Statically hosting a wiki/Fetcher
Jump to navigation
Jump to search
Note that unlike the prior version, I didn't provide any customization here. Everything relating to the wiki is hard-baked in. You will need to get gumbo from Luarocks and json from https://github.com/rxi/json.lua. You'll also need to paste MediaWiki:Common.js to common.js manually.
By the way, make sure that you're using a file system which allows for case-sensitivity, otherwise you'll have issues.
local gumbo = require("gumbo")
local json = require("json")
function urlencode(str)
if str then
str = string.gsub(str, "\n", "\r\n")
str = string.gsub(str, "([^%w%-_%.~])", function(c)
return string.format("%%%02X", string.byte(c))
end)
str = string.gsub(str, " ", "+")
end
return str
end
function params(data)
local raw = ""
for i,v in pairs(data) do
raw = raw..urlencode(v)..(i%2==1 and "=" or "&")
end
return raw:sub(1, -2)
end
local nhbase = [[<!DOCTYPE html><html><head>
<meta charset="UTF-8">
<title>{TITLE} - aadenboy wiki</title>
<link rel="stylesheet" href="/wiki/main.css">
<script src="https://code.jquery.com/jquery-4.0.0.min.js"></script>
<meta name="ResourceLoaderDynamicStyles" content="">
<link rel="stylesheet" href="/wiki/sitestyles.css">
<meta name="generator" content="MediaWiki 1.43.0">
<meta name="robots" content="max-image-preview:standard">
<meta name="format-detection" content="telephone=no">
<link rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/">
</head><body></body></html>]]
function meta(document, name, property, content)
local m = document:createElement("meta")
if name then m:setAttribute("name", name) end
m:setAttribute("property", property)
m:setAttribute("content", content)
document:getElementsByTagName("head")[1]:appendChild(m)
end
function text(node)
if node.nodeType == 3 or node.data then return node.data end
local t = ""
for i,v in ipairs(node.childNodes) do
t = t..text(v)
end
return t
end
function fix(title, mode)
local file = io.open("localhost:8888/mediawiki/index.php/"..title..(mode or "")..".html", "r")
local page = file:read("*a")
file:close()
--[[
Changes:
* Replace the head
* Also the title
* But carry over any custom meta tags
* Change "/mediawiki/", "/mediawiki/index.php/", and "/mediawiki/index.php?title=" in URLs to "/wiki/"
* If the link contains "&action=history", replace it with the repository link
* Remove element with class "printfooter"
* Remove element with ID "p-personal"
* Remove element with ID "left-navigation"
* Change element with ID "right-navigation" to have the id "left-navigation" instead
* Change first child of element with ID "ca-viewsource" to have the title "View the source of this page for editing."
* Remove element with ID "p-cactions"
* Remove element with ID "p-search"
* Remove element with ID "p-tb"
* Replace the final script element with `<script async="" src="/common.js"></script>`
* Generate metadata
]]
local document = gumbo.parse(page)
local oldhead = document:getElementsByTagName("head")[1]
local newhead = gumbo.parse(nhbase):getElementsByTagName("head")[1]
oldhead.parentNode:replaceChild(newhead, oldhead)
local telem = oldhead:getElementsByTagName("title")[1]
newhead:replaceChild(telem, newhead:getElementsByTagName("title")[1]) -- lazy
local metas = document:getElementsByTagName("meta")
for i,v in ipairs(metas) do
if v:getAttribute("name") == "page-no-include" and v:getAttribute("content") == "yes" then
return false
end
end
-- no querySelector :(
local anchors = document:getElementsByTagName("a")
local images = document:getElementsByTagName("img")
for i,v in ipairs(anchors) do
v:setAttribute("href", (v:getAttribute("href") or "")
:gsub("^/mediawiki/index%.php%?title=", "/wiki/")
:gsub("^/mediawiki/index%.php/", "/wiki/")
:gsub("^/mediawiki/", "/wiki/")
:gsub("&action=edit", "%%3Faction=edit")
:gsub("^.*&action=history.*$", "https://github.com/aadenboy/wiki/commits/main/"..title..".html")
)
end
for i,v in ipairs(images) do
v:setAttribute("src", (v:getAttribute("src") or "")
:gsub("^/mediawiki/index%.php%?title=", "/wiki/")
:gsub("^/mediawiki/index%.php/", "/wiki/")
:gsub("^/mediawiki/", "/wiki/")
)
v:setAttribute("srcset", (v:getAttribute("srcset") or "")
:gsub("/mediawiki/index%.php%?title=", "/wiki/")
:gsub("/mediawiki/index%.php/", "/wiki/")
:gsub("/mediawiki/", "/wiki/")
)
end
document:getElementsByClassName("printfooter")[1]:remove()
document:getElementById("p-personal"):remove()
document:getElementById("left-navigation"):remove()
document:getElementById("right-navigation"):setAttribute("id", "left-navigation")
document:getElementById("ca-viewsource").childNodes[1]:setAttribute("title", "View the source of this page for editing.")
document:getElementById("p-search"):remove()
document:getElementById("p-cactions"):remove()
document:getElementById("p-tb"):remove()
local scripts = document:getElementsByTagName("script")
local commonscript = document:createElement("script")
commonscript:setAttribute("async", "")
commonscript:setAttribute("src", "/wiki/common.js")
if scripts[#scripts].parentNode.tagName == "BODY" then
scripts[#scripts].parentNode:replaceChild(commonscript, scripts[#scripts])
else
document.body:appendChild(commonscript)
end
--[[
reference
<meta property="og:site_name" content="Tecknar" />
<meta property="og:url" content="https://aadenboy.github.io/tecknar/" />
<meta property="og:type" content="website" />
<meta name="title" property="og:title" content="Tecknar" />
<meta name="image" property="og:image" content="https://aadenboy.github.io/tecknar/tecknar.png"/>
<meta property="og:image:type" content="image/png">
<meta property="og:image:width" content="906">
<meta property="og:image:height" content="413">
<meta name="description" property="og:description" content="A lightweight, embeddable drawing pad for the web." />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="Tecknar" />
<meta name="twitter:description" content="A lightweight, embeddable drawing pad for the web." />
<meta name="twitter:image:src" content="https://aadenboy.github.io/tecknar/tecknar.png">
<meta name="twitter:image" content="https://aadenboy.github.io/tecknar/tecknar.png">
<meta name="twitter:domain" content="https://aadenboy.github.io/tecknar/">
]]
meta(document, nil, "og:site_name", "Aadenboy Wiki")
meta(document, nil, "og:url", "https://aadenboy.github.io/wiki/"..urlencode(title))
meta(document, nil, "og:type", "website")
meta(document, "title", "og:title", telem.innerHTML:gsub(" %- .+$", ""))
local firstimage = document:getElementsByTagName("img")[1]
if firstimage then meta(document, "image", "og:image", "https://aadenboy.github.io"..firstimage:getAttribute("src")) end
local paragraphs = document:getElementsByTagName("p")
local description = ""
for i,v in pairs(paragraphs) do
if type(v) == "table" and not (v:getAttribute("class") or ""):match("mw%-empty%-elt") and v.parentNode.parentNode:getAttribute("id") == "mw-content-text" then
description = text(v)
break
end
end
description = description:gsub("%s*\n%s*", " "):gsub("^%s*(.-)%s*$", "%1")
meta(document, "description", "description", description)
meta(document, nil, "twitter:card", "summary_large_image")
meta(document, nil, "twitter:title", telem.innerHTML:gsub(" %- .+$", ""))
meta(document, nil, "twitter:description", description)
if firstimage then
meta(document, nil, "twitter:image:src", "https://aadenboy.github.io"..firstimage:getAttribute("src"))
meta(document, nil, "twitter:image", "https://aadenboy.github.io"..firstimage:getAttribute("src"))
end
meta(document, nil, "twitter:domain", "https://aadenboy.github.io/wiki/")
local final = document:serialize():gsub("<!%-%-.-%-%->", "") -- shrug
if title:match("/") then
os.execute("mkdir -p localhost:8888/mediawiki/"..title:gsub("/[^/]+$", "/"))
end
local outfile = io.open("localhost:8888/mediawiki/"..title..".html", "w")
outfile:write(final)
outfile:close()
return true
end
function apijson(action)
local call = io.popen('wget -q -o /dev/null --no-cache -O - "http://localhost:8888/mediawiki/api.php?'..params(action)..'"')
local data = json.decode(call:read("*a"))
call:close()
return data
end
local nsdata = apijson{
"action","query";
"format","json";
"meta","siteinfo";
"siprop","namespaces";
}
local sorted = {}
for i,v in pairs(nsdata.query.namespaces) do
if i+0 > -1 and (i+0)%2==0 and i ~= "2" and i ~= "3" then table.insert(sorted, tonumber(i)) end
end
table.sort(sorted)
for i,v in ipairs(sorted) do
print("Starting namespace "..nsdata.query.namespaces[v..""]["*"]:gsub("^$", "Main").." ("..i.." of "..#sorted..")")
local pages = apijson{
"action","query";
"format","json";
"list","allpages",
"aplimit",500;
"apnamespace",v;
}
print("\tQueried "..#pages.query.allpages.." pages")
local totalpages = pages.query.allpages
while pages.continue do
pages = apijson{
"action","query";
"format","json";
"list","allpages",
"aplimit",500;
"apnamespace",v;
"apcontinue",pages.continue.apcontinue;
"continue",pages.continue.continue;
}
for i,v in ipairs(pages.query.allpages) do
totalpages[#totalpages+1] = v
end
print("\tQueried "..#pages.query.allpages.." additional pages (now "..#totalpages.." total pages)")
end
for i,v in ipairs(totalpages) do
print("\tGenerating page "..v.title)
v.title = v.title:gsub(" ", "_")
local title = urlencode(v.title):gsub("%%2F", "/")
local call = io.popen('wget -E -H -p --no-cache --content-on-error -o - "http://localhost:8888/mediawiki/index.php/'..title..'?redirect=no" | grep "Saving to"')
--[[for f in call:read("*a"):gmatch('‘([^\n]+%?[0-9a-zA-Z]+)’') do
print("\t\tGenerated "..f:gsub("^.+/", "")..", moving")
os.execute("mv "..f.." "..f:gsub("%?[0-9a-zA-Z]+$", ""))
end]]
call:close()
local included = fix(v.title, "?redirect=no")
if not included then
print("\t\tPage skipped, not included")
else
print("\t\tBase article finished")
local call = io.popen('wget -E -H -p --no-cache --content-on-error -o - "http://localhost:8888/mediawiki/index.php/'..title..'?action=edit" | grep "Saving to"')
--[[for f in call:read("*a"):gmatch('‘([^\n]+%?[0-9a-zA-Z]+)’') do
print("\t\tGenerated "..f..", moving")
os.execute("mv "..f.." "..f:gsub("%?[0-9a-zA-Z]+$"))
end]]
call:close()
fix(v.title.."?action=edit")
print("\t\tEdit page finished")
end
end
end
-- http://localhost:8888/mediawiki/api.php?action=query&format=json&list=allpages&aplimit=10&apnamespace=6&apcontinue=CatAlignmentChart.jpeg&continue=-||
--[[local pages = {}
for page in pagesstr:gmatch("[^\n]+") do
local code = os.execute('wget -E -H -p --content-on-error "http://localhost:8888/mediawiki/index.php/'..urlencode(page)..'?redirect=no"')
fix(page, "?redirect=no")
local code = os.execute('wget -E -H -p --content-on-error "http://localhost:8888/mediawiki/index.php/'..urlencode(page)..'?action=edit"')
fix(page.."?action=edit")
end]]
I also use this Shell file, though your mileage may vary.
#!/bin/bash
export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
cd ~/THE/FILE/PATH # Obviously not what it is called; this is just an example.
printf "\n\n%s\n" "== Automatic update: $(date -u +'%H:%M:%S, %-d %B %Y (UTC)') ==" > console.out
lua fetcher.lua
cd ~/THE/FILE/PATH/localhost:8888/mediawiki
rm -rf index.php
cp Main_Page.html index.html
shopt -s globstar
for i in **/*/*.*\?*; do
mv "$i" "${i%%\?*}"
echo "Moved $i to ${i%%\?*}"
end
: > main.css
for i in load.php*.css; do
sed -E 's|http://localhost:8888/mediawiki/|/wiki/|g; s|/mediawiki/|/wiki/|g; s|\?[0-9a-zA-Z]+||g' $i >> main.css
done
if [[ -n $(git status -s) ]]; then
echo "Committing"
git add .
git commit -m "Automatic update: $(date -u +'%H:%M:%S, %-d %B %Y (UTC)')"
git push -u origin main
echo "Pushed"
fi