From 855c1571f9287b6bfab9ab8e085d5f2cfb8491d5 Mon Sep 17 00:00:00 2001 From: Erin Date: Thu, 4 Sep 2025 18:29:32 -0400 Subject: [PATCH 01/11] asdf --- temp.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 temp.js diff --git a/temp.js b/temp.js new file mode 100644 index 0000000..56f0961 --- /dev/null +++ b/temp.js @@ -0,0 +1,19 @@ +const flavorTextRegexp = /\| flavor-text = ([\s\S]+?)\n\|/; +const flavorTextReplacement = newFlavorText => `| flavor-text = ${newFlavorText}\n|` + +function fixPageContent (pageContent) { + let match = pageContent.match(flavorTextRegexp); + if (!match) return pageContent; + + let flavorText = match[1]; + let newFlavorText = flavorText + // linebreaks with no preceding
or
etc + .replace(/(?)\n/g, '
\n') + // spans that set color via inline style instead of using {{colorize}} + .replace(/([^<]+)<\/span>/g, '`$1`') + .replace(/([^<]+)<\/span>/g, '``$1``') + + return pageContent.replace(flavorTextRegexp, flavorTextReplacement(newFlavorText)); +} + +const pages = //... From d0e5bf33ffbf35e02b188f9ef54b9c1af0937bbc Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 22:41:22 -0400 Subject: [PATCH 02/11] uhh i have no idea how this happened --- bin/add-gt-ids | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/add-gt-ids b/bin/add-gt-ids index f6988be..458f3c9 100755 --- a/bin/add-gt-ids +++ b/bin/add-gt-ids @@ -1,8 +1,5 @@ #!/usr/bin/env -S node --env-file-if-exists=.env -console.log(process.env); -process.exit(1); - import {findItemGTID} from '../lib/api/xivapi.js'; import {getMediawikiClient} from '../lib/config.js'; import {diff} from '../lib/util/diff.js'; From 436acf72990ccdb73babd45831ff334773cfc8b7 Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:04:54 -0400 Subject: [PATCH 03/11] improve killpage logic relying on an interval causes the process to never exit, because the interval is still running in the background. this is bad for oneshot scripts like ours. instead the check works by comparing the current date to the last known date any time a post request is sent --- lib/api/mediawiki.js | 69 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/lib/api/mediawiki.js b/lib/api/mediawiki.js index 5dc2b7b..2af925b 100644 --- a/lib/api/mediawiki.js +++ b/lib/api/mediawiki.js @@ -27,9 +27,26 @@ export class MediaWikiClient { constructor (wikiURL, {killPage}) { this.wikiURL = wikiURL; this.killPage = killPage; + this.lastKillPageCheck = 0; // first check is triggered after login this.fetch = makeFetchCookie(fetch); } + /** Checks the kill page if needed, killing the process if not empty. */ + async tryKillPageCheck () { + if (!this.killPage) return; + // wait at least 10 seconds since the last check + if (Date.now() > this.lastKillPageCheck + 10 * 1000) return; + this.lastKillPageCheck = Date.now(); + + const response = await this.fetch(`${this.wikiURL}/index.php?action=raw&title=${encodeURIComponent(this.killPage)}`); + const content = await response.text(); + if (content.trim()) { + console.error('*** Kill page is not empty; stopping ***\n'); + console.error(content); + process.exit(1); + } + }; + /** * Makes a GET request against `index.php`. * @param {Record} params Query string parameters @@ -71,6 +88,7 @@ export class MediaWikiClient { * @returns {Promise} */ async fetchApiPost (params, options = {}) { + await this.tryKillPageCheck(); const response = await this.fetch(`${this.wikiURL}/api.php`, { ...options, method: 'POST', @@ -128,20 +146,6 @@ export class MediaWikiClient { if (body.login.result === 'Failed') { throw new Error(body.login.reason); } - - if (this.killPage) { - // start the kill page check loop as soon as we're logged in - const checkKillPage = async () => { - let content = await this.readPage(this.killPage); - if (content.trim()) { - console.error('*** Kill page is not empty; stopping ***\n'); - console.error(content); - process.exit(1); - } - setTimeout(checkKillPage, 30 * 1000); // every 30 seconds - }; - checkKillPage(); - } } /** @@ -182,6 +186,19 @@ export class MediaWikiClient { return body; } + async purgePages (titles) { + if (!titles.length) return; + + // mediawiki has a 50 title per request limit, so we grab the first 50 + // and recurse to handle the rest + let currentTitles = titles.splice(0, 50); + const body = await this.fetchApiPost({ + action: 'purge', + titles: currentTitles.join('|'), + }); + return this.purgePages(titles); + } + /** * * @param {string} from The page's current name @@ -239,4 +256,28 @@ export class MediaWikiClient { }); return body.query.categorymembers; } + + /** + * Gets the list of a user's contributions. + * @param {string} username Name of the user whose contribs should be fetched + * @param {object} options + * @param {number | number[] | '*'} [options.namespaces] List of namespaces from which to return results + * @param {number} [options.limit] Maximum number of items to return + * @param {string} [options.show] See the documentation of `ucshow` at https://www.mediawiki.org/wiki/API:Usercontribs + * @returns {Promise<{pageid: string; revid: string; timestamp: string; title: string}[]>} + */ + async listUserContribs (username, {namespaces = '*', limit = 50, show}) { + if (Array.isArray(namespaces)) { + namespaces = namespaces.join('|'); + } + const body = await this.fetchApiGet({ + action: 'query', + list: 'usercontribs', + ucuser: username, + uclimit: limit, + ucnamespace: namespaces, + ucshow: show, + }); + return body.query.usercontribs; + } } From 4f17754f4ca2995a0ec119df8175e328f2ecefd5 Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:05:54 -0400 Subject: [PATCH 04/11] allow custom diff header labels --- lib/util/diff.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/util/diff.js b/lib/util/diff.js index d2e2991..c30fa7a 100644 --- a/lib/util/diff.js +++ b/lib/util/diff.js @@ -13,16 +13,25 @@ const toBase64 = str => Buffer.from(str, 'utf-8').toString('base64'); * terrible terrible terrible string diff helper for debugging * @param {string} a * @param {string} b + * @param {object} page Page object from the API, used to generate diff labels */ -export function diff (a, b) { +export function diff (a, b, page) { + let labels = null; + if (page) { // generate diff labels based on + labels = [ + `${page.title} (${page.revid ? `revision ${page.revid}` : 'current revision'}${page.timestamp ? ` from ${page.timestamp}` : ''})`, + `${page.title} (pending edit)`, + ]; + } // base64 input strings before passing to shell to avoid escaping issues // https://stackoverflow.com/a/60221847 - // use tail to cut off useless file info lines - execSync(String.raw`bash <<- EOF + // use tail to cut off file info lines and re-add with fake filenames/dates + // this function is so extra now holy shit + execSync(String.raw`bash <<- 'EOF' diff --color=always -u \ - <(echo "${toBase64(a)}" | base64 -d) \ - <(echo "${toBase64(b)}" | base64 -d) \ - | tail -n +3 + ${labels ? `--label="$(echo '${toBase64(labels[0])}' | base64 -d)" ` : ''}<(echo '${toBase64(a)}' | base64 -d) \ + ${labels ? `--label="$(echo '${toBase64(labels[1])}' | base64 -d)" ` : ''}<(echo '${toBase64(b)}' | base64 -d) \ + ${labels ? `|| true` : `| tail -n +3` /* cut off header if no labels */} EOF`, { // display result directly in terminal stdio: 'inherit', From ded35a743daaf05dd46e9c93b769716fa6520fb3 Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:07:04 -0400 Subject: [PATCH 05/11] fix potential base64 injection "EOF" is made up of entirely valid base64 characters, it would be bad if the program crashed because we tried to diff a file whose base64 representation contained the string EOF. underscores don't appear in base64 strings --- lib/util/diff.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/util/diff.js b/lib/util/diff.js index c30fa7a..eaa2fc3 100644 --- a/lib/util/diff.js +++ b/lib/util/diff.js @@ -27,12 +27,12 @@ export function diff (a, b, page) { // https://stackoverflow.com/a/60221847 // use tail to cut off file info lines and re-add with fake filenames/dates // this function is so extra now holy shit - execSync(String.raw`bash <<- 'EOF' + execSync(String.raw`bash <<- '_EOF_' diff --color=always -u \ ${labels ? `--label="$(echo '${toBase64(labels[0])}' | base64 -d)" ` : ''}<(echo '${toBase64(a)}' | base64 -d) \ ${labels ? `--label="$(echo '${toBase64(labels[1])}' | base64 -d)" ` : ''}<(echo '${toBase64(b)}' | base64 -d) \ ${labels ? `|| true` : `| tail -n +3` /* cut off header if no labels */} - EOF`, { + _EOF_`, { // display result directly in terminal stdio: 'inherit', }); From ef69ee4c9f168b1ca315a6f27c38b42fddeb88d3 Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:07:48 -0400 Subject: [PATCH 06/11] use new fancy logging in edb and gt scripts --- bin/add-edb-ids | 5 +++-- bin/add-gt-ids | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/add-edb-ids b/bin/add-edb-ids index cbb1f07..84ccfa4 100755 --- a/bin/add-edb-ids +++ b/bin/add-edb-ids @@ -47,7 +47,8 @@ const pages = (await Promise.all(Object.entries(categoryTypes).map(async ([categ console.log('Processing', pages.length, 'items\n'); -for (const {title, type} of pages) { +for (const page of pages) { + const {title, type} = page; // this runs serially with an artificial delay between requests to decrease // the chance of sqenix sending ninjas to my house await new Promise(resolve => setTimeout(resolve, 1000)); @@ -73,7 +74,7 @@ for (const {title, type} of pages) { let updatedText; try { updatedText = insertInfoboxEDBID(originalText, edbID); - diff(originalText, updatedText); + diff(originalText, updatedText, page); } catch (error) { console.log('Error inserting parameter:', error); console.group('Bad page content:'); diff --git a/bin/add-gt-ids b/bin/add-gt-ids index 458f3c9..d6817a4 100755 --- a/bin/add-gt-ids +++ b/bin/add-gt-ids @@ -38,7 +38,8 @@ const mw = await getMediawikiClient(); const itemPagesWithoutGTIDs = await mw.listCategoryPages('Category:Missing internal ID', [0], +process.env.LIMIT || 500); console.log('Processing', itemPagesWithoutGTIDs.length, 'item pages from [[Category:Missing internal ID]]\n'); -for (const {title} of itemPagesWithoutGTIDs) { +for (const page of itemPagesWithoutGTIDs) { + const {title} = page; console.log('Page:', title); // look up on XIVAPI let gtID; @@ -65,7 +66,7 @@ for (const {title} of itemPagesWithoutGTIDs) { let updatedText; try { updatedText = insertInfoboxGTBID(originalText, gtID); - diff(originalText, updatedText); + diff(originalText, updatedText, page); } catch (error) { console.log('Error inserting parameter:', error); console.group('Bad page content:'); From 6804184cf10c69709a405b130a540f37e8f13148 Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:08:02 -0400 Subject: [PATCH 07/11] update items without GT ID category name --- bin/add-gt-ids | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/add-gt-ids b/bin/add-gt-ids index d6817a4..f450704 100755 --- a/bin/add-gt-ids +++ b/bin/add-gt-ids @@ -35,8 +35,8 @@ const mw = await getMediawikiClient(); // TODO: update this to work with the new maintenance category hierarchy // Get pages in the "Missing internal ID" category from the main article namespace -const itemPagesWithoutGTIDs = await mw.listCategoryPages('Category:Missing internal ID', [0], +process.env.LIMIT || 500); -console.log('Processing', itemPagesWithoutGTIDs.length, 'item pages from [[Category:Missing internal ID]]\n'); +const itemPagesWithoutGTIDs = await mw.listCategoryPages('Category:Items with no internal ID specified', [0], +process.env.LIMIT || 500); +console.log('Processing', itemPagesWithoutGTIDs.length, 'item pages from [[Category:Items with no internal ID specified]]\n'); for (const page of itemPagesWithoutGTIDs) { const {title} = page; From a8ad47f830ffd487ca6cf9f4e4f4e92d2c7c858d Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:08:53 -0400 Subject: [PATCH 08/11] whoops i didnt want to commit that yet lol --- lib/api/mediawiki.js | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/api/mediawiki.js b/lib/api/mediawiki.js index 2af925b..2c3d017 100644 --- a/lib/api/mediawiki.js +++ b/lib/api/mediawiki.js @@ -186,19 +186,6 @@ export class MediaWikiClient { return body; } - async purgePages (titles) { - if (!titles.length) return; - - // mediawiki has a 50 title per request limit, so we grab the first 50 - // and recurse to handle the rest - let currentTitles = titles.splice(0, 50); - const body = await this.fetchApiPost({ - action: 'purge', - titles: currentTitles.join('|'), - }); - return this.purgePages(titles); - } - /** * * @param {string} from The page's current name From 80bf3193787ada9c3272671d3cab4ee4e6d4917e Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:55:43 -0400 Subject: [PATCH 09/11] add script for fixing flavor text formatting from robogurgum see --- bin/one-time/flavor-text-color-fix | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 bin/one-time/flavor-text-color-fix diff --git a/bin/one-time/flavor-text-color-fix b/bin/one-time/flavor-text-color-fix new file mode 100755 index 0000000..898691a --- /dev/null +++ b/bin/one-time/flavor-text-color-fix @@ -0,0 +1,53 @@ +#!/usr/bin/env -S node --env-file-if-exists=.env +// Fixes flavor text formatting on files robogurgum created for patches 7.3/7.31 +// see https://discord.com/channels/312060255469305856/1412850908668100810 + +import {getMediawikiClient} from '../../lib/config.js'; +import {diff} from '../../lib/util/diff.js'; + +const mw = await getMediawikiClient(); + +function fixPageContent (pageContent) { + const flavorTextRegexp = /\| flavor-text = ([\s\S]*?)\n\|/; + + let match = pageContent.match(flavorTextRegexp); + if (!match) return pageContent; + let flavorText = match[1]; + if (!flavorText.trim()) return pageContent; + + let newFlavorText = flavorText + // linebreaks with + // - no
or
etc before or after + // - no {{action fact}} after (that renders a line break at the start) + .replace(/(?)\n(?!)(?!{{action fact|\s*$)/gi, '
\n') + // spans that set color via inline style instead of using {{colorize}} + .replace(/([^<]+)<\/span>/g, '`$1`') + .replace(/([^<]+)<\/span>/g, '``$1``') + .replace(/([^<]+)<\/span>/g, '```$1```') + // convert lunar/phaenna credit bonuses to use {{action fact}}, but only + // when we can be sure it won't add extra line breaks + .replace( + /(\n?)`([\w\d _-]+)(?<=Credit Bonus)(?::`|`:) +([^\n<]+)/g, + '$1{{action fact|$2|$3}}', + ) + // flavor-text is automatically colorized, remove redundant invocations + .replace(/{{colorize\|([^|{}]+)}}/g, '$1'); + + return pageContent.replace(flavorTextRegexp, `| flavor-text = ${newFlavorText}\n|`); +} + +const pages = await mw.listUserContribs('RoboGurgum', {namespaces: [0], show: 'new', limit: 1000}); +for (const page of pages) { + const {title} = page; + const pageContent = await mw.readPage(title); + let fixedContent = fixPageContent(pageContent); + if (fixedContent === pageContent) { + console.log('#', title, ' No change\n'); + continue; + } + console.write('\n'); + diff(pageContent, fixedContent, page); + console.log(); + + await mw.editPage(title, fixedContent, 'fix flavor text formatting', true); +} From ecedf23c76af510c05a2a39aa96ac4e0fe8c4dac Mon Sep 17 00:00:00 2001 From: ewin Date: Thu, 4 Sep 2025 23:57:33 -0400 Subject: [PATCH 10/11] this bug was my girlfriend's fault --- bin/one-time/flavor-text-color-fix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/one-time/flavor-text-color-fix b/bin/one-time/flavor-text-color-fix index 898691a..984fa3d 100755 --- a/bin/one-time/flavor-text-color-fix +++ b/bin/one-time/flavor-text-color-fix @@ -45,9 +45,8 @@ for (const page of pages) { console.log('#', title, ' No change\n'); continue; } - console.write('\n'); diff(pageContent, fixedContent, page); console.log(); - await mw.editPage(title, fixedContent, 'fix flavor text formatting', true); + await mw.editPage(title, fixedContent, 'Fix flavor text formatting', true); } From 53d48912d02c99509216c9f73ddda7cceaf4c360 Mon Sep 17 00:00:00 2001 From: ewin Date: Wed, 24 Sep 2025 06:41:38 -0400 Subject: [PATCH 11/11] add stupid shit i just ran --- bin/one-time/add-quest-sync-info | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 bin/one-time/add-quest-sync-info diff --git a/bin/one-time/add-quest-sync-info b/bin/one-time/add-quest-sync-info new file mode 100755 index 0000000..bf63da9 --- /dev/null +++ b/bin/one-time/add-quest-sync-info @@ -0,0 +1,60 @@ +#!/usr/bin/env -S node --env-file-if-exists=.env +// Adds `quest-sync` parameters to quest infoboxes that don't have them + +import { findEDBQuestSync } from '../../lib/api/lodestone.js'; +import {getMediawikiClient} from '../../lib/config.js'; +import {diff} from '../../lib/util/diff.js'; +import { addParameterBesideExistingParameter } from '../../lib/util/template-parameters.js'; + +const mw = await getMediawikiClient(); + +const allquestcats = ` +Category:Main Scenario quests +Category:Side Story quests +Category:Guildleves +Category:Sidequests +Category:other quests +Category:Daily quests +Category:Feature quests +Category:Repeatable Feature quests +Category:Quasi-quests +Category:Feature quasi-quests +Category:Quests with no type specified +`.split('\n').filter(s => s); +console.log(allquestcats) + +const pages = (await Promise.all(allquestcats.map(category => mw.listCategoryPages(category, [0], 'max')))).flat(); + +console.log(pages.length); + +function insertQuestSync (pageContent) { + if (pageContent.includes('quest-sync')) { + throw new Error('Page already contains a `quest-sync` parameter'); + } + let result = addParameterBesideExistingParameter(pageContent, 'quest-sync', 'true', 'after', 'level'); + if (result == null) throw new Error('Failed to add parameter'); + return result; +} + +const start = parseInt(process.env.START ?? '0', 10); +if (start) { + console.log('Starting from item index', start); + pages.splice(0, start); +} + +for (const [i, {title}] of Object.entries(pages)) { + console.log(start+parseInt(i), title); + const content = await mw.readPage(title); + const hasQuestSync = await findEDBQuestSync(title.replace(' (Quest)', '')); + if (!hasQuestSync) continue; + let newContent; + try { + newContent = insertQuestSync(content); + } catch (error) { + console.error(error); + continue; + } + diff(content, newContent); + + await mw.editPage(title, newContent, 'Add quest-sync parameter', true); +}