diff --git a/bin/add-edb-ids b/bin/add-edb-ids index 84ccfa4..cbb1f07 100755 --- a/bin/add-edb-ids +++ b/bin/add-edb-ids @@ -47,8 +47,7 @@ const pages = (await Promise.all(Object.entries(categoryTypes).map(async ([categ console.log('Processing', pages.length, 'items\n'); -for (const page of pages) { - const {title, type} = page; +for (const {title, type} of pages) { // this runs serially with an artificial delay between requests to decrease // the chance of sqenix sending ninjas to my house await new Promise(resolve => setTimeout(resolve, 1000)); @@ -74,7 +73,7 @@ for (const page of pages) { let updatedText; try { updatedText = insertInfoboxEDBID(originalText, edbID); - diff(originalText, updatedText, page); + diff(originalText, updatedText); } catch (error) { console.log('Error inserting parameter:', error); console.group('Bad page content:'); diff --git a/bin/add-gt-ids b/bin/add-gt-ids index f450704..f6988be 100755 --- a/bin/add-gt-ids +++ b/bin/add-gt-ids @@ -1,5 +1,8 @@ #!/usr/bin/env -S node --env-file-if-exists=.env +console.log(process.env); +process.exit(1); + import {findItemGTID} from '../lib/api/xivapi.js'; import {getMediawikiClient} from '../lib/config.js'; import {diff} from '../lib/util/diff.js'; @@ -35,11 +38,10 @@ const mw = await getMediawikiClient(); // TODO: update this to work with the new maintenance category hierarchy // Get pages in the "Missing internal ID" category from the main article namespace -const itemPagesWithoutGTIDs = await mw.listCategoryPages('Category:Items with no internal ID specified', [0], +process.env.LIMIT || 500); -console.log('Processing', itemPagesWithoutGTIDs.length, 'item pages from [[Category:Items with no internal ID specified]]\n'); +const itemPagesWithoutGTIDs = await mw.listCategoryPages('Category:Missing internal ID', [0], +process.env.LIMIT || 500); +console.log('Processing', itemPagesWithoutGTIDs.length, 'item pages from [[Category:Missing internal ID]]\n'); -for (const page of itemPagesWithoutGTIDs) { - const {title} = page; +for (const {title} of itemPagesWithoutGTIDs) { console.log('Page:', title); // look up on XIVAPI let gtID; @@ -66,7 +68,7 @@ for (const page of itemPagesWithoutGTIDs) { let updatedText; try { updatedText = insertInfoboxGTBID(originalText, gtID); - diff(originalText, updatedText, page); + diff(originalText, updatedText); } catch (error) { console.log('Error inserting parameter:', error); console.group('Bad page content:'); diff --git a/bin/one-time/add-quest-sync-info b/bin/one-time/add-quest-sync-info deleted file mode 100755 index bf63da9..0000000 --- a/bin/one-time/add-quest-sync-info +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env -S node --env-file-if-exists=.env -// Adds `quest-sync` parameters to quest infoboxes that don't have them - -import { findEDBQuestSync } from '../../lib/api/lodestone.js'; -import {getMediawikiClient} from '../../lib/config.js'; -import {diff} from '../../lib/util/diff.js'; -import { addParameterBesideExistingParameter } from '../../lib/util/template-parameters.js'; - -const mw = await getMediawikiClient(); - -const allquestcats = ` -Category:Main Scenario quests -Category:Side Story quests -Category:Guildleves -Category:Sidequests -Category:other quests -Category:Daily quests -Category:Feature quests -Category:Repeatable Feature quests -Category:Quasi-quests -Category:Feature quasi-quests -Category:Quests with no type specified -`.split('\n').filter(s => s); -console.log(allquestcats) - -const pages = (await Promise.all(allquestcats.map(category => mw.listCategoryPages(category, [0], 'max')))).flat(); - -console.log(pages.length); - -function insertQuestSync (pageContent) { - if (pageContent.includes('quest-sync')) { - throw new Error('Page already contains a `quest-sync` parameter'); - } - let result = addParameterBesideExistingParameter(pageContent, 'quest-sync', 'true', 'after', 'level'); - if (result == null) throw new Error('Failed to add parameter'); - return result; -} - -const start = parseInt(process.env.START ?? '0', 10); -if (start) { - console.log('Starting from item index', start); - pages.splice(0, start); -} - -for (const [i, {title}] of Object.entries(pages)) { - console.log(start+parseInt(i), title); - const content = await mw.readPage(title); - const hasQuestSync = await findEDBQuestSync(title.replace(' (Quest)', '')); - if (!hasQuestSync) continue; - let newContent; - try { - newContent = insertQuestSync(content); - } catch (error) { - console.error(error); - continue; - } - diff(content, newContent); - - await mw.editPage(title, newContent, 'Add quest-sync parameter', true); -} diff --git a/bin/one-time/flavor-text-color-fix b/bin/one-time/flavor-text-color-fix deleted file mode 100755 index 984fa3d..0000000 --- a/bin/one-time/flavor-text-color-fix +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env -S node --env-file-if-exists=.env -// Fixes flavor text formatting on files robogurgum created for patches 7.3/7.31 -// see https://discord.com/channels/312060255469305856/1412850908668100810 - -import {getMediawikiClient} from '../../lib/config.js'; -import {diff} from '../../lib/util/diff.js'; - -const mw = await getMediawikiClient(); - -function fixPageContent (pageContent) { - const flavorTextRegexp = /\| flavor-text = ([\s\S]*?)\n\|/; - - let match = pageContent.match(flavorTextRegexp); - if (!match) return pageContent; - let flavorText = match[1]; - if (!flavorText.trim()) return pageContent; - - let newFlavorText = flavorText - // linebreaks with - // - no
or
etc before or after - // - no {{action fact}} after (that renders a line break at the start) - .replace(/(?)\n(?!)(?!{{action fact|\s*$)/gi, '
\n') - // spans that set color via inline style instead of using {{colorize}} - .replace(/([^<]+)<\/span>/g, '`$1`') - .replace(/([^<]+)<\/span>/g, '``$1``') - .replace(/([^<]+)<\/span>/g, '```$1```') - // convert lunar/phaenna credit bonuses to use {{action fact}}, but only - // when we can be sure it won't add extra line breaks - .replace( - /(\n?)`([\w\d _-]+)(?<=Credit Bonus)(?::`|`:) +([^\n<]+)/g, - '$1{{action fact|$2|$3}}', - ) - // flavor-text is automatically colorized, remove redundant invocations - .replace(/{{colorize\|([^|{}]+)}}/g, '$1'); - - return pageContent.replace(flavorTextRegexp, `| flavor-text = ${newFlavorText}\n|`); -} - -const pages = await mw.listUserContribs('RoboGurgum', {namespaces: [0], show: 'new', limit: 1000}); -for (const page of pages) { - const {title} = page; - const pageContent = await mw.readPage(title); - let fixedContent = fixPageContent(pageContent); - if (fixedContent === pageContent) { - console.log('#', title, ' No change\n'); - continue; - } - diff(pageContent, fixedContent, page); - console.log(); - - await mw.editPage(title, fixedContent, 'Fix flavor text formatting', true); -} diff --git a/lib/api/mediawiki.js b/lib/api/mediawiki.js index 2c3d017..5dc2b7b 100644 --- a/lib/api/mediawiki.js +++ b/lib/api/mediawiki.js @@ -27,26 +27,9 @@ export class MediaWikiClient { constructor (wikiURL, {killPage}) { this.wikiURL = wikiURL; this.killPage = killPage; - this.lastKillPageCheck = 0; // first check is triggered after login this.fetch = makeFetchCookie(fetch); } - /** Checks the kill page if needed, killing the process if not empty. */ - async tryKillPageCheck () { - if (!this.killPage) return; - // wait at least 10 seconds since the last check - if (Date.now() > this.lastKillPageCheck + 10 * 1000) return; - this.lastKillPageCheck = Date.now(); - - const response = await this.fetch(`${this.wikiURL}/index.php?action=raw&title=${encodeURIComponent(this.killPage)}`); - const content = await response.text(); - if (content.trim()) { - console.error('*** Kill page is not empty; stopping ***\n'); - console.error(content); - process.exit(1); - } - }; - /** * Makes a GET request against `index.php`. * @param {Record} params Query string parameters @@ -88,7 +71,6 @@ export class MediaWikiClient { * @returns {Promise} */ async fetchApiPost (params, options = {}) { - await this.tryKillPageCheck(); const response = await this.fetch(`${this.wikiURL}/api.php`, { ...options, method: 'POST', @@ -146,6 +128,20 @@ export class MediaWikiClient { if (body.login.result === 'Failed') { throw new Error(body.login.reason); } + + if (this.killPage) { + // start the kill page check loop as soon as we're logged in + const checkKillPage = async () => { + let content = await this.readPage(this.killPage); + if (content.trim()) { + console.error('*** Kill page is not empty; stopping ***\n'); + console.error(content); + process.exit(1); + } + setTimeout(checkKillPage, 30 * 1000); // every 30 seconds + }; + checkKillPage(); + } } /** @@ -243,28 +239,4 @@ export class MediaWikiClient { }); return body.query.categorymembers; } - - /** - * Gets the list of a user's contributions. - * @param {string} username Name of the user whose contribs should be fetched - * @param {object} options - * @param {number | number[] | '*'} [options.namespaces] List of namespaces from which to return results - * @param {number} [options.limit] Maximum number of items to return - * @param {string} [options.show] See the documentation of `ucshow` at https://www.mediawiki.org/wiki/API:Usercontribs - * @returns {Promise<{pageid: string; revid: string; timestamp: string; title: string}[]>} - */ - async listUserContribs (username, {namespaces = '*', limit = 50, show}) { - if (Array.isArray(namespaces)) { - namespaces = namespaces.join('|'); - } - const body = await this.fetchApiGet({ - action: 'query', - list: 'usercontribs', - ucuser: username, - uclimit: limit, - ucnamespace: namespaces, - ucshow: show, - }); - return body.query.usercontribs; - } } diff --git a/lib/util/diff.js b/lib/util/diff.js index eaa2fc3..d2e2991 100644 --- a/lib/util/diff.js +++ b/lib/util/diff.js @@ -13,26 +13,17 @@ const toBase64 = str => Buffer.from(str, 'utf-8').toString('base64'); * terrible terrible terrible string diff helper for debugging * @param {string} a * @param {string} b - * @param {object} page Page object from the API, used to generate diff labels */ -export function diff (a, b, page) { - let labels = null; - if (page) { // generate diff labels based on - labels = [ - `${page.title} (${page.revid ? `revision ${page.revid}` : 'current revision'}${page.timestamp ? ` from ${page.timestamp}` : ''})`, - `${page.title} (pending edit)`, - ]; - } +export function diff (a, b) { // base64 input strings before passing to shell to avoid escaping issues // https://stackoverflow.com/a/60221847 - // use tail to cut off file info lines and re-add with fake filenames/dates - // this function is so extra now holy shit - execSync(String.raw`bash <<- '_EOF_' + // use tail to cut off useless file info lines + execSync(String.raw`bash <<- EOF diff --color=always -u \ - ${labels ? `--label="$(echo '${toBase64(labels[0])}' | base64 -d)" ` : ''}<(echo '${toBase64(a)}' | base64 -d) \ - ${labels ? `--label="$(echo '${toBase64(labels[1])}' | base64 -d)" ` : ''}<(echo '${toBase64(b)}' | base64 -d) \ - ${labels ? `|| true` : `| tail -n +3` /* cut off header if no labels */} - _EOF_`, { + <(echo "${toBase64(a)}" | base64 -d) \ + <(echo "${toBase64(b)}" | base64 -d) \ + | tail -n +3 + EOF`, { // display result directly in terminal stdio: 'inherit', }); diff --git a/temp.js b/temp.js new file mode 100644 index 0000000..56f0961 --- /dev/null +++ b/temp.js @@ -0,0 +1,19 @@ +const flavorTextRegexp = /\| flavor-text = ([\s\S]+?)\n\|/; +const flavorTextReplacement = newFlavorText => `| flavor-text = ${newFlavorText}\n|` + +function fixPageContent (pageContent) { + let match = pageContent.match(flavorTextRegexp); + if (!match) return pageContent; + + let flavorText = match[1]; + let newFlavorText = flavorText + // linebreaks with no preceding
or
etc + .replace(/(?)\n/g, '
\n') + // spans that set color via inline style instead of using {{colorize}} + .replace(/([^<]+)<\/span>/g, '`$1`') + .replace(/([^<]+)<\/span>/g, '``$1``') + + return pageContent.replace(flavorTextRegexp, flavorTextReplacement(newFlavorText)); +} + +const pages = //...