/**
* Copyright (c) Moodle Pty Ltd.
*
* Moodle is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Moodle is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Moodle. If not, see .
*/
/* eslint-disable import/no-extraneous-dependencies */
const Bot = require('nodemw'); // cspell:ignore nodemw
const path = require('path');
const winston = require('winston');
const yaml = require('js-yaml');
const { readFile, writeFile } = require('fs/promises');
/**
* Get the fetchDoc function with a logger and client applied.
*
* @param {winston} logger
* @param {Bot} client
* @param {string} pageTitle
* @returns {Promise}
*/
const getFetchDoc = (logger) => (client) => (pageTitle) => new Promise((resolve, reject) => {
logger.debug(`Fetching article with title ${pageTitle}`);
client.getArticle(pageTitle, (err, data) => {
if (err) {
reject(err);
}
resolve({
pageTitle,
data,
});
});
});
const getGetPagesByPrefix = (logger) => (client) => (prefix) => new Promise((resolve, reject) => {
logger.debug(`Fetching all articles with prefix ${prefix}`);
client.getPagesByPrefix(prefix, (err, data) => {
if (err) {
reject(err);
}
resolve(data);
});
});
const getGetMigratedPageIds = (logger) => (client) => async (pageTitle) => {
const articleData = await getFetchDoc(logger)(client)(pageTitle);
const docRegexp = /\{\{Template:Migrated\|newDocId=(?[^}]*)\}\}/g;
articleData.newDocIds = Array.from(articleData.data.matchAll(docRegexp))
.map((result) => result.groups.newDocId)
.filter((result) => !!result);
return articleData;
};
const getGetPageProtectionState = (logger) => (client) => async (pageTitle) => {
logger.debug(`===> Fetching protection state for ${pageTitle}`);
const options = {
inprop: [
'protection',
],
};
return new Promise((resolve, reject) => {
client.getArticleInfo([pageTitle], options, (err, [data]) => {
if (err) {
reject(err);
}
resolve({
pageTitle,
data,
});
});
});
};
const getProtectPage = (logger) => (client) => async (pageTitle, protections, options) => {
logger.debug(`===> Setting protection state for ${pageTitle} to ${protections}`);
return new Promise((resolve, reject) => {
client.protect(pageTitle, protections, options, (err, data) => {
if (err) {
reject(err);
}
resolve({
pageTitle,
data,
});
});
});
};
const getClient = (remoteHost) => new Bot({
protocol: 'https',
server: remoteHost,
debug: !!process.env.WIKIMEDIA_DEBUG,
username: process.env.WIKIMEDIA_USER,
password: process.env.WIKIMEDIA_SECRET,
userAgent: 'MoodleBot',
});
const getLogger = () => winston.createLogger({
level: 'info',
transports: [
new winston.transports.Console({
format: winston.format.simple(),
}),
],
});
/**
* Get all pages which transclude the named template.
*
* @param {String} templateName
* @returns {String[]}
*/
const getGetPagesTranscluding = (logger) => (client) => (templateName) => new Promise((resolve, reject) => {
logger.info(`Fetching all pages which transclude ${templateName}`);
client.getPagesTranscluding(templateName, (err, data) => {
if (err) {
reject(new Error(err));
}
resolve(data.map((pageData) => pageData.title));
});
});
const getGetImageInfo = (logger) => (client) => (filename) => new Promise((resolve, reject) => {
logger.info(`Fetching image info for ${filename}`);
client.getImageInfo(filename, (err, data) => {
if (err) {
reject(new Error(err));
}
resolve(data);
});
});
const getGetImagesFromArticle = (logger) => (client) => (title) => new Promise((resolve, reject) => {
logger.info(`Fetching images forpage ${title}`);
client.getImagesFromArticle(title, (err, data) => {
if (err) {
reject(new Error(err));
}
resolve(data);
});
});
const getLogIn = (logger) => (client) => async () => new Promise((resolve, reject) => {
if (!process.env.WIKIMEDIA_SECRET) {
reject(new Error('No credentials to log in with'));
return;
}
client.logIn((err, data) => {
if (err) {
reject(err);
}
if (data.result !== 'Success') {
reject(new Error('Unknown log in failure'));
}
client.whoami((whoAmIErr, whoAmI) => {
if (whoAmIErr) {
reject(new Error(whoAmIErr));
}
logger.warn(`Logged in as ${whoAmI.name}`);
resolve(data);
});
});
});
const getNormalizedPath = (...subPath) => path.normalize(path.join(__dirname, '../', ...subPath));
/**
* Get the path used to store obsolete page data.
*
* @returns {String}
*/
const getObsoletePagePath = () => getNormalizedPath('data', 'obsoletePages.json');
/**
* Get the path used to store migrated page data.
*
* @returns {String}
*/
const getMigrationPagePath = () => getNormalizedPath('data', 'migratedPages.yml');
const getUpdateMigratedPagesProtection = (logger) => (client) => async () => {
const migratedPageData = yaml.load(await readFile(getMigrationPagePath(), 'utf8'));
const getPageProtectionState = (pageTitle) => getGetPageProtectionState(logger)(client)(pageTitle);
const protectPage = (pageTitle, protections, reason) => getProtectPage(logger)(client)(
pageTitle,
protections,
{
// This is a bot. It does not need to subscribe.
watchlist: 'nochange',
reason,
},
);
/**
* Get the intended protection for the current page, modifying the 'edit' level to sysop.
* Other protections will be preserved.
* If an existing protection exists for edit=sysop, then it will only be modified if the expiry must be updated.
* If no change are required, none will be made.
*
* @param {object[]} current
* @returns {object}
*/
const getTargetProtection = (current) => {
// There are synonyms for 'infinite' which can be returned by the API.
const infiniteNames = [
'never',
'infinite',
'indefinite',
'infinity',
];
const protectionValue = {
type: 'edit',
level: 'sysop',
expiry: 'infinite',
};
let isProtected = false;
let changed = false;
if (!Array.isArray(current)) {
return {
changed: true,
protection: [protectionValue],
};
}
const protection = current.map((protectionItem) => {
if (protectionItem.type !== 'edit') {
return protectionItem;
}
if (protectionItem.level !== 'sysop') {
protectionItem.level = 'sysop';
changed = true;
}
if (infiniteNames.indexOf(protectionItem.expiry) === -1) {
protectionItem.expiry = 'infinite';
changed = true;
}
isProtected = true;
return protectionItem;
});
if (!isProtected) {
changed = true;
protection.push(protectionValue);
}
return {
protection,
changed,
};
};
for (const [legacyPage] of Object.entries(migratedPageData)) {
logger.debug(`=> Checking ${legacyPage}`);
const [
protectionState,
] = await Promise.all([
getPageProtectionState(legacyPage),
]);
const protectionReason = 'Developer Docs Migration';
const { protection, changed } = getTargetProtection(protectionState.data.protection);
if (changed) {
logger.info(`==> Updating page protection for ${legacyPage}`);
protectPage(legacyPage, protection, protectionReason)
.then(() => {
logger.debug(`===> Updated ${legacyPage}`);
})
.catch((...err) => {
logger.error(err);
});
} else {
logger.debug(`===> No need to update protection for ${legacyPage}`);
}
}
};
/**
* Update the migrated page docs in Wikimedia.
*/
const getUpdateMigratedPages = (logger) => (client) => async (force = false) => {
const migratedPageData = yaml.load(await readFile(getMigrationPagePath(), 'utf8'));
const getDocIdList = (newDocIds) => {
// Ensure that doc ids always have a single leading slash.
const ids = Array.isArray(newDocIds) ? newDocIds : [newDocIds];
return ids.map((newDoc) => newDoc.slug.replace(/^\/*/, '/'));
};
const getCurrentMigratedIdsForPage = (pageTitle) => getGetMigratedPageIds(logger)(client)(pageTitle);
for (const [legacyPage, newPages] of Object.entries(migratedPageData)) {
logger.debug(`=> Checking ${legacyPage}`);
const newDocIds = getDocIdList(newPages);
const migratedDocData = await getCurrentMigratedIdsForPage(legacyPage);
const docIds = migratedDocData.newDocIds.sort();
if (!force && JSON.stringify(docIds) === JSON.stringify(newDocIds)) {
logger.debug(`==> No changes for ${legacyPage} (${docIds.join(', ')})`);
} else {
logger.debug(`===> Current docIds for ${legacyPage} are: ${docIds.join(', ')}`);
logger.info(`===> Setting docIds for ${legacyPage} to: ${newDocIds.join(', ')}`);
const newTemplates = newDocIds.map((newDocId) => `{{Template:Migrated|newDocId=${newDocId}}}\n`).join('');
client.edit(
legacyPage,
newTemplates.trimStart(),
'Update migration status and path',
// This is a major edit. minor = false.
false,
(err, data) => {
if (err) {
throw err;
}
logger.debug(`===> Updated ${legacyPage} to ${data.newrevid}`);
},
);
}
}
};
const guessSlug = (filePath) => {
const normalisedPath = path.join('/', path.relative(process.env.PWD, filePath));
if (normalisedPath.endsWith('/index.md')) {
return normalisedPath.replace(/\/index.md$/, '/');
}
return normalisedPath.replace(/\.md$/, '');
};
const addMigratedPage = async (legacyPath, filePath, slug) => {
const migratedPageData = yaml.load(await readFile(getMigrationPagePath(), 'utf8'));
migratedPageData[legacyPath] = [{
filePath: path.join('/', path.relative(process.env.PWD, filePath)),
slug,
}];
const updatedPageData = Object.fromEntries(Array.from(Object.entries(migratedPageData)).sort());
await writeFile(getMigrationPagePath(), yaml.dump(updatedPageData, {
noArrayIndent: true,
forceQuotes: true,
quotingType: '"',
}), 'utf8');
};
module.exports = {
addMigratedPage,
getClient,
getFetchDoc,
getGetMigratedPageIds,
getGetImagesFromArticle,
getGetImageInfo,
getGetPagesTranscluding,
getLogIn,
getLogger,
getMigrationPagePath,
getObsoletePagePath,
getNormalizedPath,
getGetPagesByPrefix,
getUpdateMigratedPages,
getUpdateMigratedPagesProtection,
guessSlug,
};