Wikipedia

User:Polygnotus/DuplicateReferences.js: Difference between revisions

Browse history interactively
← Previous edit
Content deleted Content added
No edit summary
Tag: Reverted
No edit summary
 
(58 intermediate revisions by the same user not shown)
Line 1:
//Testpage: https://en.wikipedia.org/wiki/User:Polygnotus/DuplicateReferencesTest
 
// <nowiki>
mw.loader.using(['mediawiki.util'], function () {
$(document).ready(function () {
console.log("Script started");
 
const DEBUG = false;
if ((mw.config.get('wgNamespaceNumber') !== 0 && mw.config.get('wgPageName') !== 'User:Polygnotus/dupreftest') || mw.config.get('wgAction') !== 'view') {
 
console.log("Not the correct page or action, script terminated");
function debug(...args) return;{
if (DEBUG) {
console.log('[DuplicateReferences]', ...args);
}
}
 
if (
console.log("Page title:", document.title);
mw.config.get('wgAction') !== 'view' ||
console.log("URL:", window.___location.href);
mw.config.get('wgDiffNewId') ||
mw.config.get('wgDiffOldId') ||
(mw.config.get('wgNamespaceNumber') !== 0 && mw.config.get('wgPageName') !== 'User:Polygnotus/DuplicateReferencesTest')
) {
debug("Not the correct page or action, script terminated");
return;
}
 
debug("Page title:", document.title);
debug("URL:", window.___location.href);
 
function findNextReflistDiv(element) {
let nextElement = element.nextElementSibling;
while (nextElement) {
if (nextElement.tagName.toLowerCase() === 'div' &&
(nextElement.classList.contains('reflist') || nextElement.classList.contains('mw-references-wrap'))) {
return nextElement;
Line 25 ⟶ 40:
const referencesHeader = document.querySelector("h2#References");
if (!referencesHeader) {
console.logdebug("References heading not found, script terminated");
return;
}
Line 31 ⟶ 46:
const containerDiv = referencesHeader.closest("div");
if (!containerDiv) {
console.logdebug("Container div not found, script terminated");
return;
}
Line 37 ⟶ 52:
const reflistDiv = findNextReflistDiv(containerDiv);
if (!reflistDiv) {
console.logdebug("Reflist div not found, script terminated");
return;
}
Line 43 ⟶ 58:
const referencesList = reflistDiv.querySelector('ol.references');
if (!referencesList) {
console.logdebug("ol.references not found within reflist div");
return;
}
Line 50 ⟶ 65:
style.textContent = `
li:target { border: 1px dotted red; padding: 2px; background-color: #ffcccc !important;}
.duplicate-citation-highlight { background-color: #cce0ff; border: 1px dotted bluee1eeff; }
.duplicate-citation-hover { background-color: #e1eeffcce0ff; border: 1px dotted blue; }
.duplicate-citation-clicked { border: 1px dotted red; padding: 2px; background-color: #ffe6e6; }
.mw-collapsible-toggle { font-weight: normal; float: right; }
.duplicate-references-table { width: 100%; }
@media only screen and (max-width: 768px) {
.duplicate-references-table { display: none; }
}
`;
document.head.appendChild(style);
 
function addDuplicateCitationsTemplate(linkElement) {
let newParagraph = document.createElement("p");
debug("Adding duplicate citations template");
newParagraph.style.color = "red";
showLoading(linkElement);
 
function addDuplicateCitationsTemplate() {
console.log("Adding duplicate citations template");
var api = new mw.Api();
var pageTitle = mw.config.get('wgPageName');
 
let duplicateInfo = getDuplicateInfo();
 
// Get current date
const currentDate = new Date();
const monthNames = ["January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December"
];
const currentMonth = monthNames[currentDate.getMonth()];
const currentYear = currentDate.getFullYear();
const dateParam = `|date=${currentMonth} ${currentYear}`;
 
api.get({
Line 73 ⟶ 100:
rvslots: 'main',
formatversion: 2
}).then(function (data) {
var page = data.query.pages[0];
var content = page.revisions[0].slots.main.content;
 
var// newContentCreate =the '{{Duplicatereason citations}}\n' + content;string
let reason = 'DuplicateReferences script detected:\n\n';
 
let summary = '+{{Duplicate citations}}';
if (duplicateInfo.length > 0) {
summaryduplicateInfo.forEach((info) +=> ': ';{
duplicateInfo reason += `* ${info.forEach(url} (refs: ${info, index).refs.map(r => {r.number).join(', ')})\n\n`;
summary += `${info.url} (refs: ${info.refs.map(r => r.number).join(', ')})`;
if (index < duplicateInfo.length - 1) {
summary += '; ';
}
});
}
 
// Create the template to insert
const templateToInsert = `{{Duplicated citations|reason=${reason}${dateParam}}}\n`;
 
// Use Morebits to handle the template insertion
const wikitextPage = new Morebits.wikitext.page(content);
// Define templates that should come before the duplicated citations template
const precedingTemplates = [
'short description',
'displaytitle',
'lowercase title',
'italic title',
'about',
'redirect',
'distinguish',
'for',
'Featured list',
'Featured article',
'Good article',
'Other uses',
'Redirect2',
'Use mdy dates',
'Use dmy dates',
'Use American English',
'Use British English'
];
 
// Insert the template after the specified templates
// The third parameter is flags (default 'i' for case-insensitive)
// The fourth parameter can include pre-template content like HTML comments
wikitextPage.insertAfterTemplates(templateToInsert, precedingTemplates, 'i', ['<!--[\\s\\S]*?-->']);
var newContent = wikitextPage.getText();
 
let summary = `Tagged [[WP:DUPREF|duplicate citations]] using [[User:Polygnotus/DuplicateReferences|DuplicateReferences]]`;
 
return api.postWithToken('csrf', {
Line 96 ⟶ 154:
summary: summary
});
}).then(function () {
mw.notifyshowSuccess('Successfully added the Duplicate citations template!'linkElement);
___location.reloadsetTimeout(function (); {
}) ___location.catchreload(function(error) {;
}, 100); // Reload after 0.1 second
}).catch(function (error) {
console.error('Error:', error);
showError(linkElement);
mw.notify('Failed to add the template. See console for details.', {type: 'error'});
});
}
 
function showLoading(element) {
element.innerHTML = '<sup><small>[ Working... ]</small></sup>';
}
 
function showSuccess(element) {
element.innerHTML = '<sup><small>[ Done ]</small></sup>';
}
 
function showError(element) {
element.innerHTML = '<sup><small>[ Error ]</small></sup>';
}
 
function getVisibleText(element) {
// Recursively get the visible text content of an element
let text = '';
for (let node of element.childNodes) {
if (node.nodeType === Node.TEXT_NODE) {
text += node.textContent.trim() + ' ';
} else if (node.nodeType === Node.ELEMENT_NODE) {
// Skip hidden elements
const style = window.getComputedStyle(node);
if (style.display !== 'none' && style.visibility !== 'hidden') {
text += getVisibleText(node) + ' ';
}
}
}
return text.trim();
}
 
function calculateLevenshteinDistance(a, b) {
debug("Comparing:");
debug("Text 1:", a);
debug("Text 2:", b);
 
if (a.length === 0) return b.length;
if (b.length === 0) return a.length;
 
const matrix = [];
 
// Increment along the first column of each row
for (let i = 0; i <= b.length; i++) {
matrix[i] = [i];
}
 
// Increment each column in the first row
for (let j = 0; j <= a.length; j++) {
matrix[0][j] = j;
}
 
// Fill in the rest of the matrix
for (let i = 1; i <= b.length; i++) {
for (let j = 1; j <= a.length; j++) {
if (b.charAt(i - 1) === a.charAt(j - 1)) {
matrix[i][j] = matrix[i - 1][j - 1];
} else {
matrix[i][j] = Math.min(
matrix[i - 1][j - 1] + 1, // substitution
Math.min(
matrix[i][j - 1] + 1, // insertion
matrix[i - 1][j] + 1 // deletion
)
);
}
}
}
 
debug("Levenshtein distance:", matrix[b.length][a.length]);
return matrix[b.length][a.length];
}
 
function calculateSimilarityPercentage(distance, maxLength) {
const similarity = ((maxLength - distance) / maxLength) * 100;
debug("Similarity percentage:", similarity.toFixed(2) + "%");
return Math.round(similarity) + '%';
}
 
function getDuplicateInfo() {
console.logdebug("Getting duplicate info");
 
const referenceItems = referencesList.querySelectorAll('li');
console.log("Number of reference items:", referenceItems.length);
const urlMap = new Map();
const duplicates = [];
const urlMap = new Map();
const referenceItems = Array.from(referencesList.children);
 
debug("Number of reference items:", referenceItems.length);
 
referenceItems.forEach((item, index) => {
constif refNumber(item.tagName.toLowerCase() === index +'li') 1;{
const refId = item.id;
console.log(`Processing reference item ${ const refNumber} (${refId})`)= index + 1;
const span = item.querySelector debug('span.`Processing reference-text' item ${refNumber} (${refId})`);
if (!span) {
console.log(` No reference-text span found in item ${refNumber}`);
return;
}
const links = span.querySelectorAll('a');
console.log(` Number of links in this span: ${links.length}`);
 
let validLink = null; // Get the visible text of the entire reference item
for (let link of links)const {refText = getVisibleText(item);
constdebug(` url =Reference link.hreftext: ${refText}`);
 
const linkText = link.textContent.trim();
// Find the first valid link in the reference
ifconst links = item.querySelectorAll('a');
let validLink linkText !== "Archived" &&null;
for (let link of (!url.includes("wikipedia.org/wiki/") || url.includes("Special:BookSources")links) &&{
!const url = link.includes("_(identifier)")href;
 
) {
validLink// =Skip link;this reference if the URL doesn't contain 'http'
consoleif (!url.logincludes(`'http')) Valid link found: ${url}`);
break debug(` Skipping reference ${refNumber} - URL does not contain 'http'`);
return; // This 'return' is equivalent to 'continue' in a regular for loop
}
const linkText = link.textContent.trim();
 
if (
// (!url.includes("wikipedia.org/wiki/") || url.includes("Special:BookSources")) &&
linkText !== "Archived" &&
!url.includes("wikipedia.org") &&
!url.includes("_(identifier)") && // Templates like ISBN and ISSN and OCLC and S2CID contain (identifier)
!url.startsWith("https://search.worldcat.org/") && // |issn= parameter in cite news
!url.startsWith("https://www.bbc.co.uk/news/live/") && // live articles get frequent updates
!url.startsWith("https://www.aljazeera.com/news/liveblog/") &&
!url.startsWith("https://www.nbcnews.com/news/world/live-blog/") &&
!url.startsWith("https://www.theguardian.com/world/live/") &&
!url.startsWith("https://www.nytimes.com/live/") &&
!url.startsWith("https://edition.cnn.com/world/live-news/") &&
!url.startsWith("https://www.timesofisrael.com/liveblog") &&
!url.startsWith("https://www.france24.com/en/live-news/") &&
!url.startsWith("https://books.google.com/") && //may be 2 different pages of the same book
!url.startsWith("https://archive.org/details/isbn_")
) {
validLink = link;
debug(` Valid link found: ${url}`);
break;
}
}
}
 
if (validLink) {
const url = validLink.href;
if (urlMap.has(url)) {
urlMap.get(url).push({id: refId, number: refNumber, text: refText});
console.log debug(` Duplicate found for URL: ${url}`);
} else {
urlMap.set(url, [{id: refId, number: refNumber, text: refText}]);
debug(` New URL added to map: ${url}`);
}
} else {
urlMap.setdebug(url,` [{id: refId,No number:valid refNumber}]link found in this item`);
console.log(` New URL added to map: ${url}`);
}
} else {
console.log(` No valid link found in this item`);
}
});
Line 158 ⟶ 314:
urlMap.forEach((refs, url) => {
if (refs.length > 1) {
duplicates.push({// url,Calculate refsLevenshtein });distance for each pair of refs
for (let i = 0; i < refs.length - 1; i++) {
for (let j = i + 1; j < refs.length; j++) {
debug(`Comparing references ${refs[i].number} and ${refs[j].number}:`);
const distance = calculateLevenshteinDistance(refs[i].text, refs[j].text);
const maxLength = Math.max(refs[i].text.length, refs[j].text.length);
const similarity = calculateSimilarityPercentage(distance, maxLength);
refs[i].similarity = refs[i].similarity || {};
refs[i].similarity[refs[j].id] = similarity;
}
}
duplicates.push({url, refs});
}
});
 
console.logdebug("Number of duplicate sets found:", duplicates.length);
console.logdebug("Duplicate sets:", duplicates);
return duplicates;
}
 
function checkDuplicateReferenceLinkscreateCollapsibleTable(duplicateInfo) {
console.log("Checkingconst fortable duplicate= reference links"document.createElement('table');
table.className = 'wikitable mw-collapsible duplicate-references-table';
const duplicateInfo = getDuplicateInfo();
table.setAttribute('role', 'presentation');
if (duplicateInfo.length > 0) {
console.log("Duplicates found, creating list");
if (document.querySelector('table.box-Duplicated_citations') === null) {
const editSections = containerDiv.querySelectorAll('span.mw-editsection');
editSections.forEach(editSection => {
let spanBefore = document.createElement('span');
spanBefore.className = 'mw-editsection-bracket';
spanBefore.textContent = '[';
let addTemplateLink = document.createElement('a');
addTemplateLink.textContent = ' add {{duplicated citations}} ';
addTemplateLink.href = '#';
addTemplateLink.addEventListener('click', function(e) {
e.preventDefault();
addDuplicateCitationsTemplate();
});
let spanAfter = document.createElement('span');
spanAfter.className = 'mw-editsection-bracket';
spanAfter.textContent = ']';
editSection.appendChild(spanBefore);
editSection.appendChild(addTemplateLink);
editSection.appendChild(spanAfter);
});
}
duplicateInfo.forEach(({ url, refs }) => {
let paragraphInfo = document.createElement('span');
let urlLink = document.createElement('a');
urlLink.href = url;
urlLink.textContent = url;
urlLink.target = "_blank";
urlLink.rel = "noopener noreferrer";
paragraphInfo.appendChild(document.createTextNode('Duplicate URL: '));
paragraphInfo.appendChild(urlLink);
paragraphInfo.appendChild(document.createTextNode(' in refs: '));
refs.forEach((ref, index) => {
let link = document.createElement('a');
link.href = `#${ref.id}`;
link.textContent = ref.number;
paragraphInfo.appendChild(link);
 
const tbody = linkdocument.addEventListenercreateElement('mouseovertbody', () => {;
refstable.forEachappendChild(r => {tbody);
 
const citationElement = document.getElementById(r.id);
const headerRow = if document.createElement(citationElement'tr') {;
const if (r.idheaderCell === refdocument.idcreateElement('td') {;
headerCell.innerHTML = '<strong>Duplicate References</strong>';
citationElement.classList.add('duplicate-citation-hover');
 
} else {
const toggleSpan = document.createElement('span');
citationElement.classList.add('duplicate-citation-highlight');
toggleSpan.className = }'mw-collapsible-toggle';
toggleSpan.innerHTML = '[<a href="#" class="mw-collapsible-text">hide</a>]';
headerCell.appendChild(toggleSpan);
 
 
// Check if the {{Duplicated citations}} template is already present
const duplicatedCitationsTemplate = document.querySelector('table.box-Duplicated_citations');
 
// Only add the link if the template is not present
if (!duplicatedCitationsTemplate) {
 
// Add the "add {{duplicated citations}}" link to the header
const addTemplateLink = document.createElement('a');
addTemplateLink.textContent = ' add {{duplicated citations}} ';
addTemplateLink.href = '#';
addTemplateLink.addEventListener('click', function (e) {
e.preventDefault();
addDuplicateCitationsTemplate(this);
});
//headerCell.appendChild(document.createTextNode(' ['));
headerCell.appendChild(addTemplateLink);
//headerCell.appendChild(document.createTextNode(']'));
}
headerRow.appendChild(headerCell);
tbody.appendChild(headerRow);
 
const pageTitle = mw.config.get('wgPageName').replace(/_/g, ' ');
 
duplicateInfo.forEach(({url, refs}) => {
const row = document.createElement('tr');
const cell = document.createElement('td');
 
// Create report icon
const reportIcon = document.createElement('a');
reportIcon.href = `https://en.wikipedia.org/wiki/User_talk:Polygnotus?action=edit&section=new&preloadtitle=Reporting%20%5B%5BUser%3APolygnotus%2FDuplicateReferences%7CDuplicateReferences%5D%5D%20false-positive&preload=User:Polygnotus/$1&preloadparams%5b%5d=${encodeURIComponent(`[[${pageTitle}]] ${url}`)}%20~~~~`;
reportIcon.innerHTML = '<img src="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ef/Cross_CSS_Red.svg/15px-Cross_CSS_Red.svg.png" width="15" height="15" alt="Report false positive" title="Report false positive" />';
reportIcon.style.marginRight = '5px';
cell.appendChild(reportIcon);
 
let urlLink = document.createElement('a');
urlLink.href = url;
urlLink.textContent = url;
urlLink.target = "_blank";
urlLink.rel = "noopener noreferrer";
 
cell.appendChild(urlLink);
cell.appendChild(document.createTextNode(' in refs: '));
 
const originalRef = refs[0];
refs.forEach((ref, index) => {
let link = document.createElement('a');
link.href = `#${ref.id}`;
link.textContent = ref.number;
cell.appendChild(link);
 
// Add similarity information
if (index > 0) {
const similarity = calculateSimilarityPercentage(
calculateLevenshteinDistance(originalRef.text, ref.text),
Math.max(originalRef.text.length, ref.text.length)
);
let similarityInfo = document.createElement('span');
similarityInfo.textContent = ` (${similarity})`;
cell.appendChild(similarityInfo);
}
link.addEventListener('mouseover', () => {
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
if (r.id === ref.id) {
citationElement.classList.add('duplicate-citation-hover');
} else {
citationElement.classList.add('duplicate-citation-highlight');
}
});
});
});
link.addEventListener('mouseout', () => {
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
citationElement.classList.remove('duplicate-citation-hover');
citationElement.classList.remove('duplicate-citation-highlight');
}
});
});
});
 
link.addEventListener('click', () => {
document.querySelectorAll('.duplicate-citation-clicked').forEach(el => {
el.classList.remove('duplicate-citation-clicked');
});
refs.forEach(r => {
const citationElement = document.getElementById(r.id);
if (citationElement) {
citationElement.classList.add('duplicate-citation-clicked');
}
});
});
if (index < refs.length - 1) {
paragraphInfo.appendChild(document.createTextNode(', '));
}
});
 
paragraphInfo.appendChildif (documentindex < refs.createElement('br')length - 1); {
newParagraph cell.appendChild(paragraphInfodocument.createTextNode(', '));
}
});
 
row.appendChild(cell);
tbody.appendChild(row);
});
 
return table;
}
 
function checkDuplicateReferenceLinks() {
debug("Checking for duplicate reference links");
const duplicateInfo = getDuplicateInfo();
 
if (duplicateInfo.length > 0) {
debug("Duplicates found, creating collapsible table");
 
const table = createCollapsibleTable(duplicateInfo);
containerDiv.after(table);
 
// Set up collapsible functionality
const toggleLink = table.querySelector('.mw-collapsible-toggle a');
const tableBody = $(table).find('tr:not(:first-child)');
const storageKey = 'duplicateReferencesTableState';
 
function setTableState(isCollapsed) {
if (isCollapsed) {
tableBody.hide();
toggleLink.textContent = 'show';
} else {
tableBody.show();
toggleLink.textContent = 'hide';
}
localStorage.setItem(storageKey, isCollapsed);
}
 
// Initialize state from localStorage
const initialState = localStorage.getItem(storageKey) === 'true';
setTableState(initialState);
 
toggleLink.addEventListener('click', function (e) {
e.preventDefault();
const isCurrentlyCollapsed = tableBody.is(':hidden');
setTableState(!isCurrentlyCollapsed);
});
console.log("Appending duplicate list to page");
containerDiv.after(newParagraph);
} else {
console.logdebug("No duplicates found");
}
}
 
checkDuplicateReferenceLinks();
console.logdebug("Script execution completed");
});
});
// </nowiki>
Retrieved from "https://en.wikipedia.org/wiki/User:Polygnotus/DuplicateReferences.js"