User:Polygnotus/Scripts/CategoryToClipboard.js: Difference between revisions

Revision as of 07:40, 25 May 2025

Revision as of 07:49, 25 May 2025

Line 1:

// Wikipedia Category Items Copier - Improved Version

// This script adds three buttons to Wikipedia category pages:

// 1. "Copy Items" - Copies all items in the current category via API

Line 6:

const API_DELAY = 500; // Delay between API requests in milliseconds

const MAX_RETRIES = 3; // Maximum number of retries for failed requests

// Only run on Wikipedia category pages

Line 20 ⟶ 21:

container.style.border = '1px solid #a2a9b1';

container.style.borderRadius = '3px';

// Helper function to create tooltip

function addTooltip(element, text) {

element.title = text;

element.style.position = 'relative';

}

// Create the "Copy Items" button

Line 27 ⟶ 34:

copyItemsBtn.style.padding = '8px 12px';

copyItemsBtn.style.cursor = 'pointer';

addTooltip(copyItemsBtn, 'Copy all articles and pages from this category only (not subcategories). Fast operation.');

// Create the "Copy All Items" button

Line 34 ⟶ 42:

copyAllItemsBtn.style.padding = '8px 12px';

copyAllItemsBtn.style.cursor = 'pointer';

addTooltip(copyAllItemsBtn, 'Copy all articles and pages from this category AND all its subcategories. May take several minutes for large category trees.');

// Create the "Copy Subcategories" button

Line 40 ⟶ 49:

copySubcatsBtn.style.padding = '8px 12px';

copySubcatsBtn.style.cursor = 'pointer';

addTooltip(copySubcatsBtn, 'Copy only the category names (not articles) from this category tree. Useful for exploring category structure.');

// Add checkbox for URL export

const urlCheckbox = document.createElement('input');

urlCheckbox.type = 'checkbox';

urlCheckbox.id = 'includeUrls';

urlCheckbox.style.marginLeft = '15px';

const urlLabel = document.createElement('label');

urlLabel.htmlFor = 'includeUrls';

urlLabel.textContent = 'Include URLs';

urlLabel.style.marginLeft = '5px';

addTooltip(urlLabel, 'Include full Wikipedia URLs for each item in the export');

// Create status text

Line 50 ⟶ 72:

container.appendChild(copyAllItemsBtn);

container.appendChild(copySubcatsBtn);

container.appendChild(urlCheckbox);

container.appendChild(urlLabel);

container.appendChild(statusText);

Line 58 ⟶ 82:

} else {

document.querySelector('#content').prepend(container);

}

// Global visited set to prevent visiting any page more than once across all operations

const globalVisited = new Set();

// Function to format items with URLs if requested

function formatItems(items, includeUrls) {

if (!includeUrls) {

return items.join('\n');

}

return items.map(item => {

const encodedTitle = encodeURIComponent(item.replace(/ /g, '_'));

return `${item}\nhttps://en.wikipedia.org/wiki/${encodedTitle}`;

}).join('\n\n');

}

Line 164 ⟶ 203:

}

// Enhanced API request function with retry logic, rate limiting, and maxlag handling

// Function to get all members of a category using Wikipedia API

async function getCategoryMembersmakeApiRequest(categoryTitleurl, continueTokenretryCount = null0) {

try {

//await Basenew APIPromise(resolve URL=> setTimeout(resolve, API_DELAY));

let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmlimit=max&format=json&origin=*`;

const response = await fetch(url);

// AddHandle continuerate tokenlimiting if(HTTP provided429) or server errors (5xx)

if (continueTokenresponse.status === 429 || response.status >= 500) {

apiUrlif +=(retryCount < MAX_RETRIES) `&cmcontinue=${continueToken}`;

const waitTime = Math.pow(2, retryCount) * 1000; // Exponential backoff

statusText.innerHTML += `<br>Rate limited or server error, waiting ${waitTime/1000}s before retry ${retryCount + 1}/${MAX_RETRIES}...`;

await new Promise(resolve => setTimeout(resolve, waitTime));

return makeApiRequest(url, retryCount + 1);

} else {

throw new Error(`Request failed after ${MAX_RETRIES} retries: ${response.status}`);

}

}

if (!response.ok) {

statusText.textContent = `Fetching category members for: ${categoryTitle}...`;

throw new Error(`HTTP ${response.status}: ${response.statusText}`);

}

await new Promise(resolve => setTimeout(resolve, API_DELAY));

const response = await fetch(apiUrl);

const data = await response.json();

// Handle maxlag errors - these don't count as retries since they're not real failures

if (!data.query || !data.query.categorymembers) {

if (data.error && consoledata.error("Unexpected.code API=== response:", data'maxlag'); {

const lagTime = data.error.lag || 5; // Default to 5 seconds if lag not specified

return { members: [], continueToken: null };

const waitTime = (lagTime + 2) * 1000; // Add 2 second buffer

statusText.innerHTML += `<br>Database lagged (${lagTime}s), waiting ${waitTime/1000}s before retry...`;

await new Promise(resolve => setTimeout(resolve, waitTime));

return makeApiRequest(url, retryCount); // Don't increment retry count for maxlag

}

// ExtractHandle membersother andAPI continue tokenerrors

const members =if (data.query.categorymembers.map(membererror) => member.title);{

throw new Error(`API Error: ${data.error.code} - ${data.error.info}`);

const nextContinueToken = data.continue ? data.continue.cmcontinue : null;

}

return data;

return { members, continueToken: nextContinueToken };

} catch (error) {

console.errorif ("APIretryCount request< error:", errorMAX_RETRIES); {

statusText.textContentinnerHTML += `Error<br>Request fetchingfailed, categoryretrying members:${retryCount + 1}/${error.messageMAX_RETRIES}...`;

await new Promise(resolve => setTimeout(resolve, 1000));

return { members: [], continueToken: null };

return makeApiRequest(url, retryCount + 1);

} else {

throw error;

}

}

}

// Function to get all subcategories of a category

async function getSubcategories(categoryTitle, continueToken = null) {

try {

// Base API URL for subcategories (only get items with namespace 14, which is Category)

// Add maxlag parameter to be respectful of server load

let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=14&cmlimit=max&format=json&origin=*`;

let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=14&cmlimit=max&maxlag=5&format=json&origin=*`;

// Add continue token if provided

Line 212 ⟶ 269:

statusText.textContent = `Fetching subcategories for: ${categoryTitle}...`;

awaitconst new Promise(resolvedata => setTimeoutawait makeApiRequest(resolve, API_DELAY)apiUrl);

const response = await fetch(apiUrl);

const data = await response.json();

if (!data.query || !data.query.categorymembers) {

Line 222 ⟶ 276:

}

// Extract subcategories and continue token, prefix with "Category:"

const subcategories = data.query.categorymembers.map(member => member.title.replace('); // Keep full "Category:'," ''));prefix

const nextContinueToken = data.continue ? data.continue.cmcontinue : null;

Line 229 ⟶ 283:

} catch (error) {

console.error("API request error:", error);

statusText.textContentinnerHTML += `<br>Error fetching subcategories: ${error.message}`;

return { subcategories: [], continueToken: null };

}

Line 238 ⟶ 292:

try {

// Base API URL for non-category members (exclude namespace 14, which is Category)

// Add maxlag parameter to be respectful of server load

let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=0|1|2|3|4|5|6|7|8|9|10|11|12|13|15&cmlimit=max&format=json&origin=*`;

let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=0|1|2|3|4|5|6|7|8|9|10|11|12|13|15&cmlimit=max&maxlag=5&format=json&origin=*`;

// Add continue token if provided

Line 247 ⟶ 302:

statusText.textContent = `Fetching items for: ${categoryTitle}...`;

awaitconst new Promise(resolvedata => setTimeoutawait makeApiRequest(resolve, API_DELAY)apiUrl);

const response = await fetch(apiUrl);

const data = await response.json();

if (!data.query || !data.query.categorymembers) {

Line 264 ⟶ 316:

} catch (error) {

console.error("API request error:", error);

statusText.textContentinnerHTML += `<br>Error fetching items: ${error.message}`;

return { members: [], continueToken: null };

}

Line 283 ⟶ 335:

statusText.innerHTML = `Retrieved ${allMembers.length} items from "${categoryTitle}" (page ${pagesProcessed})...`;

if (continueToken) {

statusText.innerHTML += ` Pausing before next request...`;

await new Promise(resolve => setTimeout(resolve, API_DELAY));

}

} while (continueToken);

Line 304 ⟶ 352:

pagesProcessed++;

if (continueToken) {

statusText.innerHTML += ` Pausing before next subcategory request...`;

await new Promise(resolve => setTimeout(resolve, API_DELAY));

}

} while (continueToken);

Line 313 ⟶ 357:

}

// Function to recursively get all subcategories with circular reference detection

async function getAllSubcategoriesRecursive(categoryTitle, processedCategories = new Set(), depth = 0) {

//const Preventvisited infinite= loopsnew and excessive depthSet();

const allSubcategories = [];

if (processedCategories.has(categoryTitle) || depth > 10) {

const queue = [`Category:${categoryTitle}`]; // Start with prefixed category

return [];

}

processedCategories.add(categoryTitle);

statusText.innerHTML = `Exploring subcategories of "${categoryTitle}" (depth ${depth}, found ${processedCategories.size} categories so far)...`;

//while Get(queue.length direct> subcategories0) {

const directSubcategoriescurrentCategory = await getAllSubcategoriesqueue.shift(categoryTitle);

let allSubcategories = [...directSubcategories];

// Skip if already visited (circular reference detection)

if (visited.has(currentCategory) || globalVisited.has(currentCategory)) {

// Recursively get subcategories of each subcategory

continue;

for (const subcategory of directSubcategories) {

if (!processedCategories.has(subcategory)) {}

const nestedSubcategories = await getAllSubcategoriesRecursive(subcategory, processedCategories, depth + 1);

visited.add(currentCategory);

allSubcategories = allSubcategories.concat(nestedSubcategories);

globalVisited.add(currentCategory);

statusText.innerHTML = `Exploring subcategories (found ${allSubcategories.length} categories, queue: ${queue.length})...`;

// Get direct subcategories (remove "Category:" prefix for API call)

const categoryNameForApi = currentCategory.replace('Category:', '');

const directSubcategories = await getAllSubcategories(categoryNameForApi);

// Add new subcategories to results and queue

for (const subcategory of directSubcategories) {

if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {

allSubcategories.push(subcategory);

queue.push(subcategory);

}

}

}

Line 340 ⟶ 393:

// Function to recursively get all items from a category and all its subcategories

async function getAllItemsRecursive(categoryTitle, processedCategories = new Set(), depth = 0, itemStats = { totalCategories: 0, totalItems: 0 }) {

const visited = new Set();

// Prevent infinite loops and excessive depth || depth > 10

const allItems = [];

if (processedCategories.has(categoryTitle)) {

const queue = [categoryTitle]; // Start without prefix for consistency

return [];

}let totalCategories = 0;

while (queue.length > 0) {

processedCategories.add(categoryTitle);

const currentCategory = queue.shift();

itemStats.totalCategories++;

const categoryKey = `Category:${currentCategory}`;

statusText.innerHTML = `Getting items from "${categoryTitle}" (depth ${depth}, processed ${itemStats.totalCategories} categories, found ${itemStats.totalItems} items so far)...`;

// Skip if already visited (circular reference detection)

if (visited.has(categoryKey) || globalVisited.has(categoryKey)) {

// Get items from current category

continue;

const currentItems = await getAllCategoryMembers(categoryTitle);

}

itemStats.totalItems += currentItems.length;

let allItems = [...currentItems];

visited.add(categoryKey);

// Get direct subcategories globalVisited.add(categoryKey);

totalCategories++;

const directSubcategories = await getAllSubcategories(categoryTitle);

statusText.innerHTML = `Getting items from "${currentCategory}" (processed ${totalCategories} categories, found ${allItems.length} items, queue: ${queue.length})...`;

// Recursively get items from each subcategory

for (const subcategory of directSubcategories) {

// Get items from current category

if (!processedCategories.has(subcategory)) {

const nestedItemscurrentItems = await getAllItemsRecursivegetAllCategoryMembers(subcategory, processedCategories, depth + 1, itemStatscurrentCategory);

allItems = allItems.concatpush(nestedItems...currentItems);

// Get direct subcategories and add to queue

const directSubcategories = await getAllSubcategories(currentCategory);

for (const subcategory of directSubcategories) {

if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {

// Remove "Category:" prefix for queue consistency

const subcategoryName = subcategory.replace('Category:', '');

queue.push(subcategoryName);

}

}

}

return { items: allItems, totalCategories };

}

Line 382 ⟶ 444:

}

const copySuccessincludeUrls = await copyToClipboardOrDownload(itemsurlCheckbox.join('\n'), categoryName)checked;

const formattedText = formatItems(items, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName);

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${items.length} items to clipboard.`;

Line 397 ⟶ 462:

try {

const// processedCategoriesClear =global newvisited Set();set for this operation

globalVisited.clear();

const itemStats = { totalCategories: 0, totalItems: 0 };

// Get all items recursively

const { items: allItems, totalCategories } = await getAllItemsRecursive(categoryName, processedCategories, 0, itemStats);

// Deduplicate items

Line 411 ⟶ 476:

}

const includeUrls = urlCheckbox.checked;

const copySuccess = await copyToClipboardOrDownload(uniqueItems.join('\n'), categoryName + '_all_recursive');

const formattedText = formatItems(uniqueItems, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_all_recursive');

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${uniqueItems.length} unique items to clipboard from ${itemStats.totalCategories} categories.`;

}

} catch (error) {

Line 426 ⟶ 494:

try {

const// processedCategoriesClear =global newvisited Set();set for this operation

globalVisited.clear();

const allSubcategories = await getAllSubcategoriesRecursive(categoryName, processedCategories);

const allSubcategories = await getAllSubcategoriesRecursive(categoryName);

// Deduplicate subcategories

Line 437 ⟶ 507:

}

const includeUrls = urlCheckbox.checked;

const copySuccess = await copyToClipboardOrDownload(uniqueSubcategories.join('\n'), categoryName + '_subcategories');

const formattedText = formatItems(uniqueSubcategories, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_subcategories');

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${uniqueSubcategories.length} unique subcategories to clipboard.`;