firefox-keyword-grab/content-script.js

718 lines
25 KiB
JavaScript

(function() {
// Don't start automatically, wait for custom event
let hasStarted = false;
// Listen for the start event
document.addEventListener('startKeywordExtraction', function() {
if (!hasStarted) {
hasStarted = true;
waitForContent();
}
});
// Helper function to wait for an element to be present and visible
function waitForElement(selector, timeout = 10000) {
return new Promise((resolve, reject) => {
const startTime = Date.now();
function checkElement() {
const element = document.querySelector(selector);
if (element && element.offsetParent !== null) { // Check if element is visible
resolve(element);
} else if (Date.now() - startTime > timeout) {
reject(new Error(`Timeout waiting for element: ${selector}`));
} else {
setTimeout(checkElement, 100);
}
}
checkElement();
});
}
// Helper function to load all rows by scrolling
async function loadAllRows() {
console.log('Waiting for all rows to load...');
// Find scrollable container
const containerSelectors = [
'.main-container',
'.keyword-ideas',
'[role="grid"]'
];
let container = null;
for (const selector of containerSelectors) {
container = document.querySelector(selector);
if (container) {
console.log(`Found scrollable container using selector: ${selector}`);
break;
}
}
if (!container) {
console.log('Could not find scrollable container');
return false;
}
console.log('Container dimensions:', {
scrollHeight: container.scrollHeight,
clientHeight: container.clientHeight,
offsetHeight: container.offsetHeight
});
// Function to get current row count
const getRowCount = () => {
const rows = document.querySelectorAll('.particle-table-row');
return rows.length;
};
let lastRowCount = 0;
let sameCountIterations = 0;
let maxAttempts = 10;
let attempts = 0;
while (attempts < maxAttempts) {
const currentRowCount = getRowCount();
console.log('Current row count:', currentRowCount, sameCountIterations);
if (currentRowCount === lastRowCount) {
sameCountIterations++;
if (sameCountIterations >= 2) {
console.log('Row loading complete. Total rows:', currentRowCount);
return true;
}
} else {
sameCountIterations = 0;
}
lastRowCount = currentRowCount;
// Scroll to bottom
container.scrollTop = container.scrollHeight;
await new Promise(resolve => setTimeout(resolve, 1000));
attempts++;
}
console.log('Could not load all rows after maximum attempts');
return false;
}
// Helper function to wait for content to load after navigation
async function waitForContentLoad() {
// First wait for any loading indicators
await new Promise(resolve => setTimeout(resolve, 1000));
// Then wait for all rows to load using our comprehensive row loading function
const success = await loadAllRows();
if (!success) {
console.log('Warning: Could not load all rows after navigation');
}
// Final wait to ensure everything is stable
await new Promise(resolve => setTimeout(resolve, 500));
}
// Helper function to wait for page content to update
async function waitForPageUpdate(previousStart, maxAttempts = 10) {
console.log('Waiting for page update...');
for (let i = 0; i < maxAttempts; i++) {
await new Promise(resolve => setTimeout(resolve, 500));
const currentInfo = getCurrentPageInfo();
if (currentInfo && currentInfo.start !== previousStart) {
console.log('Page updated successfully');
return true;
}
console.log(`Attempt ${i + 1}: Still waiting for page update...`);
}
console.log('Page update timeout');
return false;
}
// Helper function to ensure content is visible
async function ensureContentVisible() {
const container = document.querySelector('.main-container');
if (!container) return;
// Scroll to bottom to ensure all content is loaded
console.log('Scrolling to bottom to ensure content is loaded...');
container.scrollTo(0, container.scrollHeight);
await new Promise(resolve => setTimeout(resolve, 1000));
// Scroll back to top
container.scrollTo(0, 0);
await new Promise(resolve => setTimeout(resolve, 500));
}
// Helper function to get current page info
function getCurrentPageInfo() {
try {
// Try multiple selectors to find the pagination text
const selectors = [
'.wrap-content > div.selected > div', // Main selector
'.wrap-content div:nth-child(1) > div:nth-child(1)', // Alternative selector
'.selected > div' // Fallback selector
];
let paginationText = null;
for (const selector of selectors) {
const element = document.querySelector(selector);
if (element?.textContent) {
paginationText = element.textContent.trim();
console.log(`Found pagination text using selector "${selector}": "${paginationText}"`);
break;
}
}
if (!paginationText) {
// Check if we have enough rows to determine if pagination should exist
const rows = document.querySelectorAll('table tbody tr');
if (rows.length < 500) {
console.log(`Only ${rows.length} rows found, assuming single page`);
return {
start: 1,
end: rows.length,
total: rows.length
};
}
console.log('No pagination text found');
return null;
}
// Handle various pagination text formats
const match = paginationText.match(/(\d+[\d,]*)\s*-\s*(\d+[\d,]*)\s+of\s+(\d+[\d,]*)/);
if (!match) {
console.log('Could not parse pagination text:', paginationText);
return null;
}
// Remove commas from numbers before parsing
return {
start: parseInt(match[1].replace(/,/g, '')),
end: parseInt(match[2].replace(/,/g, '')),
total: parseInt(match[3].replace(/,/g, ''))
};
} catch (error) {
console.error('Error getting page info:', error);
return null;
}
}
// Helper function to wait for pagination text to update
async function waitForPaginationUpdate(maxAttempts = 10) {
for (let i = 0; i < maxAttempts; i++) {
const pageInfo = getCurrentPageInfo();
if (pageInfo && pageInfo.end >= 500) {
console.log('Pagination text updated to show 500 rows:', pageInfo);
return true;
}
console.log(`Waiting for pagination text to update (attempt ${i + 1}/${maxAttempts})`);
await new Promise(resolve => setTimeout(resolve, 1000));
}
// Even if pagination text isn't found, check if we have 500 rows
const rows = document.querySelectorAll('table tbody tr');
if (rows.length >= 500) {
console.log(`Found ${rows.length} rows, considering update successful`);
return true;
}
console.log('Pagination text did not update to 500 rows');
return false;
}
// Helper function to format search volume
function formatSearchVolume(volume) {
if (!volume) return 'N/A';
// Remove commas and convert to number
return volume.replace(/,/g, '');
}
// Helper function to check if we're on the last page
function isLastPage() {
const nextButton = findNextButton();
return !nextButton || nextButton.getAttribute('aria-disabled') === 'true';
}
// Helper function to find the next button
function findNextButton() {
// Try multiple selectors
const nextButtonSelectors = [
'.next',
'material-button[aria-label="Go to the next page"]',
'[aria-label="Go to the next page"]',
'material-button.next'
];
let nextButton = null;
for (const selector of nextButtonSelectors) {
nextButton = document.querySelector(selector);
if (nextButton) {
console.log(`Found next button using selector: ${selector}`);
break;
}
}
if (!nextButton) {
// Try XPath as fallback
const xpath = '/html/body/div[1]/root/div/div[1]/div[2]/div/div[3]/div/div/awsm-child-content/content-main/div/div/kp-root/div[1]/div/view-loader/combined-ideas-view/ideas-view/div/div/tableview/div[6]/div/div/div/pagination-bar/div/div[2]/div[2]/div[2]/material-button[3]';
const result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
nextButton = result.singleNodeValue;
if (nextButton) {
console.log('Found next button using XPath');
}
}
return nextButton;
}
// Helper function to go to next page
async function goToNextPage() {
const nextButton = findNextButton();
if (!nextButton) {
console.log('Could not find next page button');
return false;
}
if (nextButton.getAttribute('aria-disabled') === 'true') {
console.log('Next button is disabled - on last page');
return false;
}
// Ensure content is visible before clicking next
await ensureContentVisible();
const previousStart = getCurrentPageInfo()?.start;
if (!previousStart) {
console.log('Could not determine current page position');
return false;
}
console.log('Clicking next page button...');
nextButton.click();
// Wait for the page to update
await new Promise(resolve => setTimeout(resolve, 1000));
const updated = await waitForPageUpdate(previousStart);
if (!updated) {
console.log('Page did not update after clicking next');
return false;
}
// Wait for content to load after page change
await waitForContentLoad();
console.log('Successfully moved to next page');
return true;
}
// Helper function to extract current page
async function extractCurrentPage() {
try {
// Ensure content is visible before extraction
await ensureContentVisible();
const sections = [];
const rows = document.querySelectorAll('.particle-table-row');
console.log(`Found ${rows.length} rows to process`);
let currentSource = "Keyword Ideas"; // Default source
for (const row of rows) {
try {
// Check if this is a header row
if (row.classList.contains('group-header')) {
const headerText = row.querySelector('ess-cell')?.textContent.trim();
if (headerText === 'Keywords you provided') {
currentSource = "Keyword Provided";
console.log('Switching to Keywords you provided section');
} else if (headerText === 'Keyword ideas') {
currentSource = "Keyword Ideas";
console.log('Switching to Keyword ideas section');
}
continue; // Skip processing this row as it's a header
}
// Get keyword text
const keywordText = row.querySelector('keyword-text')?.textContent.trim();
if (!keywordText || keywordText.includes('...')) {
continue; // Skip empty or truncated keywords
}
// Get search volume - try multiple selectors
let volumeText = null;
const volumeSelectors = [
'sparkline-graph .value-text',
'.value-text',
'ess-cell[essfield="search_volume"] .value-text',
'ess-cell.data-numeric .value-text'
];
for (const selector of volumeSelectors) {
const volumeElement = row.querySelector(selector);
if (volumeElement) {
volumeText = volumeElement.textContent.trim();
if (volumeText) {
console.log(`Found volume "${volumeText}" using selector: ${selector}`);
break;
}
}
}
// Format the volume (remove commas)
volumeText = formatSearchVolume(volumeText);
// Skip provided keywords with no search volume
if (currentSource === "Keyword Provided" && (volumeText === "—" || volumeText === "N/A")) {
console.log(`Skipping provided keyword "${keywordText}" with no search volume`);
continue;
}
sections.push({
keyword: keywordText,
volume: volumeText,
source: currentSource
});
} catch (error) {
console.error('Error processing row:', error);
}
}
console.log(`Found ${sections.length} keyword sections`);
return sections;
} catch (error) {
console.error('Error extracting current page:', error);
return null;
}
}
// Helper function to create notifications
function createNotification(message, duration = 3000, isError = false) {
const notification = document.createElement('div');
notification.style.cssText = `
position: fixed;
top: 20px;
right: 20px;
padding: 10px 20px;
background-color: ${isError ? '#ff4444' : '#4CAF50'};
color: white;
border-radius: 4px;
z-index: 10000;
font-family: Arial, sans-serif;
box-shadow: 0 2px 5px rgba(0,0,0,0.2);
max-width: 300px;
word-wrap: break-word;
`;
notification.textContent = message;
document.body.appendChild(notification);
if (duration > 0) {
setTimeout(() => notification.remove(), duration);
}
return notification;
}
// Helper function to set show rows to 500
async function setShowRows500() {
console.log('Checking current rows setting...');
// First find the dropdown container
const dropdownSelectors = [
'.dropdown-label-container',
'[aria-label="Show rows: 100 selected."]',
'dropdown-button'
];
let dropdownButton = null;
for (const selector of dropdownSelectors) {
const elements = document.querySelectorAll(selector);
for (const el of elements) {
const text = el.textContent.trim().toLowerCase();
console.log(`Found potential dropdown element with text: "${text}"`);
if (text.includes('show rows') || text.includes('100')) {
dropdownButton = el.querySelector('[buttondecorator], [role="button"]') || el;
console.log(`Selected dropdown button with text: "${text}"`);
break;
}
}
if (dropdownButton) break;
}
if (!dropdownButton) {
// Try direct button selector
dropdownButton = document.querySelector('[aria-label="Show rows: 100 selected."]');
if (dropdownButton) {
console.log('Found dropdown button using aria-label');
}
}
if (!dropdownButton) {
console.log('Could not find rows dropdown button');
return false;
}
// Click the dropdown to open it
console.log('Clicking dropdown button...');
dropdownButton.click();
await new Promise(resolve => setTimeout(resolve, 1000));
// Look for the 500 option
const optionSelectors = [
'material-select-dropdown-item .label',
'material-select-dropdown-item[role="option"] .label',
'[role="option"] .label',
'.label'
];
let found500Option = null;
for (const selector of optionSelectors) {
const options = document.querySelectorAll(selector);
console.log(`Found ${options.length} options with selector: ${selector}`);
for (const option of options) {
const text = option.textContent.trim();
console.log(`Option text: "${text}"`);
if (text === '500') {
found500Option = option.closest('material-select-dropdown-item') || option.closest('[role="option"]') || option;
console.log('Found 500 option');
break;
}
}
if (found500Option) break;
}
if (found500Option) {
console.log('Clicking 500 option...');
found500Option.click();
await new Promise(resolve => setTimeout(resolve, 2000));
return true;
}
console.log('Could not find 500 option');
return false;
}
// Helper function to check if pagination is needed
function shouldHandlePagination(options) {
try {
// First check if user wants all pages
if (!options.extractAllPages) {
console.log('Skipping pagination: extractAllPages is false');
return false;
}
// Get current page info
const pageInfo = getCurrentPageInfo();
if (!pageInfo) {
console.log('Skipping pagination: could not get page info');
return false;
}
// If total is less than or equal to 100, no need for pagination
if (pageInfo.total <= 100) {
console.log(`Skipping pagination: only ${pageInfo.total} total items`);
return false;
}
console.log(`Pagination needed: ${pageInfo.total} total items`);
return true;
} catch (error) {
console.error('Error checking pagination:', error);
return false;
}
}
// Since this is an Angular app, content might load dynamically
// Let's try to wait for the content to load
function waitForContent() {
const maxAttempts = 20;
let attempts = 0;
async function tryFindContent() {
console.log('Attempt', attempts + 1, 'to find keyword content...');
// Look for the keyword grid and at least one row
const hasContent = document.querySelector('[role="grid"][aria-label="Keyword ideas"]') &&
document.querySelector('.particle-table-row');
if (hasContent) {
console.log('Content found, proceeding with row loading...');
// First ensure all rows are loaded, then extract
const success = await loadAllRows();
if (success) {
setTimeout(extractKeywords, 500);
} else {
createNotification('Could not load all rows. Please try again.', 5000, true);
}
} else if (attempts < maxAttempts) {
attempts++;
setTimeout(tryFindContent, 1000); // Wait 1 second before trying again
} else {
createNotification('Could not find keyword content after multiple attempts. Please make sure you are on the correct page.', 5000, true);
}
}
tryFindContent();
}
// Main function to extract keywords
async function extractKeywords() {
try {
// Get the options passed from the popup
const options = window.keywordGrabberOptions || { includeHeaders: true, extractAllPages: true };
// Create a Map to store unique keywords with their volumes and sources
const processedKeywords = new Map(); // Using Map to store keyword -> { volume, source } pairs
// Check if we need pagination
const needsPagination = shouldHandlePagination(options);
// Only try to set rows to 500 if we need pagination
if (needsPagination) {
// First set the rows to 500 if possible
const rowsSet = await setShowRows500();
if (!rowsSet) {
console.log('Warning: Could not set rows to 500');
}
// Add a delay and wait for pagination to update
await new Promise(resolve => setTimeout(resolve, 2000));
await waitForPaginationUpdate();
}
// Ensure content is fully loaded
await ensureContentVisible();
await loadAllRows();
// Get current page info
const pageInfo = getCurrentPageInfo();
if (!pageInfo) {
throw new Error('Could not determine current page information');
}
// Process current page
const currentPageKeywords = await extractCurrentPage();
if (!currentPageKeywords || currentPageKeywords.length === 0) {
throw new Error('No keywords found on current page');
}
// Add current page keywords to processed set
currentPageKeywords.forEach(({ keyword, volume, source }) => {
if (!processedKeywords.has(keyword)) {
processedKeywords.set(keyword, { volume, source });
}
});
// If we're extracting all pages and there are more pages
if (options.extractAllPages && !isLastPage()) {
const notification = createNotification('Processing multiple pages...', 0);
let currentStart = pageInfo.start;
let pageCount = 1;
while (!isLastPage() && pageCount < 20) { // Add safety limit
// Click next page button
const nextButton = findNextButton();
if (!nextButton) {
console.log('Could not find next page button');
break;
}
console.log('Clicking next page button...');
nextButton.click();
// Wait for page to update
const updated = await waitForPageUpdate(currentStart);
if (!updated) {
console.log('Page did not update after clicking next');
break;
}
// Wait for content to load
await waitForContentLoad();
// Extract keywords from this page
const pageKeywords = await extractCurrentPage();
if (pageKeywords && pageKeywords.length > 0) {
pageKeywords.forEach(({ keyword, volume, source }) => {
if (!processedKeywords.has(keyword)) {
processedKeywords.set(keyword, { volume, source });
}
});
notification.textContent = `Processing... ${processedKeywords.size} keywords found`;
}
// Update current start for next iteration
const newPageInfo = getCurrentPageInfo();
if (newPageInfo) {
currentStart = newPageInfo.start;
}
pageCount++;
console.log(`Processed page ${pageCount}`);
}
// Remove the progress notification
notification.remove();
}
// Convert the Map to an array and sort alphabetically
const sortedKeywords = Array.from(processedKeywords.entries())
.sort(([a], [b]) => a.localeCompare(b));
// Create CSV content
let csvContent = '';
if (options.includeHeaders) {
csvContent = 'Keyword,Search Volume,Source\n';
}
csvContent += sortedKeywords.map(([keyword, {volume, source}]) => {
// Escape quotes and wrap in quotes if contains comma
let escapedKeyword = keyword;
if (keyword.includes('"')) {
escapedKeyword = keyword.replace(/"/g, '""');
}
if (keyword.includes(',')) {
escapedKeyword = `"${escapedKeyword}"`;
}
return `${escapedKeyword},${volume},"${source}"`;
}).join('\n');
// Copy to clipboard
const textarea = document.createElement('textarea');
textarea.style.position = 'fixed';
textarea.style.opacity = '0';
document.body.appendChild(textarea);
textarea.value = csvContent;
textarea.select();
try {
document.execCommand('copy');
const headerStatus = options.includeHeaders ? 'with headers ' : '';
createNotification(`${processedKeywords.size} unique keywords copied to clipboard ${headerStatus}in CSV format!`);
console.log(`Total keywords processed: ${processedKeywords.size}`);
} catch (err) {
createNotification('Failed to copy keywords: ' + err.message, 5000, true);
} finally {
document.body.removeChild(textarea);
}
} catch (error) {
console.error('Error:', error);
createNotification('Error: ' + error.message, 5000, true);
}
}
// Listen for messages from the popup
browser.runtime.onMessage.addListener((message) => {
if (message.action === "GrabKeywords") {
window.keywordGrabberOptions = {
includeHeaders: message.includeHeaders
};
// Trigger the start via a custom event
document.dispatchEvent(new CustomEvent('startKeywordExtraction'));
}
});
})();