From 21fdd18e07c02b2b1ca3dc3b6bcb4db1a0e22c83 Mon Sep 17 00:00:00 2001 From: Lord_Devi Date: Thu, 21 Nov 2024 14:03:45 -0500 Subject: [PATCH] Got the progress indicator working. --- content-script.js | 958 +++++++++++++++++++++----------------- manifest.json | 4 +- semrush-content-script.js | 202 ++++---- shared-ui.js | 83 ++++ 4 files changed, 745 insertions(+), 502 deletions(-) create mode 100644 shared-ui.js diff --git a/content-script.js b/content-script.js index bf803c0..7fad141 100644 --- a/content-script.js +++ b/content-script.js @@ -1,10 +1,11 @@ (function() { // Don't start automatically, wait for custom event let hasStarted = false; + let isExtracting = false; // Listen for the start event document.addEventListener('startKeywordExtraction', function() { - if (!hasStarted) { + if (!hasStarted && !isExtracting) { hasStarted = true; waitForContent(); } @@ -30,103 +31,104 @@ }); } + // Helper function to get current row count + function getRowCount() { + const rows = document.querySelectorAll('.particle-table-row'); + return rows.length; + } + + // Helper function to wait for page update + async function waitForPageUpdate(previousStart) { + console.log('Waiting for page update...'); + + // Initial wait for page transition + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Check for page update + for (let i = 0; i < 10; i++) { + const pageInfo = getCurrentPageInfo(); + if (!pageInfo) { + await new Promise(resolve => setTimeout(resolve, 1000)); + continue; + } + + if (pageInfo.start !== previousStart) { + console.log(`Page updated: ${previousStart} -> ${pageInfo.start}`); + return true; + } + + await new Promise(resolve => setTimeout(resolve, 1000)); + } + + return false; + } + // Helper function to load all rows by scrolling async function loadAllRows() { console.log('Waiting for all rows to load...'); - // Find scrollable container - const containerSelectors = [ - '.main-container', - '.keyword-ideas', - '[role="grid"]' - ]; - - let container = null; - for (const selector of containerSelectors) { - container = document.querySelector(selector); - if (container) { - console.log(`Found scrollable container using selector: ${selector}`); - break; - } - } - + const container = document.querySelector('.main-container'); if (!container) { console.log('Could not find scrollable container'); return false; } - console.log('Container dimensions:', { - scrollHeight: container.scrollHeight, - clientHeight: container.clientHeight, - offsetHeight: container.offsetHeight - }); - - // Function to get current row count - const getRowCount = () => { - const rows = document.querySelectorAll('.particle-table-row'); - return rows.length; - }; + // Scroll to bottom and wait + container.scrollTop = container.scrollHeight; + await new Promise(resolve => setTimeout(resolve, 2000)); let lastRowCount = 0; - let sameCountIterations = 0; - let maxAttempts = 10; + let currentRowCount = getRowCount(); let attempts = 0; - while (attempts < maxAttempts) { - const currentRowCount = getRowCount(); - console.log('Current row count:', currentRowCount, sameCountIterations); - - if (currentRowCount === lastRowCount) { - sameCountIterations++; - if (sameCountIterations >= 2) { - console.log('Row loading complete. Total rows:', currentRowCount); - return true; - } - } else { - sameCountIterations = 0; - } - + while (lastRowCount !== currentRowCount && attempts < 5) { lastRowCount = currentRowCount; - - // Scroll to bottom container.scrollTop = container.scrollHeight; await new Promise(resolve => setTimeout(resolve, 1000)); + currentRowCount = getRowCount(); attempts++; + console.log(`Row loading attempt ${attempts}: ${currentRowCount} rows`); } - console.log('Could not load all rows after maximum attempts'); - return false; + return true; } // Helper function to wait for content to load after navigation async function waitForContentLoad() { - // First wait for any loading indicators - await new Promise(resolve => setTimeout(resolve, 1000)); + console.log('Waiting for content to load...'); - // Then wait for all rows to load using our comprehensive row loading function + // First wait for any loading indicators to disappear + await delay(2000); + + // Try to force content load by scrolling multiple times + const container = document.querySelector('.main-container'); + if (container) { + for (let i = 0; i < 3; i++) { + console.log(`Scroll attempt ${i + 1}/3 to force content load`); + // Scroll to different positions + container.scrollTo(0, container.scrollHeight * 0.3); + await delay(500); + container.scrollTo(0, container.scrollHeight * 0.6); + await delay(500); + container.scrollTo(0, container.scrollHeight); + await delay(1000); + } + } + + // Then wait for all rows to load const success = await loadAllRows(); if (!success) { - console.log('Warning: Could not load all rows after navigation'); + console.log('Warning: Could not verify all rows loaded after navigation'); } // Final wait to ensure everything is stable - await new Promise(resolve => setTimeout(resolve, 500)); - } - - // Helper function to wait for page content to update - async function waitForPageUpdate(previousStart, maxAttempts = 10) { - console.log('Waiting for page update...'); - for (let i = 0; i < maxAttempts; i++) { - await new Promise(resolve => setTimeout(resolve, 500)); - const currentInfo = getCurrentPageInfo(); - if (currentInfo && currentInfo.start !== previousStart) { - console.log('Page updated successfully'); - return true; - } - console.log(`Attempt ${i + 1}: Still waiting for page update...`); - } - console.log('Page update timeout'); - return false; + await delay(1000); + + // Verify we have content + const rows = document.querySelectorAll('.particle-table-row'); + console.log(`Found ${rows.length} rows after content load`); + + return rows.length > 0; } // Helper function to ensure content is visible @@ -172,7 +174,8 @@ return { start: 1, end: rows.length, - total: rows.length + total: rows.length, + perPage: rows.length }; } console.log('No pagination text found'); @@ -187,10 +190,15 @@ } // Remove commas from numbers before parsing + const start = parseInt(match[1].replace(/,/g, '')); + const end = parseInt(match[2].replace(/,/g, '')); + const total = parseInt(match[3].replace(/,/g, '')); + return { - start: parseInt(match[1].replace(/,/g, '')), - end: parseInt(match[2].replace(/,/g, '')), - total: parseInt(match[3].replace(/,/g, '')) + start, + end, + total, + perPage: end - start + 1 }; } catch (error) { console.error('Error getting page info:', error); @@ -198,72 +206,112 @@ } } - // Helper function to wait for pagination text to update - async function waitForPaginationUpdate(maxAttempts = 10) { + // Helper function to wait for paginator to be ready + async function waitForPaginatorReady(maxAttempts = 10) { for (let i = 0; i < maxAttempts; i++) { - const pageInfo = getCurrentPageInfo(); - if (pageInfo && pageInfo.end >= 500) { - console.log('Pagination text updated to show 500 rows:', pageInfo); + // Check for paginator container + const paginator = document.querySelector('.mat-mdc-paginator-container, .mat-paginator-container'); + if (!paginator) { + await new Promise(resolve => setTimeout(resolve, 500)); + continue; + } + + // Check if paginator is interactive + const buttons = paginator.querySelectorAll('button'); + let allDisabled = true; + for (const button of buttons) { + if (!button.hasAttribute('disabled')) { + allDisabled = false; + break; + } + } + + if (!allDisabled) { return true; } - console.log(`Waiting for pagination text to update (attempt ${i + 1}/${maxAttempts})`); - await new Promise(resolve => setTimeout(resolve, 1000)); - } - - // Even if pagination text isn't found, check if we have 500 rows - const rows = document.querySelectorAll('table tbody tr'); - if (rows.length >= 500) { - console.log(`Found ${rows.length} rows, considering update successful`); - return true; - } - - console.log('Pagination text did not update to 500 rows'); - return false; - } - // Helper function to format search volume - function formatSearchVolume(volume) { - if (!volume) return 'N/A'; - // Remove commas and convert to number - return volume.replace(/,/g, ''); + await new Promise(resolve => setTimeout(resolve, 500)); + } + return false; } // Helper function to check if we're on the last page function isLastPage() { + const pageInfo = getCurrentPageInfo(); + if (!pageInfo) return true; + + // Check if we're at the end based on current range + const lastItemOnPage = pageInfo.start + pageInfo.perPage - 1; + if (lastItemOnPage >= pageInfo.total) { + console.log('Reached last item, on last page'); + return true; + } + const nextButton = findNextButton(); - return !nextButton || nextButton.getAttribute('aria-disabled') === 'true'; + if (!nextButton) { + console.log('No next button found, assuming last page'); + return true; + } + + // Check various disabled states + const isDisabled = nextButton.hasAttribute('disabled') || + nextButton.getAttribute('aria-disabled') === 'true' || + nextButton.classList.contains('disabled') || + nextButton.closest('[aria-disabled="true"]') !== null || + window.getComputedStyle(nextButton).opacity === '0.5'; + + if (isDisabled) { + console.log('Next button is disabled, on last page'); + return true; + } + + return false; } - // Helper function to find the next button + // Helper function to find next button function findNextButton() { - // Try multiple selectors const nextButtonSelectors = [ - '.next', - 'material-button[aria-label="Go to the next page"]', - '[aria-label="Go to the next page"]', + // Specific selector for this case + 'material-button.next[aria-label="Go to the next page"]', + '.next[role="button"]:not([aria-disabled="true"])', + 'material-button.next:not([aria-disabled="true"])', + // Generic next page selectors + 'material-button[aria-label="Go to the next page"]:not([aria-disabled="true"])', + '[aria-label="Go to the next page"]:not([aria-disabled="true"])', + // Fallback selectors + '.next:not([aria-disabled="true"])', 'material-button.next' ]; - let nextButton = null; for (const selector of nextButtonSelectors) { - nextButton = document.querySelector(selector); - if (nextButton) { + const buttons = document.querySelectorAll(selector); + for (const button of buttons) { + // Check if button exists and is connected to DOM + if (!button || !button.isConnected) continue; + + // Check computed style + const style = window.getComputedStyle(button); + if (style.display === 'none' || style.visibility === 'hidden') continue; + + // Check if button is in viewport and clickable + const rect = button.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) continue; + + // Check if button is truly enabled + if (button.getAttribute('aria-disabled') === 'true' || + button.hasAttribute('disabled') || + button.classList.contains('disabled')) { + console.log('Next button found but is disabled'); + continue; + } + console.log(`Found next button using selector: ${selector}`); - break; + return button; } } - if (!nextButton) { - // Try XPath as fallback - const xpath = '/html/body/div[1]/root/div/div[1]/div[2]/div/div[3]/div/div/awsm-child-content/content-main/div/div/kp-root/div[1]/div/view-loader/combined-ideas-view/ideas-view/div/div/tableview/div[6]/div/div/div/pagination-bar/div/div[2]/div[2]/div[2]/material-button[3]'; - const result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null); - nextButton = result.singleNodeValue; - if (nextButton) { - console.log('Found next button using XPath'); - } - } - - return nextButton; + console.log('Could not find next button'); + return null; } // Helper function to go to next page @@ -309,83 +357,56 @@ // Helper function to extract current page async function extractCurrentPage() { - try { - // Ensure content is visible before extraction - await ensureContentVisible(); - - const sections = []; - const rows = document.querySelectorAll('.particle-table-row'); - - console.log(`Found ${rows.length} rows to process`); - - let currentSource = "Keyword Ideas"; // Default source - - for (const row of rows) { - try { - // Check if this is a header row - if (row.classList.contains('group-header')) { - const headerText = row.querySelector('ess-cell')?.textContent.trim(); - if (headerText === 'Keywords you provided') { - currentSource = "Keyword Provided"; - console.log('Switching to Keywords you provided section'); - } else if (headerText === 'Keyword ideas') { - currentSource = "Keyword Ideas"; - console.log('Switching to Keyword ideas section'); - } - continue; // Skip processing this row as it's a header - } - - // Get keyword text - const keywordText = row.querySelector('keyword-text')?.textContent.trim(); - if (!keywordText || keywordText.includes('...')) { - continue; // Skip empty or truncated keywords + // Wait for rows to be loaded + await waitForRowsToLoad(); + + // Find all keyword sections + const rows = document.querySelectorAll('.particle-table-row'); + console.log(`Found ${rows.length} rows to process`); + + const keywords = []; + rows.forEach(row => { + try { + // Get keyword text + const keywordElement = row.querySelector('.keyword-text'); + const volumeElement = row.querySelector('sparkline-graph .value-text'); + + if (keywordElement && volumeElement) { + const keyword = keywordElement.textContent.trim(); + const volume = volumeElement.textContent.trim(); + + // Skip keywords with "—" volume + if (volume === '—') { + return; } - // Get search volume - try multiple selectors - let volumeText = null; - const volumeSelectors = [ - 'sparkline-graph .value-text', - '.value-text', - 'ess-cell[essfield="search_volume"] .value-text', - 'ess-cell.data-numeric .value-text' - ]; - - for (const selector of volumeSelectors) { - const volumeElement = row.querySelector(selector); - if (volumeElement) { - volumeText = volumeElement.textContent.trim(); - if (volumeText) { - console.log(`Found volume "${volumeText}" using selector: ${selector}`); - break; + // Find the closest group header + let currentRow = row; + let source = 'Keyword Ideas'; // default + while (currentRow) { + const header = currentRow.previousElementSibling; + if (header && header.classList.contains('group-header')) { + const headerText = header.textContent.trim(); + if (headerText.includes('Keywords you provided')) { + source = 'Provided Keywords'; } + break; } + currentRow = currentRow.previousElementSibling; } - // Format the volume (remove commas) - volumeText = formatSearchVolume(volumeText); - - // Skip provided keywords with no search volume - if (currentSource === "Keyword Provided" && (volumeText === "—" || volumeText === "N/A")) { - console.log(`Skipping provided keyword "${keywordText}" with no search volume`); - continue; - } - - sections.push({ - keyword: keywordText, - volume: volumeText, - source: currentSource + keywords.push({ + keyword, + volume: volume.replace(/,/g, ''), + source }); - } catch (error) { - console.error('Error processing row:', error); } + } catch (error) { + console.error('Error processing row:', error); } - - console.log(`Found ${sections.length} keyword sections`); - return sections; - } catch (error) { - console.error('Error extracting current page:', error); - return null; - } + }); + + return keywords; } // Helper function to create notifications @@ -417,111 +438,343 @@ // Helper function to set show rows to 500 async function setShowRows500() { - console.log('Checking current rows setting...'); - - // First find the dropdown container - const dropdownSelectors = [ - '.dropdown-label-container', - '[aria-label="Show rows: 100 selected."]', - 'dropdown-button' - ]; - - let dropdownButton = null; - for (const selector of dropdownSelectors) { - const elements = document.querySelectorAll(selector); - for (const el of elements) { - const text = el.textContent.trim().toLowerCase(); - console.log(`Found potential dropdown element with text: "${text}"`); - if (text.includes('show rows') || text.includes('100')) { - dropdownButton = el.querySelector('[buttondecorator], [role="button"]') || el; - console.log(`Selected dropdown button with text: "${text}"`); + try { + // First check if we're already at 500 rows + const currentInfo = getCurrentPageInfo(); + if (currentInfo && currentInfo.perPage === 500) { + console.log('Already at 500 rows per page'); + return true; + } + + // First find the dropdown container + const dropdownSelectors = [ + 'material-dropdown-select[aria-label="Show rows"]', + 'material-dropdown-select[aria-label="Show rows"] material-dropdown-select', + '[aria-label="Show rows: 100 selected."]', + '.dropdown-label-container' + ]; + + let dropdownButton = null; + for (const selector of dropdownSelectors) { + const element = document.querySelector(selector); + if (element) { + console.log(`Found dropdown using selector: ${selector}`); + dropdownButton = element; break; } } - if (dropdownButton) break; - } - if (!dropdownButton) { - // Try direct button selector - dropdownButton = document.querySelector('[aria-label="Show rows: 100 selected."]'); - if (dropdownButton) { - console.log('Found dropdown button using aria-label'); + if (!dropdownButton) { + console.log('Could not find rows dropdown button'); + return false; } - } - if (!dropdownButton) { - console.log('Could not find rows dropdown button'); + // Click the dropdown to open it + console.log('Clicking dropdown button...'); + dropdownButton.click(); + await new Promise(resolve => setTimeout(resolve, 2000)); + + // Look for the 500 option + const optionSelectors = [ + 'material-select-dropdown-item', + '[role="option"]' + ]; + + let found500Option = null; + for (const selector of optionSelectors) { + const options = Array.from(document.querySelectorAll(selector)); + for (const option of options) { + const text = option.textContent.trim(); + if (text === '500') { + found500Option = option; + console.log(`Found 500 option using selector: ${selector}`); + break; + } + } + if (found500Option) break; + } + + if (found500Option) { + console.log('Clicking 500 option...'); + found500Option.click(); + + // Wait for the page to update + await new Promise(resolve => setTimeout(resolve, 3000)); + + // Verify the change by checking page info + const updatedInfo = getCurrentPageInfo(); + if (updatedInfo && updatedInfo.perPage === 500) { + console.log('Successfully changed to 500 rows'); + return true; + } + } + + console.log('Could not find or click 500 option'); + return false; + } catch (error) { + console.error('Error setting rows to 500:', error); + return false; + } + } + + // Helper function to check if we should handle pagination + function shouldHandlePagination(options) { + // Always return true since we need to set rows to 500 regardless of extractAllPages + return true; + } + + // Helper function to handle pagination + async function handlePagination() { + // Initial scroll to ensure current page is fully loaded + await scrollToBottom(); + await waitForRowsToLoad(); + + // Get current page info + const paginationText = await getPaginationText(); + if (!paginationText) { + console.log('No pagination text found'); return false; } - // Click the dropdown to open it - console.log('Clicking dropdown button...'); - dropdownButton.click(); + // Find and click next button + const nextButton = findNextButton(); + if (!nextButton) { + console.log('No next button found, assuming last page'); + return false; + } + + // Get current info before moving + const currentInfo = getCurrentPageInfo(); + if (!currentInfo) { + console.log('Could not get current page info'); + return false; + } + + // Click the next button + console.log('Clicking next button...'); + nextButton.click(); + + // Wait for the page to update await new Promise(resolve => setTimeout(resolve, 1000)); + const updated = await waitForPageUpdate(currentInfo.start); + + if (!updated) { + console.log('Page did not update after clicking next'); + return false; + } - // Look for the 500 option - const optionSelectors = [ - 'material-select-dropdown-item .label', - 'material-select-dropdown-item[role="option"] .label', - '[role="option"] .label', - '.label' - ]; + // Wait for content to load after page change + await waitForContentLoad(); + console.log('Successfully moved to next page'); + return true; + } - let found500Option = null; - for (const selector of optionSelectors) { - const options = document.querySelectorAll(selector); - console.log(`Found ${options.length} options with selector: ${selector}`); + // Helper function to create a delay + function delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + // Helper function to scroll to bottom + async function scrollToBottom() { + const container = document.querySelector('.main-container'); + if (!container) return; + + // Scroll to bottom to ensure all content is loaded + console.log('Scrolling to bottom to ensure content is loaded...'); + container.scrollTo(0, container.scrollHeight); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + + // Helper function to wait for rows to load + async function waitForRowsToLoad() { + // Function to get current row count + const getRowCount = () => { + const rows = document.querySelectorAll('.particle-table-row'); + return rows.length; + }; + + let lastRowCount = 0; + let sameCountIterations = 0; + let maxAttempts = 10; + let attempts = 0; + + while (attempts < maxAttempts) { + const currentRowCount = getRowCount(); + console.log('Current row count:', currentRowCount, sameCountIterations); - for (const option of options) { - const text = option.textContent.trim(); - console.log(`Option text: "${text}"`); - if (text === '500') { - found500Option = option.closest('material-select-dropdown-item') || option.closest('[role="option"]') || option; - console.log('Found 500 option'); - break; + if (currentRowCount === lastRowCount) { + sameCountIterations++; + if (sameCountIterations >= 2) { + console.log('Row loading complete. Total rows:', currentRowCount); + return true; } + } else { + sameCountIterations = 0; } - if (found500Option) break; + + lastRowCount = currentRowCount; + + // Scroll to bottom + const container = document.querySelector('.main-container'); + if (container) { + container.scrollTop = container.scrollHeight; + } + await new Promise(resolve => setTimeout(resolve, 1000)); + attempts++; } - - if (found500Option) { - console.log('Clicking 500 option...'); - found500Option.click(); - await new Promise(resolve => setTimeout(resolve, 2000)); - return true; - } - - console.log('Could not find 500 option'); + + console.log('Could not load all rows after maximum attempts'); return false; } - // Helper function to check if pagination is needed - function shouldHandlePagination(options) { + // Helper function to get pagination text + async function getPaginationText() { + // Try multiple selectors to find the pagination text + const selectors = [ + '.wrap-content > div.selected > div', // Main selector + '.wrap-content div:nth-child(1) > div:nth-child(1)', // Alternative selector + '.selected > div' // Fallback selector + ]; + + let paginationText = null; + for (const selector of selectors) { + const element = document.querySelector(selector); + if (element?.textContent) { + paginationText = element.textContent.trim(); + console.log(`Found pagination text using selector "${selector}": "${paginationText}"`); + break; + } + } + + return paginationText; + } + + // Main function to extract keywords + async function extractKeywords() { + if (isExtracting) { + console.log('Extraction already in progress, skipping...'); + return; + } + + isExtracting = true; + const progressDialog = new KeywordGrabberUI.ProgressDialog().create(); + try { - // First check if user wants all pages - if (!options.extractAllPages) { - console.log('Skipping pagination: extractAllPages is false'); - return false; + progressDialog.updateProgress('Starting extraction...'); + const processedKeywords = new Map(); + + // Initial setup + await waitForContent(); + + // Set rows to 500 + progressDialog.updateProgress('Setting rows to 500...'); + await setShowRows500(); + await waitForContentLoad(); + + // Get total pages + const initialPageInfo = getCurrentPageInfo(); + const totalPages = initialPageInfo && initialPageInfo.total ? + Math.ceil(initialPageInfo.total / initialPageInfo.perPage) : 0; + + let pageCount = 1; + + do { + progressDialog.updateProgress(`Processing page ${pageCount}${totalPages ? ` of ${totalPages}` : ''}...`); + + // Load rows for current page + await loadAllRows(); + + // Extract keywords from current page + const currentPageKeywords = await extractCurrentPage(); + if (!currentPageKeywords || currentPageKeywords.length === 0) { + console.log('No keywords found on current page'); + break; + } + + console.log(`Found ${currentPageKeywords.length} keywords on page ${pageCount}`); + + // Add to processed set + currentPageKeywords.forEach(({ keyword, volume, source }) => { + if (!processedKeywords.has(keyword)) { + processedKeywords.set(keyword, { volume, source }); + } + }); + + progressDialog.updateProgress(`Found ${processedKeywords.size} keywords...`); + + // Check if we're on the last page + if (isLastPage()) { + console.log('On last page, finished pagination'); + break; + } + + // Get current page info before moving + const currentStart = getCurrentPageInfo()?.start; + if (!currentStart) { + console.log('Could not get current page info'); + break; + } + + // Move to next page + progressDialog.updateProgress('Moving to next page...'); + const nextButton = findNextButton(); + if (!nextButton) { + console.log('Could not find next button'); + break; + } + + nextButton.click(); + + // Wait for page update + const updated = await waitForPageUpdate(currentStart); + if (!updated) { + console.log('Page did not update after clicking next'); + break; + } + + // Wait for content to load on new page + await waitForContentLoad(); + pageCount++; + + } while (pageCount <= totalPages || totalPages === 0); + + // Convert keywords to array and sort by volume + const keywordArray = Array.from(processedKeywords.entries()).map(([keyword, { volume, source }]) => ({ + keyword, + volume, + source + })); + + // Sort by volume (highest to lowest) + keywordArray.sort((a, b) => { + const volA = parseInt(a.volume.replace(/,/g, '')) || 0; + const volB = parseInt(b.volume.replace(/,/g, '')) || 0; + return volB - volA; + }); + + // Create CSV content + let csvContent = ''; + if (window.keywordGrabberOptions.includeHeaders) { + csvContent = 'Keyword,Search Volume,Source\n'; } - - // Get current page info - const pageInfo = getCurrentPageInfo(); - if (!pageInfo) { - console.log('Skipping pagination: could not get page info'); - return false; - } - - // If total is less than or equal to 100, no need for pagination - if (pageInfo.total <= 100) { - console.log(`Skipping pagination: only ${pageInfo.total} total items`); - return false; - } - - console.log(`Pagination needed: ${pageInfo.total} total items`); - return true; + csvContent += keywordArray.map(k => `${k.keyword},${k.volume},${k.source}`).join('\n'); + + // Copy to clipboard + await navigator.clipboard.writeText(csvContent); + + console.log(`Total keywords processed: ${processedKeywords.size}`); + progressDialog.updateProgress(`Successfully copied ${processedKeywords.size} keywords!`); + await new Promise(resolve => setTimeout(resolve, 2000)); + } catch (error) { - console.error('Error checking pagination:', error); - return false; + console.error('Error:', error); + progressDialog.updateProgress(`Error: ${error.message}`); + await new Promise(resolve => setTimeout(resolve, 3000)); + throw error; + } finally { + progressDialog.remove(); + isExtracting = false; + hasStarted = false; } } @@ -532,6 +785,11 @@ let attempts = 0; async function tryFindContent() { + if (!hasStarted) { + console.log('Extraction cancelled'); + return; + } + console.log('Attempt', attempts + 1, 'to find keyword content...'); // Look for the keyword grid and at least one row @@ -542,175 +800,33 @@ console.log('Content found, proceeding with row loading...'); // First ensure all rows are loaded, then extract const success = await loadAllRows(); - if (success) { - setTimeout(extractKeywords, 500); + if (success && hasStarted) { + await extractKeywords(); } else { createNotification('Could not load all rows. Please try again.', 5000, true); + hasStarted = false; } - } else if (attempts < maxAttempts) { + } else if (attempts < maxAttempts && hasStarted) { attempts++; setTimeout(tryFindContent, 1000); // Wait 1 second before trying again } else { createNotification('Could not find keyword content after multiple attempts. Please make sure you are on the correct page.', 5000, true); + hasStarted = false; } } tryFindContent(); } - // Main function to extract keywords - async function extractKeywords() { - try { - // Get the options passed from the popup - const options = window.keywordGrabberOptions || { includeHeaders: true, extractAllPages: true }; - - // Create a Map to store unique keywords with their volumes and sources - const processedKeywords = new Map(); // Using Map to store keyword -> { volume, source } pairs - - // Check if we need pagination - const needsPagination = shouldHandlePagination(options); - - // Only try to set rows to 500 if we need pagination - if (needsPagination) { - // First set the rows to 500 if possible - const rowsSet = await setShowRows500(); - if (!rowsSet) { - console.log('Warning: Could not set rows to 500'); - } - - // Add a delay and wait for pagination to update - await new Promise(resolve => setTimeout(resolve, 2000)); - await waitForPaginationUpdate(); - } - - // Ensure content is fully loaded - await ensureContentVisible(); - await loadAllRows(); - - // Get current page info - const pageInfo = getCurrentPageInfo(); - if (!pageInfo) { - throw new Error('Could not determine current page information'); - } - - // Process current page - const currentPageKeywords = await extractCurrentPage(); - if (!currentPageKeywords || currentPageKeywords.length === 0) { - throw new Error('No keywords found on current page'); - } - - // Add current page keywords to processed set - currentPageKeywords.forEach(({ keyword, volume, source }) => { - if (!processedKeywords.has(keyword)) { - processedKeywords.set(keyword, { volume, source }); - } - }); - - // If we're extracting all pages and there are more pages - if (options.extractAllPages && !isLastPage()) { - const notification = createNotification('Processing multiple pages...', 0); - - let currentStart = pageInfo.start; - let pageCount = 1; - - while (!isLastPage() && pageCount < 20) { // Add safety limit - // Click next page button - const nextButton = findNextButton(); - if (!nextButton) { - console.log('Could not find next page button'); - break; - } - - console.log('Clicking next page button...'); - nextButton.click(); - - // Wait for page to update - const updated = await waitForPageUpdate(currentStart); - if (!updated) { - console.log('Page did not update after clicking next'); - break; - } - - // Wait for content to load - await waitForContentLoad(); - - // Extract keywords from this page - const pageKeywords = await extractCurrentPage(); - if (pageKeywords && pageKeywords.length > 0) { - pageKeywords.forEach(({ keyword, volume, source }) => { - if (!processedKeywords.has(keyword)) { - processedKeywords.set(keyword, { volume, source }); - } - }); - notification.textContent = `Processing... ${processedKeywords.size} keywords found`; - } - - // Update current start for next iteration - const newPageInfo = getCurrentPageInfo(); - if (newPageInfo) { - currentStart = newPageInfo.start; - } - - pageCount++; - console.log(`Processed page ${pageCount}`); - } - - // Remove the progress notification - notification.remove(); - } - - // Convert the Map to an array and sort alphabetically - const sortedKeywords = Array.from(processedKeywords.entries()) - .sort(([a], [b]) => a.localeCompare(b)); - - // Create CSV content - let csvContent = ''; - if (options.includeHeaders) { - csvContent = 'Keyword,Search Volume,Source\n'; - } - csvContent += sortedKeywords.map(([keyword, {volume, source}]) => { - // Escape quotes and wrap in quotes if contains comma - let escapedKeyword = keyword; - if (keyword.includes('"')) { - escapedKeyword = keyword.replace(/"/g, '""'); - } - if (keyword.includes(',')) { - escapedKeyword = `"${escapedKeyword}"`; - } - return `${escapedKeyword},${volume},"${source}"`; - }).join('\n'); - - // Copy to clipboard - const textarea = document.createElement('textarea'); - textarea.style.position = 'fixed'; - textarea.style.opacity = '0'; - document.body.appendChild(textarea); - textarea.value = csvContent; - textarea.select(); - - try { - document.execCommand('copy'); - const headerStatus = options.includeHeaders ? 'with headers ' : ''; - createNotification(`${processedKeywords.size} unique keywords copied to clipboard ${headerStatus}in CSV format!`); - console.log(`Total keywords processed: ${processedKeywords.size}`); - } catch (err) { - createNotification('Failed to copy keywords: ' + err.message, 5000, true); - } finally { - document.body.removeChild(textarea); - } - - } catch (error) { - console.error('Error:', error); - createNotification('Error: ' + error.message, 5000, true); - } - } - // Listen for messages from the popup browser.runtime.onMessage.addListener((message) => { - if (message.action === "GrabKeywords") { + if (message.action === "GrabKeywords" && !isExtracting) { window.keywordGrabberOptions = { - includeHeaders: message.includeHeaders + includeHeaders: message.includeHeaders, + extractAllPages: true // Always extract all pages }; + // Reset state and trigger start + hasStarted = false; // Trigger the start via a custom event document.dispatchEvent(new CustomEvent('startKeywordExtraction')); } diff --git a/manifest.json b/manifest.json index a042d2c..b17398f 100644 --- a/manifest.json +++ b/manifest.json @@ -16,11 +16,11 @@ "content_scripts": [ { "matches": ["*://ads.google.com/*"], - "js": ["content-script.js"] + "js": ["shared-ui.js", "content-script.js"] }, { "matches": ["*://*.semrush.com/*"], - "js": ["semrush-content-script.js"] + "js": ["shared-ui.js", "semrush-content-script.js"] } ], "browser_action": { diff --git a/semrush-content-script.js b/semrush-content-script.js index 0174a35..7f802bd 100644 --- a/semrush-content-script.js +++ b/semrush-content-script.js @@ -26,53 +26,96 @@ // Helper function to get current page info function getCurrentPageInfo() { try { - const pageInput = document.querySelector('.sm-pagination__input input'); - const totalPagesElement = document.querySelector('.___STotalPages_175b1-kmt_ span'); + // Try multiple selectors for pagination elements + const pageInputSelectors = [ + '.sm-pagination__input input', + '[aria-label="Current page"]', + 'input[type="number"][aria-label*="page"]', + '.pagination input[type="number"]' + ]; + + const totalPagesSelectors = [ + '.___STotalPages_175b1-kmt_ span', + '.pagination__total', + '.pagination__pages-total', + '.sm-pagination__total' + ]; + + // Find page input + let pageInput = null; + for (const selector of pageInputSelectors) { + pageInput = document.querySelector(selector); + if (pageInput) { + console.log(`Found page input using selector: ${selector}`); + break; + } + } + + // Find total pages element + let totalPagesElement = null; + for (const selector of totalPagesSelectors) { + totalPagesElement = document.querySelector(selector); + if (totalPagesElement) { + console.log(`Found total pages using selector: ${selector}`); + break; + } + } - // Log the elements we found for debugging console.log('Page Input Element found:', !!pageInput); console.log('Total Pages Element found:', !!totalPagesElement); if (!pageInput) { - // Try alternative selector for the page input - const altPageInput = document.querySelector('[aria-label="Current page"]'); - if (altPageInput) { - console.log('Found page input using alternative selector'); - pageInput = altPageInput; - } else { - console.log('Could not find page input element with either selector'); - return null; - } + console.log('Could not find page input element with any selector'); + return null; } // Get the current page value const currentPage = parseInt(pageInput.value); + if (isNaN(currentPage)) { + console.log('Invalid current page value:', pageInput.value); + return null; + } - // If we can't find the total pages element, but we know we're on page 5 - // and the next button is disabled, we can assume we're on the last page - if (!totalPagesElement && currentPage === 5) { - const nextButton = document.querySelector('.___SNextPage_175b1-kmt_'); - if (nextButton && (nextButton.getAttribute('aria-disabled') === 'true' || - nextButton.hasAttribute('disabled') || - nextButton.classList.contains('__disabled_se82q-kmt_'))) { - console.log('On last page (page 5) with disabled next button'); - return { currentPage: 5, totalPages: 5 }; + let totalPages = null; + + // Try to get total pages from element + if (totalPagesElement) { + const totalText = totalPagesElement.textContent.trim(); + totalPages = parseInt(totalText.replace(/[^\d]/g, '')); + } + + // If we can't find total pages but we're on page 5 with disabled next button + if (totalPages === null && currentPage === 5) { + const nextButtonSelectors = [ + '.___SNextPage_175b1-kmt_', + '.sm-pagination__next', + '.pagination__next', + 'button[aria-label*="next"]' + ]; + + for (const selector of nextButtonSelectors) { + const nextButton = document.querySelector(selector); + if (nextButton && ( + nextButton.getAttribute('aria-disabled') === 'true' || + nextButton.hasAttribute('disabled') || + nextButton.classList.contains('__disabled_se82q-kmt_') + )) { + console.log('On last page (page 5) with disabled next button'); + totalPages = 5; + break; + } } } - const totalPages = totalPagesElement ? - parseInt(totalPagesElement.textContent.replace(/,/g, '')) : - (currentPage === 5 ? 5 : null); - - if (totalPages === null) { - console.log('Could not determine total pages'); - return null; + // If we still can't determine total pages but have current page + if (totalPages === null && currentPage > 0) { + console.log('Using current page as minimum total pages'); + totalPages = currentPage; } - // Debug the actual input field value - console.log(`Page Input Value: "${pageInput.value}", Parsed Current Page: ${currentPage}`); console.log(`Current page: ${currentPage}, Total pages: ${totalPages}`); return { currentPage, totalPages }; + } catch (error) { console.error('Error getting page info:', error); return null; @@ -242,77 +285,78 @@ document.body.removeChild(textArea); } + // Helper function to format keywords + function formatKeywords(keywords, includeHeaders) { + let csv = ''; + if (includeHeaders) { + csv = 'Keyword,Search Volume\n'; + } + + csv += keywords.map(k => `${k.keyword},${k.volume}`).join('\n'); + return csv; + } + // Main function to extract keywords async function extractKeywords() { + const progressDialog = new KeywordGrabberUI.ProgressDialog().create(); try { const options = window.keywordGrabberOptions || { includeHeaders: true }; const allKeywords = []; - - const initialPageInfo = getCurrentPageInfo(); - if (!initialPageInfo) { - throw new Error('Could not determine page information'); + + const pageInfo = getCurrentPageInfo(); + if (!pageInfo) { + throw new Error('Could not get page information'); } - - console.log(`Starting extraction from page ${initialPageInfo.currentPage} of ${initialPageInfo.totalPages}`); - - let currentPage = initialPageInfo.currentPage; - const totalPages = initialPageInfo.totalPages; - - do { - // Extract current page + + progressDialog.updateProgress(`Starting extraction from page ${pageInfo.currentPage} of ${pageInfo.totalPages}`); + console.log(`Starting extraction from page ${pageInfo.currentPage} of ${pageInfo.totalPages}`); + + let currentPage = pageInfo.currentPage; + const totalPages = pageInfo.totalPages; + + while (currentPage <= totalPages) { const keywords = await extractCurrentPage(); if (!keywords) { throw new Error('Failed to extract keywords from current page'); } - - allKeywords.push(...keywords); - console.log(`Total keywords collected: ${allKeywords.length} (Page ${currentPage} of ${totalPages})`); - - if (currentPage >= totalPages) { - console.log('Reached last page'); - break; - } - const success = await goToNextPage(currentPage, totalPages); - if (!success) { - console.log('Failed to navigate to next page'); + allKeywords.push(...keywords); + progressDialog.updateProgress(`Total keywords collected: ${allKeywords.length} (Page ${currentPage} of ${totalPages})`); + console.log(`Total keywords collected: ${allKeywords.length} (Page ${currentPage} of ${totalPages})`); + + if (currentPage < totalPages) { + progressDialog.updateProgress(`Navigating to page ${currentPage + 1}...`); + const success = await goToNextPage(currentPage, totalPages); + if (!success) { + throw new Error('Failed to navigate to next page'); + } + currentPage++; + } else { break; } - currentPage++; - - } while (true); - - // Convert to CSV without quotes - let csv = ''; - if (options.includeHeaders) { - csv = 'Keyword,Search Volume\n'; } - - csv += allKeywords.map(k => `${k.keyword},${k.volume}`).join('\n'); + + // Format the keywords + const formattedKeywords = formatKeywords(allKeywords, options.includeHeaders); // Copy to clipboard - copyToClipboard(csv); + copyToClipboard(formattedKeywords); + + progressDialog.updateProgress(`Extraction complete. Total keywords: ${allKeywords.length}`); console.log(`Extraction complete. Total keywords: ${allKeywords.length}`); - // Create notification - const div = document.createElement('div'); - div.style.position = 'fixed'; - div.style.top = '20px'; - div.style.left = '50%'; - div.style.transform = 'translateX(-50%)'; - div.style.backgroundColor = '#4CAF50'; - div.style.color = 'white'; - div.style.padding = '15px'; - div.style.borderRadius = '5px'; - div.style.zIndex = '10000'; - div.textContent = `${allKeywords.length} keywords copied to clipboard!`; - - document.body.appendChild(div); - setTimeout(() => div.remove(), 3000); + // Give users time to see the completion message + await new Promise(resolve => setTimeout(resolve, 2000)); + return allKeywords; } catch (error) { console.error('Error during keyword extraction:', error); + progressDialog.updateProgress(`Error: ${error.message}`); + // Keep error message visible for a moment + await new Promise(resolve => setTimeout(resolve, 3000)); throw error; + } finally { + progressDialog.remove(); } } diff --git a/shared-ui.js b/shared-ui.js new file mode 100644 index 0000000..ffe0840 --- /dev/null +++ b/shared-ui.js @@ -0,0 +1,83 @@ +// Shared UI components for keyword grabber extension + +// Create and manage a progress dialog +class ProgressDialog { + constructor() { + this.dialog = null; + this.messageElement = null; + } + + create() { + // Create dialog if it doesn't exist + if (!this.dialog) { + this.dialog = document.createElement('div'); + this.dialog.style.cssText = ` + position: fixed; + top: 20px; + right: 20px; + background: white; + border: 1px solid #ccc; + border-radius: 4px; + padding: 15px 20px; + box-shadow: 0 2px 10px rgba(0,0,0,0.1); + z-index: 10000; + font-family: Arial, sans-serif; + min-width: 200px; + `; + + // Add progress message + this.messageElement = document.createElement('div'); + this.messageElement.style.cssText = ` + margin-bottom: 10px; + color: #333; + font-size: 14px; + `; + this.dialog.appendChild(this.messageElement); + + // Add spinner + const spinner = document.createElement('div'); + spinner.style.cssText = ` + width: 20px; + height: 20px; + border: 2px solid #f3f3f3; + border-top: 2px solid #3498db; + border-radius: 50%; + animation: spin 1s linear infinite; + margin: 10px auto; + `; + this.dialog.appendChild(spinner); + + // Add spinner animation + const style = document.createElement('style'); + style.textContent = ` + @keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } + } + `; + document.head.appendChild(style); + + document.body.appendChild(this.dialog); + } + return this; + } + + updateProgress(message) { + if (this.messageElement) { + this.messageElement.textContent = message; + } + } + + remove() { + if (this.dialog && this.dialog.parentNode) { + this.dialog.parentNode.removeChild(this.dialog); + this.dialog = null; + this.messageElement = null; + } + } +} + +// Export for use in other scripts +window.KeywordGrabberUI = { + ProgressDialog +};