From a2e9e5eef12f94c26659d8d8bc5dc706c13eac04 Mon Sep 17 00:00:00 2001 From: Lord_Devi Date: Thu, 21 Nov 2024 10:40:07 -0500 Subject: [PATCH] Got the SEMRush grabber working, but we are missing progress bars. --- content-script.js | 11 ++ manifest.json | 21 ++- popup.html | 5 +- popup.js | 98 +++++------ semrush-content-script.js | 343 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 426 insertions(+), 52 deletions(-) create mode 100644 semrush-content-script.js diff --git a/content-script.js b/content-script.js index 822b3b5..bf803c0 100644 --- a/content-script.js +++ b/content-script.js @@ -704,4 +704,15 @@ createNotification('Error: ' + error.message, 5000, true); } } + + // Listen for messages from the popup + browser.runtime.onMessage.addListener((message) => { + if (message.action === "GrabKeywords") { + window.keywordGrabberOptions = { + includeHeaders: message.includeHeaders + }; + // Trigger the start via a custom event + document.dispatchEvent(new CustomEvent('startKeywordExtraction')); + } + }); })(); \ No newline at end of file diff --git a/manifest.json b/manifest.json index 472fdda..a042d2c 100644 --- a/manifest.json +++ b/manifest.json @@ -1,12 +1,27 @@ { "manifest_version": 2, - "name": "Keyword Scraper", + "name": "Keyword Grabber", "version": "1.0", - "description": "Scrapes keywords from Google Keyword Planner", + "description": "Extract keywords from Google Keyword Planner and SEMRush", "permissions": [ "activeTab", "clipboardWrite", - "*://*.google.com/*" + "*://*.google.com/*", + "*://*.semrush.com/*" + ], + "background": { + "scripts": ["background.js"], + "persistent": false + }, + "content_scripts": [ + { + "matches": ["*://ads.google.com/*"], + "js": ["content-script.js"] + }, + { + "matches": ["*://*.semrush.com/*"], + "js": ["semrush-content-script.js"] + } ], "browser_action": { "default_popup": "popup.html" diff --git a/popup.html b/popup.html index 8585f96..b5f4458 100644 --- a/popup.html +++ b/popup.html @@ -13,7 +13,7 @@ width: 300px; height: 200px; } - #grabKeywords { + #grabKeywords, #grabSemrushKeywords { width: 100%; padding: 8px; margin: 5px 0; @@ -23,7 +23,7 @@ border: none; border-radius: 4px; } - #grabKeywords:hover { + #grabKeywords:hover, #grabSemrushKeywords:hover { background-color: #45a049; } #dialog { @@ -89,6 +89,7 @@ +
diff --git a/popup.js b/popup.js index c05c701..40dd939 100644 --- a/popup.js +++ b/popup.js @@ -1,52 +1,56 @@ -document.getElementById('grabKeywords').addEventListener('click', function() { - // Add class to body to expand it - document.body.classList.add('dialog-open'); - // Show the dialog and overlay - document.getElementById('dialog').style.display = 'block'; - document.getElementById('overlay').style.display = 'block'; -}); - -async function handleDialogChoice(includeHeaders) { - try { - // Get extract all pages preference - const extractAllPages = document.getElementById('extractAllPages').checked; - - // Hide the dialog and overlay - document.getElementById('dialog').style.display = 'none'; - document.getElementById('overlay').style.display = 'none'; - // Remove the expanded class - document.body.classList.remove('dialog-open'); +document.addEventListener('DOMContentLoaded', function() { + const grabButton = document.getElementById('grabKeywords'); + const semrushButton = document.getElementById('grabSemrushKeywords'); + const dialog = document.getElementById('dialog'); + const overlay = document.getElementById('overlay'); + const body = document.body; - // Get the active tab - const tabs = await browser.tabs.query({active: true, currentWindow: true}); - const activeTab = tabs[0]; + function showDialog(callback) { + dialog.style.display = 'block'; + overlay.style.display = 'block'; + body.classList.add('dialog-open'); + + const yesButton = document.getElementById('yesButton'); + const noButton = document.getElementById('noButton'); + + function handleResponse(includeHeaders) { + dialog.style.display = 'none'; + overlay.style.display = 'none'; + body.classList.remove('dialog-open'); + callback(includeHeaders); + } + + yesButton.onclick = () => handleResponse(true); + noButton.onclick = () => handleResponse(false); + } - // First inject the content script with options - await browser.tabs.executeScript(activeTab.id, { - code: `window.keywordGrabberOptions = { - includeHeaders: ${includeHeaders}, - extractAllPages: ${extractAllPages} - };` + async function sendMessage(action, includeHeaders) { + try { + const tabs = await browser.tabs.query({active: true, currentWindow: true}); + if (tabs[0]) { + await browser.tabs.sendMessage(tabs[0].id, { + action: action, + includeHeaders: includeHeaders + }); + console.log(`Sent ${action} message to content script`); + window.close(); + } else { + console.error('No active tab found'); + } + } catch (error) { + console.error('Error sending message:', error); + } + } + + grabButton.addEventListener('click', function() { + showDialog((includeHeaders) => { + sendMessage("GrabKeywords", includeHeaders); + }); }); - // Then execute the main content script - await browser.tabs.executeScript(activeTab.id, { - file: 'content-script.js' + semrushButton.addEventListener('click', function() { + showDialog((includeHeaders) => { + sendMessage("GrabSemrushKeywords", includeHeaders); + }); }); - - // Trigger the start via a custom event - await browser.tabs.executeScript(activeTab.id, { - code: `document.dispatchEvent(new CustomEvent('startKeywordExtraction'));` - }); - - // Close the popup after initiating the content script - window.close(); - } catch (error) { - console.error('Error:', error); - alert('Error: ' + error.message); - } -} - -// Add click handlers for the Yes and No buttons -document.getElementById('yesButton').addEventListener('click', () => handleDialogChoice(true)); -document.getElementById('noButton').addEventListener('click', () => handleDialogChoice(false)); \ No newline at end of file +}); \ No newline at end of file diff --git a/semrush-content-script.js b/semrush-content-script.js new file mode 100644 index 0000000..0174a35 --- /dev/null +++ b/semrush-content-script.js @@ -0,0 +1,343 @@ +(function() { + // Don't start automatically, wait for custom event + let hasStarted = false; + + // Helper function to wait for an element to be present and visible + async function waitForElement(selector, timeout = 10000) { + const start = Date.now(); + + while (Date.now() - start < timeout) { + const element = document.querySelector(selector); + if (element) { + return element; + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + + throw new Error(`Element ${selector} not found after ${timeout}ms`); + } + + // Helper function to format search volume (remove commas) + function formatSearchVolume(volume) { + if (!volume) return 'N/A'; + return volume.replace(/,/g, ''); + } + + // Helper function to get current page info + function getCurrentPageInfo() { + try { + const pageInput = document.querySelector('.sm-pagination__input input'); + const totalPagesElement = document.querySelector('.___STotalPages_175b1-kmt_ span'); + + // Log the elements we found for debugging + console.log('Page Input Element found:', !!pageInput); + console.log('Total Pages Element found:', !!totalPagesElement); + + if (!pageInput) { + // Try alternative selector for the page input + const altPageInput = document.querySelector('[aria-label="Current page"]'); + if (altPageInput) { + console.log('Found page input using alternative selector'); + pageInput = altPageInput; + } else { + console.log('Could not find page input element with either selector'); + return null; + } + } + + // Get the current page value + const currentPage = parseInt(pageInput.value); + + // If we can't find the total pages element, but we know we're on page 5 + // and the next button is disabled, we can assume we're on the last page + if (!totalPagesElement && currentPage === 5) { + const nextButton = document.querySelector('.___SNextPage_175b1-kmt_'); + if (nextButton && (nextButton.getAttribute('aria-disabled') === 'true' || + nextButton.hasAttribute('disabled') || + nextButton.classList.contains('__disabled_se82q-kmt_'))) { + console.log('On last page (page 5) with disabled next button'); + return { currentPage: 5, totalPages: 5 }; + } + } + + const totalPages = totalPagesElement ? + parseInt(totalPagesElement.textContent.replace(/,/g, '')) : + (currentPage === 5 ? 5 : null); + + if (totalPages === null) { + console.log('Could not determine total pages'); + return null; + } + + // Debug the actual input field value + console.log(`Page Input Value: "${pageInput.value}", Parsed Current Page: ${currentPage}`); + console.log(`Current page: ${currentPage}, Total pages: ${totalPages}`); + return { currentPage, totalPages }; + } catch (error) { + console.error('Error getting page info:', error); + return null; + } + } + + // Helper function to wait for page load after navigation + async function waitForPageLoad(expectedPage, maxAttempts = 20) { + for (let i = 0; i < maxAttempts; i++) { + await new Promise(resolve => setTimeout(resolve, 1000)); // Increased delay + + try { + const pageInput = document.querySelector('.sm-pagination__input input'); + if (pageInput) { + const actualPage = parseInt(pageInput.value); + console.log(`Current page input value: "${pageInput.value}", Parsed: ${actualPage}, Expected: ${expectedPage}`); + + if (actualPage === expectedPage) { + console.log(`Successfully loaded page ${expectedPage}`); + // Wait for table content to load + await new Promise(resolve => setTimeout(resolve, 2000)); + return true; + } + } else { + console.log('Page input element not found during wait'); + } + } catch (error) { + console.error('Error checking page during wait:', error); + } + + console.log(`Waiting for page ${expectedPage} to load (attempt ${i + 1}/${maxAttempts})`); + } + return false; + } + + // Helper function to go to next page + async function goToNextPage(currentPage, totalPages) { + if (currentPage >= totalPages) { + console.log(`Already on last page (${currentPage} of ${totalPages})`); + return false; + } + + const nextButton = document.querySelector('.___SNextPage_175b1-kmt_'); + if (!nextButton) { + console.log('Next button not found'); + return false; + } + + const isDisabled = nextButton.getAttribute('aria-disabled') === 'true' || + nextButton.hasAttribute('disabled') || + nextButton.classList.contains('__disabled_se82q-kmt_'); + + if (isDisabled) { + console.log('Next button is disabled'); + return false; + } + + const expectedNextPage = currentPage + 1; + console.log(`Attempting to navigate from page ${currentPage} to page ${expectedNextPage}`); + + // Get current page info before clicking + const beforeClick = getCurrentPageInfo(); + nextButton.click(); + + const updated = await waitForPageLoad(expectedNextPage); + if (!updated) { + // Double check current page after failed update + const afterClick = getCurrentPageInfo(); + console.log('Navigation check:', { + beforeClick: beforeClick ? beforeClick.currentPage : 'unknown', + expectedPage: expectedNextPage, + afterClick: afterClick ? afterClick.currentPage : 'unknown' + }); + return false; + } + + return true; + } + + // Helper function to extract current page + async function extractCurrentPage() { + try { + // Get current page info for verification + const pageInfo = getCurrentPageInfo(); + if (!pageInfo) { + // If we can't get page info but we can find rows, continue anyway + const rows = document.querySelectorAll('.sm-table-layout__row'); + if (rows.length > 0) { + console.log('Proceeding with extraction despite missing page info. Found rows:', rows.length); + } else { + console.log('Could not verify current page during extraction and found no rows'); + return null; + } + } + + // Wait for table to be present + await new Promise(resolve => setTimeout(resolve, 1000)); + + const rows = document.querySelectorAll('.sm-table-layout__row'); + const keywords = []; + + if (pageInfo) { + console.log(`Found ${rows.length} rows to process on page ${pageInfo.currentPage}`); + } else { + console.log(`Found ${rows.length} rows to process`); + } + + for (const row of rows) { + try { + const keywordElement = row.querySelector('.sm-cell-phrase__link span'); + if (!keywordElement) continue; + + const keyword = keywordElement.textContent.trim(); + + const volumeElement = row.querySelector('[data-testid="table-cell-volume"]'); + const volume = volumeElement ? formatSearchVolume(volumeElement.textContent.trim()) : 'N/A'; + + if (keyword) { + keywords.push({ + keyword: keyword, + volume: volume + }); + } + } catch (error) { + console.error('Error processing row:', error); + } + } + + if (pageInfo) { + console.log(`Extracted ${keywords.length} keywords from page ${pageInfo.currentPage}`); + } else { + console.log(`Extracted ${keywords.length} keywords from current page`); + } + return keywords; + } catch (error) { + console.error('Error extracting current page:', error); + return null; + } + } + + // Helper function to copy text to clipboard + function copyToClipboard(text) { + const textArea = document.createElement('textarea'); + textArea.style.position = 'fixed'; + textArea.style.top = '0'; + textArea.style.left = '0'; + textArea.style.width = '2em'; + textArea.style.height = '2em'; + textArea.style.padding = '0'; + textArea.style.border = 'none'; + textArea.style.outline = 'none'; + textArea.style.boxShadow = 'none'; + textArea.style.background = 'transparent'; + textArea.value = text; + + document.body.appendChild(textArea); + textArea.focus(); + textArea.select(); + + try { + document.execCommand('copy'); + console.log('Text copied to clipboard'); + } catch (err) { + console.error('Failed to copy text:', err); + } + + document.body.removeChild(textArea); + } + + // Main function to extract keywords + async function extractKeywords() { + try { + const options = window.keywordGrabberOptions || { includeHeaders: true }; + const allKeywords = []; + + const initialPageInfo = getCurrentPageInfo(); + if (!initialPageInfo) { + throw new Error('Could not determine page information'); + } + + console.log(`Starting extraction from page ${initialPageInfo.currentPage} of ${initialPageInfo.totalPages}`); + + let currentPage = initialPageInfo.currentPage; + const totalPages = initialPageInfo.totalPages; + + do { + // Extract current page + const keywords = await extractCurrentPage(); + if (!keywords) { + throw new Error('Failed to extract keywords from current page'); + } + + allKeywords.push(...keywords); + console.log(`Total keywords collected: ${allKeywords.length} (Page ${currentPage} of ${totalPages})`); + + if (currentPage >= totalPages) { + console.log('Reached last page'); + break; + } + + const success = await goToNextPage(currentPage, totalPages); + if (!success) { + console.log('Failed to navigate to next page'); + break; + } + currentPage++; + + } while (true); + + // Convert to CSV without quotes + let csv = ''; + if (options.includeHeaders) { + csv = 'Keyword,Search Volume\n'; + } + + csv += allKeywords.map(k => `${k.keyword},${k.volume}`).join('\n'); + + // Copy to clipboard + copyToClipboard(csv); + console.log(`Extraction complete. Total keywords: ${allKeywords.length}`); + + // Create notification + const div = document.createElement('div'); + div.style.position = 'fixed'; + div.style.top = '20px'; + div.style.left = '50%'; + div.style.transform = 'translateX(-50%)'; + div.style.backgroundColor = '#4CAF50'; + div.style.color = 'white'; + div.style.padding = '15px'; + div.style.borderRadius = '5px'; + div.style.zIndex = '10000'; + div.textContent = `${allKeywords.length} keywords copied to clipboard!`; + + document.body.appendChild(div); + setTimeout(() => div.remove(), 3000); + + } catch (error) { + console.error('Error during keyword extraction:', error); + throw error; + } + } + + // Listen for messages from the popup + browser.runtime.onMessage.addListener((message) => { + if (message.action === "GrabSemrushKeywords") { + window.keywordGrabberOptions = { + includeHeaders: message.includeHeaders + }; + // Trigger the start via a custom event + document.dispatchEvent(new CustomEvent('GrabSemrushKeywords')); + } + }); + + // Listen for the start event + document.addEventListener('GrabSemrushKeywords', async function(e) { + if (hasStarted) return; + hasStarted = true; + + try { + await extractKeywords(); + } catch (error) { + console.error('Error during SEMRush keyword extraction:', error); + } finally { + hasStarted = false; + } + }); +})();