diff --git a/content-script.js b/content-script.js
index 822b3b5..bf803c0 100644
--- a/content-script.js
+++ b/content-script.js
@@ -704,4 +704,15 @@
createNotification('Error: ' + error.message, 5000, true);
}
}
+
+ // Listen for messages from the popup
+ browser.runtime.onMessage.addListener((message) => {
+ if (message.action === "GrabKeywords") {
+ window.keywordGrabberOptions = {
+ includeHeaders: message.includeHeaders
+ };
+ // Trigger the start via a custom event
+ document.dispatchEvent(new CustomEvent('startKeywordExtraction'));
+ }
+ });
})();
\ No newline at end of file
diff --git a/manifest.json b/manifest.json
index 472fdda..a042d2c 100644
--- a/manifest.json
+++ b/manifest.json
@@ -1,12 +1,27 @@
{
"manifest_version": 2,
- "name": "Keyword Scraper",
+ "name": "Keyword Grabber",
"version": "1.0",
- "description": "Scrapes keywords from Google Keyword Planner",
+ "description": "Extract keywords from Google Keyword Planner and SEMRush",
"permissions": [
"activeTab",
"clipboardWrite",
- "*://*.google.com/*"
+ "*://*.google.com/*",
+ "*://*.semrush.com/*"
+ ],
+ "background": {
+ "scripts": ["background.js"],
+ "persistent": false
+ },
+ "content_scripts": [
+ {
+ "matches": ["*://ads.google.com/*"],
+ "js": ["content-script.js"]
+ },
+ {
+ "matches": ["*://*.semrush.com/*"],
+ "js": ["semrush-content-script.js"]
+ }
],
"browser_action": {
"default_popup": "popup.html"
diff --git a/popup.html b/popup.html
index 8585f96..b5f4458 100644
--- a/popup.html
+++ b/popup.html
@@ -13,7 +13,7 @@
width: 300px;
height: 200px;
}
- #grabKeywords {
+ #grabKeywords, #grabSemrushKeywords {
width: 100%;
padding: 8px;
margin: 5px 0;
@@ -23,7 +23,7 @@
border: none;
border-radius: 4px;
}
- #grabKeywords:hover {
+ #grabKeywords:hover, #grabSemrushKeywords:hover {
background-color: #45a049;
}
#dialog {
@@ -89,6 +89,7 @@
+
diff --git a/popup.js b/popup.js
index c05c701..40dd939 100644
--- a/popup.js
+++ b/popup.js
@@ -1,52 +1,56 @@
-document.getElementById('grabKeywords').addEventListener('click', function() {
- // Add class to body to expand it
- document.body.classList.add('dialog-open');
- // Show the dialog and overlay
- document.getElementById('dialog').style.display = 'block';
- document.getElementById('overlay').style.display = 'block';
-});
-
-async function handleDialogChoice(includeHeaders) {
- try {
- // Get extract all pages preference
- const extractAllPages = document.getElementById('extractAllPages').checked;
-
- // Hide the dialog and overlay
- document.getElementById('dialog').style.display = 'none';
- document.getElementById('overlay').style.display = 'none';
- // Remove the expanded class
- document.body.classList.remove('dialog-open');
+document.addEventListener('DOMContentLoaded', function() {
+ const grabButton = document.getElementById('grabKeywords');
+ const semrushButton = document.getElementById('grabSemrushKeywords');
+ const dialog = document.getElementById('dialog');
+ const overlay = document.getElementById('overlay');
+ const body = document.body;
- // Get the active tab
- const tabs = await browser.tabs.query({active: true, currentWindow: true});
- const activeTab = tabs[0];
+ function showDialog(callback) {
+ dialog.style.display = 'block';
+ overlay.style.display = 'block';
+ body.classList.add('dialog-open');
+
+ const yesButton = document.getElementById('yesButton');
+ const noButton = document.getElementById('noButton');
+
+ function handleResponse(includeHeaders) {
+ dialog.style.display = 'none';
+ overlay.style.display = 'none';
+ body.classList.remove('dialog-open');
+ callback(includeHeaders);
+ }
+
+ yesButton.onclick = () => handleResponse(true);
+ noButton.onclick = () => handleResponse(false);
+ }
- // First inject the content script with options
- await browser.tabs.executeScript(activeTab.id, {
- code: `window.keywordGrabberOptions = {
- includeHeaders: ${includeHeaders},
- extractAllPages: ${extractAllPages}
- };`
+ async function sendMessage(action, includeHeaders) {
+ try {
+ const tabs = await browser.tabs.query({active: true, currentWindow: true});
+ if (tabs[0]) {
+ await browser.tabs.sendMessage(tabs[0].id, {
+ action: action,
+ includeHeaders: includeHeaders
+ });
+ console.log(`Sent ${action} message to content script`);
+ window.close();
+ } else {
+ console.error('No active tab found');
+ }
+ } catch (error) {
+ console.error('Error sending message:', error);
+ }
+ }
+
+ grabButton.addEventListener('click', function() {
+ showDialog((includeHeaders) => {
+ sendMessage("GrabKeywords", includeHeaders);
+ });
});
- // Then execute the main content script
- await browser.tabs.executeScript(activeTab.id, {
- file: 'content-script.js'
+ semrushButton.addEventListener('click', function() {
+ showDialog((includeHeaders) => {
+ sendMessage("GrabSemrushKeywords", includeHeaders);
+ });
});
-
- // Trigger the start via a custom event
- await browser.tabs.executeScript(activeTab.id, {
- code: `document.dispatchEvent(new CustomEvent('startKeywordExtraction'));`
- });
-
- // Close the popup after initiating the content script
- window.close();
- } catch (error) {
- console.error('Error:', error);
- alert('Error: ' + error.message);
- }
-}
-
-// Add click handlers for the Yes and No buttons
-document.getElementById('yesButton').addEventListener('click', () => handleDialogChoice(true));
-document.getElementById('noButton').addEventListener('click', () => handleDialogChoice(false));
\ No newline at end of file
+});
\ No newline at end of file
diff --git a/semrush-content-script.js b/semrush-content-script.js
new file mode 100644
index 0000000..0174a35
--- /dev/null
+++ b/semrush-content-script.js
@@ -0,0 +1,343 @@
+(function() {
+ // Don't start automatically, wait for custom event
+ let hasStarted = false;
+
+ // Helper function to wait for an element to be present and visible
+ async function waitForElement(selector, timeout = 10000) {
+ const start = Date.now();
+
+ while (Date.now() - start < timeout) {
+ const element = document.querySelector(selector);
+ if (element) {
+ return element;
+ }
+ await new Promise(resolve => setTimeout(resolve, 100));
+ }
+
+ throw new Error(`Element ${selector} not found after ${timeout}ms`);
+ }
+
+ // Helper function to format search volume (remove commas)
+ function formatSearchVolume(volume) {
+ if (!volume) return 'N/A';
+ return volume.replace(/,/g, '');
+ }
+
+ // Helper function to get current page info
+ function getCurrentPageInfo() {
+ try {
+ const pageInput = document.querySelector('.sm-pagination__input input');
+ const totalPagesElement = document.querySelector('.___STotalPages_175b1-kmt_ span');
+
+ // Log the elements we found for debugging
+ console.log('Page Input Element found:', !!pageInput);
+ console.log('Total Pages Element found:', !!totalPagesElement);
+
+ if (!pageInput) {
+ // Try alternative selector for the page input
+ const altPageInput = document.querySelector('[aria-label="Current page"]');
+ if (altPageInput) {
+ console.log('Found page input using alternative selector');
+ pageInput = altPageInput;
+ } else {
+ console.log('Could not find page input element with either selector');
+ return null;
+ }
+ }
+
+ // Get the current page value
+ const currentPage = parseInt(pageInput.value);
+
+ // If we can't find the total pages element, but we know we're on page 5
+ // and the next button is disabled, we can assume we're on the last page
+ if (!totalPagesElement && currentPage === 5) {
+ const nextButton = document.querySelector('.___SNextPage_175b1-kmt_');
+ if (nextButton && (nextButton.getAttribute('aria-disabled') === 'true' ||
+ nextButton.hasAttribute('disabled') ||
+ nextButton.classList.contains('__disabled_se82q-kmt_'))) {
+ console.log('On last page (page 5) with disabled next button');
+ return { currentPage: 5, totalPages: 5 };
+ }
+ }
+
+ const totalPages = totalPagesElement ?
+ parseInt(totalPagesElement.textContent.replace(/,/g, '')) :
+ (currentPage === 5 ? 5 : null);
+
+ if (totalPages === null) {
+ console.log('Could not determine total pages');
+ return null;
+ }
+
+ // Debug the actual input field value
+ console.log(`Page Input Value: "${pageInput.value}", Parsed Current Page: ${currentPage}`);
+ console.log(`Current page: ${currentPage}, Total pages: ${totalPages}`);
+ return { currentPage, totalPages };
+ } catch (error) {
+ console.error('Error getting page info:', error);
+ return null;
+ }
+ }
+
+ // Helper function to wait for page load after navigation
+ async function waitForPageLoad(expectedPage, maxAttempts = 20) {
+ for (let i = 0; i < maxAttempts; i++) {
+ await new Promise(resolve => setTimeout(resolve, 1000)); // Increased delay
+
+ try {
+ const pageInput = document.querySelector('.sm-pagination__input input');
+ if (pageInput) {
+ const actualPage = parseInt(pageInput.value);
+ console.log(`Current page input value: "${pageInput.value}", Parsed: ${actualPage}, Expected: ${expectedPage}`);
+
+ if (actualPage === expectedPage) {
+ console.log(`Successfully loaded page ${expectedPage}`);
+ // Wait for table content to load
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ return true;
+ }
+ } else {
+ console.log('Page input element not found during wait');
+ }
+ } catch (error) {
+ console.error('Error checking page during wait:', error);
+ }
+
+ console.log(`Waiting for page ${expectedPage} to load (attempt ${i + 1}/${maxAttempts})`);
+ }
+ return false;
+ }
+
+ // Helper function to go to next page
+ async function goToNextPage(currentPage, totalPages) {
+ if (currentPage >= totalPages) {
+ console.log(`Already on last page (${currentPage} of ${totalPages})`);
+ return false;
+ }
+
+ const nextButton = document.querySelector('.___SNextPage_175b1-kmt_');
+ if (!nextButton) {
+ console.log('Next button not found');
+ return false;
+ }
+
+ const isDisabled = nextButton.getAttribute('aria-disabled') === 'true' ||
+ nextButton.hasAttribute('disabled') ||
+ nextButton.classList.contains('__disabled_se82q-kmt_');
+
+ if (isDisabled) {
+ console.log('Next button is disabled');
+ return false;
+ }
+
+ const expectedNextPage = currentPage + 1;
+ console.log(`Attempting to navigate from page ${currentPage} to page ${expectedNextPage}`);
+
+ // Get current page info before clicking
+ const beforeClick = getCurrentPageInfo();
+ nextButton.click();
+
+ const updated = await waitForPageLoad(expectedNextPage);
+ if (!updated) {
+ // Double check current page after failed update
+ const afterClick = getCurrentPageInfo();
+ console.log('Navigation check:', {
+ beforeClick: beforeClick ? beforeClick.currentPage : 'unknown',
+ expectedPage: expectedNextPage,
+ afterClick: afterClick ? afterClick.currentPage : 'unknown'
+ });
+ return false;
+ }
+
+ return true;
+ }
+
+ // Helper function to extract current page
+ async function extractCurrentPage() {
+ try {
+ // Get current page info for verification
+ const pageInfo = getCurrentPageInfo();
+ if (!pageInfo) {
+ // If we can't get page info but we can find rows, continue anyway
+ const rows = document.querySelectorAll('.sm-table-layout__row');
+ if (rows.length > 0) {
+ console.log('Proceeding with extraction despite missing page info. Found rows:', rows.length);
+ } else {
+ console.log('Could not verify current page during extraction and found no rows');
+ return null;
+ }
+ }
+
+ // Wait for table to be present
+ await new Promise(resolve => setTimeout(resolve, 1000));
+
+ const rows = document.querySelectorAll('.sm-table-layout__row');
+ const keywords = [];
+
+ if (pageInfo) {
+ console.log(`Found ${rows.length} rows to process on page ${pageInfo.currentPage}`);
+ } else {
+ console.log(`Found ${rows.length} rows to process`);
+ }
+
+ for (const row of rows) {
+ try {
+ const keywordElement = row.querySelector('.sm-cell-phrase__link span');
+ if (!keywordElement) continue;
+
+ const keyword = keywordElement.textContent.trim();
+
+ const volumeElement = row.querySelector('[data-testid="table-cell-volume"]');
+ const volume = volumeElement ? formatSearchVolume(volumeElement.textContent.trim()) : 'N/A';
+
+ if (keyword) {
+ keywords.push({
+ keyword: keyword,
+ volume: volume
+ });
+ }
+ } catch (error) {
+ console.error('Error processing row:', error);
+ }
+ }
+
+ if (pageInfo) {
+ console.log(`Extracted ${keywords.length} keywords from page ${pageInfo.currentPage}`);
+ } else {
+ console.log(`Extracted ${keywords.length} keywords from current page`);
+ }
+ return keywords;
+ } catch (error) {
+ console.error('Error extracting current page:', error);
+ return null;
+ }
+ }
+
+ // Helper function to copy text to clipboard
+ function copyToClipboard(text) {
+ const textArea = document.createElement('textarea');
+ textArea.style.position = 'fixed';
+ textArea.style.top = '0';
+ textArea.style.left = '0';
+ textArea.style.width = '2em';
+ textArea.style.height = '2em';
+ textArea.style.padding = '0';
+ textArea.style.border = 'none';
+ textArea.style.outline = 'none';
+ textArea.style.boxShadow = 'none';
+ textArea.style.background = 'transparent';
+ textArea.value = text;
+
+ document.body.appendChild(textArea);
+ textArea.focus();
+ textArea.select();
+
+ try {
+ document.execCommand('copy');
+ console.log('Text copied to clipboard');
+ } catch (err) {
+ console.error('Failed to copy text:', err);
+ }
+
+ document.body.removeChild(textArea);
+ }
+
+ // Main function to extract keywords
+ async function extractKeywords() {
+ try {
+ const options = window.keywordGrabberOptions || { includeHeaders: true };
+ const allKeywords = [];
+
+ const initialPageInfo = getCurrentPageInfo();
+ if (!initialPageInfo) {
+ throw new Error('Could not determine page information');
+ }
+
+ console.log(`Starting extraction from page ${initialPageInfo.currentPage} of ${initialPageInfo.totalPages}`);
+
+ let currentPage = initialPageInfo.currentPage;
+ const totalPages = initialPageInfo.totalPages;
+
+ do {
+ // Extract current page
+ const keywords = await extractCurrentPage();
+ if (!keywords) {
+ throw new Error('Failed to extract keywords from current page');
+ }
+
+ allKeywords.push(...keywords);
+ console.log(`Total keywords collected: ${allKeywords.length} (Page ${currentPage} of ${totalPages})`);
+
+ if (currentPage >= totalPages) {
+ console.log('Reached last page');
+ break;
+ }
+
+ const success = await goToNextPage(currentPage, totalPages);
+ if (!success) {
+ console.log('Failed to navigate to next page');
+ break;
+ }
+ currentPage++;
+
+ } while (true);
+
+ // Convert to CSV without quotes
+ let csv = '';
+ if (options.includeHeaders) {
+ csv = 'Keyword,Search Volume\n';
+ }
+
+ csv += allKeywords.map(k => `${k.keyword},${k.volume}`).join('\n');
+
+ // Copy to clipboard
+ copyToClipboard(csv);
+ console.log(`Extraction complete. Total keywords: ${allKeywords.length}`);
+
+ // Create notification
+ const div = document.createElement('div');
+ div.style.position = 'fixed';
+ div.style.top = '20px';
+ div.style.left = '50%';
+ div.style.transform = 'translateX(-50%)';
+ div.style.backgroundColor = '#4CAF50';
+ div.style.color = 'white';
+ div.style.padding = '15px';
+ div.style.borderRadius = '5px';
+ div.style.zIndex = '10000';
+ div.textContent = `${allKeywords.length} keywords copied to clipboard!`;
+
+ document.body.appendChild(div);
+ setTimeout(() => div.remove(), 3000);
+
+ } catch (error) {
+ console.error('Error during keyword extraction:', error);
+ throw error;
+ }
+ }
+
+ // Listen for messages from the popup
+ browser.runtime.onMessage.addListener((message) => {
+ if (message.action === "GrabSemrushKeywords") {
+ window.keywordGrabberOptions = {
+ includeHeaders: message.includeHeaders
+ };
+ // Trigger the start via a custom event
+ document.dispatchEvent(new CustomEvent('GrabSemrushKeywords'));
+ }
+ });
+
+ // Listen for the start event
+ document.addEventListener('GrabSemrushKeywords', async function(e) {
+ if (hasStarted) return;
+ hasStarted = true;
+
+ try {
+ await extractKeywords();
+ } catch (error) {
+ console.error('Error during SEMRush keyword extraction:', error);
+ } finally {
+ hasStarted = false;
+ }
+ });
+})();