mgk/firefox-keyword-grab
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Got the SEMRush grabber working, but we are missing progress bars.

Browse source
This commit is contained in:
Lord_Devi 2024-11-21 10:40:07 -05:00
parent a93f66ba69
commit a2e9e5eef1
5 changed files with 426 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
content-script.js
View file

@ -704,4 +704,15 @@
createNotification('Error: ' + error.message, 5000, true);
}
}
// Listen for messages from the popup
browser.runtime.onMessage.addListener((message) => {
if (message.action === "GrabKeywords") {
window.keywordGrabberOptions = {
includeHeaders: message.includeHeaders
};
// Trigger the start via a custom event
document.dispatchEvent(new CustomEvent('startKeywordExtraction'));
}
});
})();

21
manifest.json
View file

@ -1,12 +1,27 @@
{
"manifest_version": 2,
"name": "Keyword Scraper",
"name": "Keyword Grabber",
"version": "1.0",
"description": "Scrapes keywords from Google Keyword Planner",
"description": "Extract keywords from Google Keyword Planner and SEMRush",
"permissions": [
"activeTab",
"clipboardWrite",
"*://*.google.com/*"
"*://*.google.com/*",
"*://*.semrush.com/*"
],
"background": {
"scripts": ["background.js"],
"persistent": false
},
"content_scripts": [
{
"matches": ["*://ads.google.com/*"],
"js": ["content-script.js"]
},
{
"matches": ["*://*.semrush.com/*"],
"js": ["semrush-content-script.js"]
}
],
"browser_action": {
"default_popup": "popup.html"

5
popup.html
View file

@ -13,7 +13,7 @@
width: 300px;
height: 200px;
}
#grabKeywords {
#grabKeywords, #grabSemrushKeywords {
width: 100%;
padding: 8px;
margin: 5px 0;
@ -23,7 +23,7 @@
border: none;
border-radius: 4px;
}
#grabKeywords:hover {
#grabKeywords:hover, #grabSemrushKeywords:hover {
background-color: #45a049;
}
#dialog {
@ -89,6 +89,7 @@
</head>
<body>
<button id="grabKeywords">Grab GKWP Keywords</button>
<button id="grabSemrushKeywords">Grab SEMRush Keywords</button>
<div id="overlay"></div>
<div id="dialog">

98
popup.js
View file

@ -1,52 +1,56 @@
document.getElementById('grabKeywords').addEventListener('click', function() {
// Add class to body to expand it
document.body.classList.add('dialog-open');
// Show the dialog and overlay
document.getElementById('dialog').style.display = 'block';
document.getElementById('overlay').style.display = 'block';
});
async function handleDialogChoice(includeHeaders) {
try {
// Get extract all pages preference
const extractAllPages = document.getElementById('extractAllPages').checked;
// Hide the dialog and overlay
document.getElementById('dialog').style.display = 'none';
document.getElementById('overlay').style.display = 'none';
// Remove the expanded class
document.body.classList.remove('dialog-open');
document.addEventListener('DOMContentLoaded', function() {
const grabButton = document.getElementById('grabKeywords');
const semrushButton = document.getElementById('grabSemrushKeywords');
const dialog = document.getElementById('dialog');
const overlay = document.getElementById('overlay');
const body = document.body;
// Get the active tab
const tabs = await browser.tabs.query({active: true, currentWindow: true});
const activeTab = tabs[0];
function showDialog(callback) {
dialog.style.display = 'block';
overlay.style.display = 'block';
body.classList.add('dialog-open');
const yesButton = document.getElementById('yesButton');
const noButton = document.getElementById('noButton');
function handleResponse(includeHeaders) {
dialog.style.display = 'none';
overlay.style.display = 'none';
body.classList.remove('dialog-open');
callback(includeHeaders);
}
yesButton.onclick = () => handleResponse(true);
noButton.onclick = () => handleResponse(false);
}
// First inject the content script with options
await browser.tabs.executeScript(activeTab.id, {
code: `window.keywordGrabberOptions = {
includeHeaders: ${includeHeaders},
extractAllPages: ${extractAllPages}
};`
async function sendMessage(action, includeHeaders) {
try {
const tabs = await browser.tabs.query({active: true, currentWindow: true});
if (tabs[0]) {
await browser.tabs.sendMessage(tabs[0].id, {
action: action,
includeHeaders: includeHeaders
});
console.log(`Sent ${action} message to content script`);
window.close();
} else {
console.error('No active tab found');
}
} catch (error) {
console.error('Error sending message:', error);
}
}
grabButton.addEventListener('click', function() {
showDialog((includeHeaders) => {
sendMessage("GrabKeywords", includeHeaders);
});
});
// Then execute the main content script
await browser.tabs.executeScript(activeTab.id, {
file: 'content-script.js'
semrushButton.addEventListener('click', function() {
showDialog((includeHeaders) => {
sendMessage("GrabSemrushKeywords", includeHeaders);
});
});
// Trigger the start via a custom event
await browser.tabs.executeScript(activeTab.id, {
code: `document.dispatchEvent(new CustomEvent('startKeywordExtraction'));`
});
// Close the popup after initiating the content script
window.close();
} catch (error) {
console.error('Error:', error);
alert('Error: ' + error.message);
}
}
// Add click handlers for the Yes and No buttons
document.getElementById('yesButton').addEventListener('click', () => handleDialogChoice(true));
document.getElementById('noButton').addEventListener('click', () => handleDialogChoice(false));
});

343
semrush-content-script.js Normal file
View file

@ -0,0 +1,343 @@
(function() {
// Don't start automatically, wait for custom event
let hasStarted = false;
// Helper function to wait for an element to be present and visible
async function waitForElement(selector, timeout = 10000) {
const start = Date.now();
while (Date.now() - start < timeout) {
const element = document.querySelector(selector);
if (element) {
return element;
}
await new Promise(resolve => setTimeout(resolve, 100));
}
throw new Error(`Element ${selector} not found after ${timeout}ms`);
}
// Helper function to format search volume (remove commas)
function formatSearchVolume(volume) {
if (!volume) return 'N/A';
return volume.replace(/,/g, '');
}
// Helper function to get current page info
function getCurrentPageInfo() {
try {
const pageInput = document.querySelector('.sm-pagination__input input');
const totalPagesElement = document.querySelector('.___STotalPages_175b1-kmt_ span');
// Log the elements we found for debugging
console.log('Page Input Element found:', !!pageInput);
console.log('Total Pages Element found:', !!totalPagesElement);
if (!pageInput) {
// Try alternative selector for the page input
const altPageInput = document.querySelector('[aria-label="Current page"]');
if (altPageInput) {
console.log('Found page input using alternative selector');
pageInput = altPageInput;
} else {
console.log('Could not find page input element with either selector');
return null;
}
}
// Get the current page value
const currentPage = parseInt(pageInput.value);
// If we can't find the total pages element, but we know we're on page 5
// and the next button is disabled, we can assume we're on the last page
if (!totalPagesElement && currentPage === 5) {
const nextButton = document.querySelector('.___SNextPage_175b1-kmt_');
if (nextButton && (nextButton.getAttribute('aria-disabled') === 'true' ||
nextButton.hasAttribute('disabled') ||
nextButton.classList.contains('__disabled_se82q-kmt_'))) {
console.log('On last page (page 5) with disabled next button');
return { currentPage: 5, totalPages: 5 };
}
}
const totalPages = totalPagesElement ?
parseInt(totalPagesElement.textContent.replace(/,/g, '')) :
(currentPage === 5 ? 5 : null);
if (totalPages === null) {
console.log('Could not determine total pages');
return null;
}
// Debug the actual input field value
console.log(`Page Input Value: "${pageInput.value}", Parsed Current Page: ${currentPage}`);
console.log(`Current page: ${currentPage}, Total pages: ${totalPages}`);
return { currentPage, totalPages };
} catch (error) {
console.error('Error getting page info:', error);
return null;
}
}
// Helper function to wait for page load after navigation
async function waitForPageLoad(expectedPage, maxAttempts = 20) {
for (let i = 0; i < maxAttempts; i++) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Increased delay
try {
const pageInput = document.querySelector('.sm-pagination__input input');
if (pageInput) {
const actualPage = parseInt(pageInput.value);
console.log(`Current page input value: "${pageInput.value}", Parsed: ${actualPage}, Expected: ${expectedPage}`);
if (actualPage === expectedPage) {
console.log(`Successfully loaded page ${expectedPage}`);
// Wait for table content to load
await new Promise(resolve => setTimeout(resolve, 2000));
return true;
}
} else {
console.log('Page input element not found during wait');
}
} catch (error) {
console.error('Error checking page during wait:', error);
}
console.log(`Waiting for page ${expectedPage} to load (attempt ${i + 1}/${maxAttempts})`);
}
return false;
}
// Helper function to go to next page
async function goToNextPage(currentPage, totalPages) {
if (currentPage >= totalPages) {
console.log(`Already on last page (${currentPage} of ${totalPages})`);
return false;
}
const nextButton = document.querySelector('.___SNextPage_175b1-kmt_');
if (!nextButton) {
console.log('Next button not found');
return false;
}
const isDisabled = nextButton.getAttribute('aria-disabled') === 'true' ||
nextButton.hasAttribute('disabled') ||
nextButton.classList.contains('__disabled_se82q-kmt_');
if (isDisabled) {
console.log('Next button is disabled');
return false;
}
const expectedNextPage = currentPage + 1;
console.log(`Attempting to navigate from page ${currentPage} to page ${expectedNextPage}`);
// Get current page info before clicking
const beforeClick = getCurrentPageInfo();
nextButton.click();
const updated = await waitForPageLoad(expectedNextPage);
if (!updated) {
// Double check current page after failed update
const afterClick = getCurrentPageInfo();
console.log('Navigation check:', {
beforeClick: beforeClick ? beforeClick.currentPage : 'unknown',
expectedPage: expectedNextPage,
afterClick: afterClick ? afterClick.currentPage : 'unknown'
});
return false;
}
return true;
}
// Helper function to extract current page
async function extractCurrentPage() {
try {
// Get current page info for verification
const pageInfo = getCurrentPageInfo();
if (!pageInfo) {
// If we can't get page info but we can find rows, continue anyway
const rows = document.querySelectorAll('.sm-table-layout__row');
if (rows.length > 0) {
console.log('Proceeding with extraction despite missing page info. Found rows:', rows.length);
} else {
console.log('Could not verify current page during extraction and found no rows');
return null;
}
}
// Wait for table to be present
await new Promise(resolve => setTimeout(resolve, 1000));
const rows = document.querySelectorAll('.sm-table-layout__row');
const keywords = [];
if (pageInfo) {
console.log(`Found ${rows.length} rows to process on page ${pageInfo.currentPage}`);
} else {
console.log(`Found ${rows.length} rows to process`);
}
for (const row of rows) {
try {
const keywordElement = row.querySelector('.sm-cell-phrase__link span');
if (!keywordElement) continue;
const keyword = keywordElement.textContent.trim();
const volumeElement = row.querySelector('[data-testid="table-cell-volume"]');
const volume = volumeElement ? formatSearchVolume(volumeElement.textContent.trim()) : 'N/A';
if (keyword) {
keywords.push({
keyword: keyword,
volume: volume
});
}
} catch (error) {
console.error('Error processing row:', error);
}
}
if (pageInfo) {
console.log(`Extracted ${keywords.length} keywords from page ${pageInfo.currentPage}`);
} else {
console.log(`Extracted ${keywords.length} keywords from current page`);
}
return keywords;
} catch (error) {
console.error('Error extracting current page:', error);
return null;
}
}
// Helper function to copy text to clipboard
function copyToClipboard(text) {
const textArea = document.createElement('textarea');
textArea.style.position = 'fixed';
textArea.style.top = '0';
textArea.style.left = '0';
textArea.style.width = '2em';
textArea.style.height = '2em';
textArea.style.padding = '0';
textArea.style.border = 'none';
textArea.style.outline = 'none';
textArea.style.boxShadow = 'none';
textArea.style.background = 'transparent';
textArea.value = text;
document.body.appendChild(textArea);
textArea.focus();
textArea.select();
try {
document.execCommand('copy');
console.log('Text copied to clipboard');
} catch (err) {
console.error('Failed to copy text:', err);
}
document.body.removeChild(textArea);
}
// Main function to extract keywords
async function extractKeywords() {
try {
const options = window.keywordGrabberOptions || { includeHeaders: true };
const allKeywords = [];
const initialPageInfo = getCurrentPageInfo();
if (!initialPageInfo) {
throw new Error('Could not determine page information');
}
console.log(`Starting extraction from page ${initialPageInfo.currentPage} of ${initialPageInfo.totalPages}`);
let currentPage = initialPageInfo.currentPage;
const totalPages = initialPageInfo.totalPages;
do {
// Extract current page
const keywords = await extractCurrentPage();
if (!keywords) {
throw new Error('Failed to extract keywords from current page');
}
allKeywords.push(...keywords);
console.log(`Total keywords collected: ${allKeywords.length} (Page ${currentPage} of ${totalPages})`);
if (currentPage >= totalPages) {
console.log('Reached last page');
break;
}
const success = await goToNextPage(currentPage, totalPages);
if (!success) {
console.log('Failed to navigate to next page');
break;
}
currentPage++;
} while (true);
// Convert to CSV without quotes
let csv = '';
if (options.includeHeaders) {
csv = 'Keyword,Search Volume\n';
}
csv += allKeywords.map(k => `${k.keyword},${k.volume}`).join('\n');
// Copy to clipboard
copyToClipboard(csv);
console.log(`Extraction complete. Total keywords: ${allKeywords.length}`);
// Create notification
const div = document.createElement('div');
div.style.position = 'fixed';
div.style.top = '20px';
div.style.left = '50%';
div.style.transform = 'translateX(-50%)';
div.style.backgroundColor = '#4CAF50';
div.style.color = 'white';
div.style.padding = '15px';
div.style.borderRadius = '5px';
div.style.zIndex = '10000';
div.textContent = `${allKeywords.length} keywords copied to clipboard!`;
document.body.appendChild(div);
setTimeout(() => div.remove(), 3000);
} catch (error) {
console.error('Error during keyword extraction:', error);
throw error;
}
}
// Listen for messages from the popup
browser.runtime.onMessage.addListener((message) => {
if (message.action === "GrabSemrushKeywords") {
window.keywordGrabberOptions = {
includeHeaders: message.includeHeaders
};
// Trigger the start via a custom event
document.dispatchEvent(new CustomEvent('GrabSemrushKeywords'));
}
});
// Listen for the start event
document.addEventListener('GrabSemrushKeywords', async function(e) {
if (hasStarted) return;
hasStarted = true;
try {
await extractKeywords();
} catch (error) {
console.error('Error during SEMRush keyword extraction:', error);
} finally {
hasStarted = false;
}
});
})();