536 lines
20 KiB
JavaScript
536 lines
20 KiB
JavaScript
(function() {
|
|
// Don't start automatically, wait for custom event
|
|
let hasStarted = false;
|
|
|
|
// Helper function to wait for an element to be present and visible
|
|
async function waitForElement(selector, timeout = 10000) {
|
|
const start = Date.now();
|
|
|
|
while (Date.now() - start < timeout) {
|
|
const element = document.querySelector(selector);
|
|
if (element) {
|
|
return element;
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
}
|
|
|
|
throw new Error(`Element ${selector} not found after ${timeout}ms`);
|
|
}
|
|
|
|
// Helper function to format search volume (remove commas)
|
|
function formatSearchVolume(volume) {
|
|
if (!volume) return 'N/A';
|
|
return volume.replace(/,/g, '');
|
|
}
|
|
|
|
// Helper function to get current page info
|
|
function getCurrentPageInfo() {
|
|
try {
|
|
// Try multiple selectors for pagination elements
|
|
const pageInputSelectors = [
|
|
'.sm-pagination__input input',
|
|
'[aria-label="Current page"]',
|
|
'input[type="number"][aria-label*="page"]',
|
|
'.pagination input[type="number"]'
|
|
];
|
|
|
|
const totalPagesSelectors = [
|
|
'.___STotalPages_175b1-kmt_ span',
|
|
'.pagination__total',
|
|
'.pagination__pages-total',
|
|
'.sm-pagination__total'
|
|
];
|
|
|
|
// Find page input
|
|
let pageInput = null;
|
|
for (const selector of pageInputSelectors) {
|
|
pageInput = document.querySelector(selector);
|
|
if (pageInput) {
|
|
console.log(`Found page input using selector: ${selector}`);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Find total pages element
|
|
let totalPagesElement = null;
|
|
for (const selector of totalPagesSelectors) {
|
|
totalPagesElement = document.querySelector(selector);
|
|
if (totalPagesElement) {
|
|
console.log(`Found total pages using selector: ${selector}`);
|
|
break;
|
|
}
|
|
}
|
|
|
|
console.log('Page Input Element found:', !!pageInput);
|
|
console.log('Total Pages Element found:', !!totalPagesElement);
|
|
|
|
if (!pageInput) {
|
|
console.log('Could not find page input element with any selector');
|
|
return null;
|
|
}
|
|
|
|
// Get the current page value
|
|
const currentPage = parseInt(pageInput.value);
|
|
if (isNaN(currentPage)) {
|
|
console.log('Invalid current page value:', pageInput.value);
|
|
return null;
|
|
}
|
|
|
|
let totalPages = null;
|
|
|
|
// Try to get total pages from element
|
|
if (totalPagesElement) {
|
|
const totalText = totalPagesElement.textContent.trim();
|
|
totalPages = parseInt(totalText.replace(/[^\d]/g, ''));
|
|
}
|
|
|
|
// If we can't find total pages but we're on page 5 with disabled next button
|
|
if (totalPages === null && currentPage === 5) {
|
|
const nextButtonSelectors = [
|
|
'.___SNextPage_175b1-kmt_',
|
|
'.sm-pagination__next',
|
|
'.pagination__next',
|
|
'button[aria-label*="next"]'
|
|
];
|
|
|
|
for (const selector of nextButtonSelectors) {
|
|
const nextButton = document.querySelector(selector);
|
|
if (nextButton && (
|
|
nextButton.getAttribute('aria-disabled') === 'true' ||
|
|
nextButton.hasAttribute('disabled') ||
|
|
nextButton.classList.contains('__disabled_se82q-kmt_')
|
|
)) {
|
|
console.log('On last page (page 5) with disabled next button');
|
|
totalPages = 5;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we still can't determine total pages but have current page
|
|
if (totalPages === null && currentPage > 0) {
|
|
console.log('Using current page as minimum total pages');
|
|
totalPages = currentPage;
|
|
}
|
|
|
|
console.log(`Current page: ${currentPage}, Total pages: ${totalPages}`);
|
|
return { currentPage, totalPages };
|
|
|
|
} catch (error) {
|
|
console.error('Error getting page info:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Helper function to wait for page load after navigation
|
|
async function waitForPageLoad(expectedPage, maxAttempts = 20) {
|
|
for (let i = 0; i < maxAttempts; i++) {
|
|
await new Promise(resolve => setTimeout(resolve, 1000)); // Increased delay
|
|
|
|
try {
|
|
const pageInput = document.querySelector('.sm-pagination__input input');
|
|
if (pageInput) {
|
|
const actualPage = parseInt(pageInput.value);
|
|
console.log(`Current page input value: "${pageInput.value}", Parsed: ${actualPage}, Expected: ${expectedPage}`);
|
|
|
|
if (actualPage === expectedPage) {
|
|
console.log(`Successfully loaded page ${expectedPage}`);
|
|
// Wait for table content to load
|
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
return true;
|
|
}
|
|
} else {
|
|
console.log('Page input element not found during wait');
|
|
}
|
|
} catch (error) {
|
|
console.error('Error checking page during wait:', error);
|
|
}
|
|
|
|
console.log(`Waiting for page ${expectedPage} to load (attempt ${i + 1}/${maxAttempts})`);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Helper function to go to next page
|
|
async function goToNextPage(currentPage, totalPages) {
|
|
if (currentPage >= totalPages) {
|
|
console.log(`Already on last page (${currentPage} of ${totalPages})`);
|
|
return false;
|
|
}
|
|
|
|
const nextButton = document.querySelector('.___SNextPage_175b1-kmt_');
|
|
if (!nextButton) {
|
|
console.log('Next button not found');
|
|
return false;
|
|
}
|
|
|
|
const isDisabled = nextButton.getAttribute('aria-disabled') === 'true' ||
|
|
nextButton.hasAttribute('disabled') ||
|
|
nextButton.classList.contains('__disabled_se82q-kmt_');
|
|
|
|
if (isDisabled) {
|
|
console.log('Next button is disabled');
|
|
return false;
|
|
}
|
|
|
|
const expectedNextPage = currentPage + 1;
|
|
console.log(`Attempting to navigate from page ${currentPage} to page ${expectedNextPage}`);
|
|
|
|
// Get current page info before clicking
|
|
const beforeClick = getCurrentPageInfo();
|
|
nextButton.click();
|
|
|
|
const updated = await waitForPageLoad(expectedNextPage);
|
|
if (!updated) {
|
|
// Double check current page after failed update
|
|
const afterClick = getCurrentPageInfo();
|
|
console.log('Navigation check:', {
|
|
beforeClick: beforeClick ? beforeClick.currentPage : 'unknown',
|
|
expectedPage: expectedNextPage,
|
|
afterClick: afterClick ? afterClick.currentPage : 'unknown'
|
|
});
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// Helper function to extract current page
|
|
async function extractCurrentPage() {
|
|
try {
|
|
// Get current page info for verification
|
|
const pageInfo = getCurrentPageInfo();
|
|
if (!pageInfo) {
|
|
// If we can't get page info but we can find rows, continue anyway
|
|
const rows = document.querySelectorAll('.sm-table-layout__row');
|
|
if (rows.length > 0) {
|
|
console.log('Proceeding with extraction despite missing page info. Found rows:', rows.length);
|
|
} else {
|
|
console.log('Could not verify current page during extraction and found no rows');
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Wait for table to be present
|
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
|
|
const rows = document.querySelectorAll('.sm-table-layout__row');
|
|
const keywords = [];
|
|
|
|
if (pageInfo) {
|
|
console.log(`Found ${rows.length} rows to process on page ${pageInfo.currentPage}`);
|
|
} else {
|
|
console.log(`Found ${rows.length} rows to process`);
|
|
}
|
|
|
|
for (const row of rows) {
|
|
try {
|
|
const keywordElement = row.querySelector('.sm-cell-phrase__link span');
|
|
if (!keywordElement) continue;
|
|
|
|
const keyword = keywordElement.textContent.trim();
|
|
|
|
const volumeElement = row.querySelector('[data-testid="table-cell-volume"]');
|
|
const volume = volumeElement ? formatSearchVolume(volumeElement.textContent.trim()) : 'N/A';
|
|
|
|
if (keyword) {
|
|
keywords.push({
|
|
keyword: keyword,
|
|
volume: volume
|
|
});
|
|
}
|
|
} catch (error) {
|
|
console.error('Error processing row:', error);
|
|
}
|
|
}
|
|
|
|
if (pageInfo) {
|
|
console.log(`Extracted ${keywords.length} keywords from page ${pageInfo.currentPage}`);
|
|
} else {
|
|
console.log(`Extracted ${keywords.length} keywords from current page`);
|
|
}
|
|
return keywords;
|
|
} catch (error) {
|
|
console.error('Error extracting current page:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Helper function to copy text to clipboard
|
|
function copyToClipboard(text) {
|
|
const textArea = document.createElement('textarea');
|
|
textArea.style.position = 'fixed';
|
|
textArea.style.top = '0';
|
|
textArea.style.left = '0';
|
|
textArea.style.width = '2em';
|
|
textArea.style.height = '2em';
|
|
textArea.style.padding = '0';
|
|
textArea.style.border = 'none';
|
|
textArea.style.outline = 'none';
|
|
textArea.style.boxShadow = 'none';
|
|
textArea.style.background = 'transparent';
|
|
textArea.value = text;
|
|
|
|
document.body.appendChild(textArea);
|
|
textArea.focus();
|
|
textArea.select();
|
|
|
|
try {
|
|
document.execCommand('copy');
|
|
console.log('Text copied to clipboard');
|
|
} catch (err) {
|
|
console.error('Failed to copy text:', err);
|
|
}
|
|
|
|
document.body.removeChild(textArea);
|
|
}
|
|
|
|
// Helper function to format keywords
|
|
function formatKeywords(keywords, includeHeaders) {
|
|
let csv = '';
|
|
if (includeHeaders) {
|
|
csv = 'Keyword,Search Volume\n';
|
|
}
|
|
|
|
csv += keywords.map(k => `${k.keyword},${k.volume}`).join('\n');
|
|
return csv;
|
|
}
|
|
|
|
// Helper function to create notification
|
|
function createNotification(message, duration = 3000, isError = false) {
|
|
const notification = document.createElement('div');
|
|
notification.style.cssText = `
|
|
position: fixed;
|
|
bottom: 20px;
|
|
right: 20px;
|
|
padding: 12px 24px;
|
|
background: ${isError ? '#f44336' : '#4CAF50'};
|
|
color: white;
|
|
border-radius: 4px;
|
|
font-family: Arial, sans-serif;
|
|
font-size: 14px;
|
|
z-index: 10000;
|
|
box-shadow: 0 2px 5px rgba(0,0,0,0.2);
|
|
transition: opacity 0.3s ease-in-out;
|
|
`;
|
|
notification.textContent = message;
|
|
document.body.appendChild(notification);
|
|
|
|
// Fade out and remove
|
|
setTimeout(() => {
|
|
notification.style.opacity = '0';
|
|
setTimeout(() => notification.remove(), 300);
|
|
}, duration);
|
|
}
|
|
|
|
// Dialog class for page count input
|
|
class PageCountDialog {
|
|
constructor() {
|
|
this.dialog = null;
|
|
}
|
|
|
|
create() {
|
|
// Create dialog container
|
|
this.dialog = document.createElement('div');
|
|
this.dialog.style.cssText = `
|
|
position: fixed;
|
|
top: 50%;
|
|
left: 50%;
|
|
transform: translate(-50%, -50%);
|
|
background: white;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
|
|
z-index: 10000;
|
|
font-family: Arial, sans-serif;
|
|
min-width: 300px;
|
|
`;
|
|
|
|
// Create content
|
|
this.dialog.innerHTML = `
|
|
<h2 style="margin: 0 0 15px 0; font-size: 16px;">SEMRush Keyword Extraction</h2>
|
|
<p style="margin: 0 0 15px 0; font-size: 14px;">How many pages would you like to scrape?</p>
|
|
<input type="number" min="1" value="5" style="
|
|
width: 100%;
|
|
padding: 8px;
|
|
margin-bottom: 15px;
|
|
border: 1px solid #ddd;
|
|
border-radius: 4px;
|
|
box-sizing: border-box;
|
|
">
|
|
<div style="text-align: right;">
|
|
<button class="cancel" style="
|
|
padding: 8px 15px;
|
|
margin-right: 10px;
|
|
border: none;
|
|
border-radius: 4px;
|
|
background: #f0f0f0;
|
|
cursor: pointer;
|
|
">Cancel</button>
|
|
<button class="confirm" style="
|
|
padding: 8px 15px;
|
|
border: none;
|
|
border-radius: 4px;
|
|
background: #4CAF50;
|
|
color: white;
|
|
cursor: pointer;
|
|
">Start Extraction</button>
|
|
</div>
|
|
`;
|
|
|
|
// Add overlay
|
|
const overlay = document.createElement('div');
|
|
overlay.style.cssText = `
|
|
position: fixed;
|
|
top: 0;
|
|
left: 0;
|
|
right: 0;
|
|
bottom: 0;
|
|
background: rgba(0, 0, 0, 0.5);
|
|
z-index: 9999;
|
|
`;
|
|
|
|
// Add to document
|
|
document.body.appendChild(overlay);
|
|
document.body.appendChild(this.dialog);
|
|
|
|
// Return promise that resolves with page count or null if cancelled
|
|
return new Promise((resolve) => {
|
|
const input = this.dialog.querySelector('input');
|
|
const confirmBtn = this.dialog.querySelector('.confirm');
|
|
const cancelBtn = this.dialog.querySelector('.cancel');
|
|
|
|
confirmBtn.addEventListener('click', () => {
|
|
const value = parseInt(input.value);
|
|
if (value > 0) {
|
|
this.close();
|
|
resolve(value);
|
|
}
|
|
});
|
|
|
|
cancelBtn.addEventListener('click', () => {
|
|
this.close();
|
|
resolve(null);
|
|
});
|
|
|
|
// Handle Enter key
|
|
input.addEventListener('keypress', (e) => {
|
|
if (e.key === 'Enter') {
|
|
const value = parseInt(input.value);
|
|
if (value > 0) {
|
|
this.close();
|
|
resolve(value);
|
|
}
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
close() {
|
|
if (this.dialog) {
|
|
this.dialog.previousElementSibling?.remove(); // Remove overlay
|
|
this.dialog.remove();
|
|
this.dialog = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Main function to extract keywords
|
|
async function extractKeywords() {
|
|
try {
|
|
// Show dialog to get page count
|
|
const dialog = new PageCountDialog();
|
|
const maxPages = await dialog.create();
|
|
|
|
if (!maxPages) {
|
|
console.log('Extraction cancelled by user');
|
|
return;
|
|
}
|
|
|
|
console.log(`Will extract up to ${maxPages} pages`);
|
|
|
|
// Wait for initial page load
|
|
await waitForElement('.sm-pagination__input input', 10000);
|
|
|
|
const pageInfo = getCurrentPageInfo();
|
|
if (!pageInfo) {
|
|
throw new Error('Could not get page information');
|
|
}
|
|
|
|
const { currentPage, totalPages } = pageInfo;
|
|
console.log(`Starting extraction from page ${currentPage} of ${totalPages}`);
|
|
|
|
const allKeywords = [];
|
|
let currentPageNum = currentPage;
|
|
|
|
// Create progress notification
|
|
createNotification(`Starting extraction: Page ${currentPageNum} of ${Math.min(totalPages, maxPages)}`, 2000);
|
|
|
|
// Extract keywords from each page
|
|
while (currentPageNum <= Math.min(totalPages, maxPages)) {
|
|
console.log(`Processing page ${currentPageNum}`);
|
|
|
|
const keywords = await extractCurrentPage();
|
|
if (keywords && keywords.length > 0) {
|
|
allKeywords.push(...keywords);
|
|
console.log(`Found ${keywords.length} keywords on page ${currentPageNum}`);
|
|
createNotification(`Extracted ${keywords.length} keywords from page ${currentPageNum}`, 1500);
|
|
}
|
|
|
|
if (currentPageNum >= Math.min(totalPages, maxPages)) {
|
|
console.log('Reached target page count');
|
|
break;
|
|
}
|
|
|
|
const success = await goToNextPage(currentPageNum, totalPages);
|
|
if (!success) {
|
|
console.log('Failed to go to next page');
|
|
break;
|
|
}
|
|
|
|
currentPageNum++;
|
|
}
|
|
|
|
if (allKeywords.length === 0) {
|
|
throw new Error('No keywords found');
|
|
}
|
|
|
|
// Format and copy to clipboard
|
|
const csvContent = formatKeywords(allKeywords, window.keywordGrabberOptions.includeHeaders);
|
|
await copyToClipboard(csvContent);
|
|
|
|
console.log(`Successfully extracted ${allKeywords.length} keywords`);
|
|
createNotification(`Successfully extracted ${allKeywords.length} keywords from ${currentPageNum} pages`, 4000);
|
|
|
|
} catch (error) {
|
|
console.error('Error during keyword extraction:', error);
|
|
createNotification(`Error: ${error.message}`, 5000, true);
|
|
}
|
|
}
|
|
|
|
// Listen for messages from the popup
|
|
browser.runtime.onMessage.addListener((message) => {
|
|
if (message.action === "GrabSemrushKeywords") {
|
|
window.keywordGrabberOptions = {
|
|
includeHeaders: message.includeHeaders
|
|
};
|
|
// Trigger the start via a custom event
|
|
document.dispatchEvent(new CustomEvent('GrabSemrushKeywords'));
|
|
}
|
|
});
|
|
|
|
// Listen for the start event
|
|
document.addEventListener('GrabSemrushKeywords', async function(e) {
|
|
if (hasStarted) return;
|
|
hasStarted = true;
|
|
|
|
try {
|
|
await extractKeywords();
|
|
} catch (error) {
|
|
console.error('Error during SEMRush keyword extraction:', error);
|
|
} finally {
|
|
hasStarted = false;
|
|
}
|
|
});
|
|
})();
|