First commit.

.gitignore

@@ -0,0 +1,2 @@
+*.csv
+*.backup

README.org

@@ -0,0 +1,93 @@
+# Domain Scraping Utils
+This is a collection of utilities I use to scrape various kinds of information from domains.
+
+## Step 1: Add Domains
+Obtain a list of domains you wish to scrape information from, and put it in a file named "input-domains.csv" in the project root.
+
+## Step 2: Normalize Data
+Here we are going to start off with a very fast and simple sort and deduplication of the provided domains.
+
+#+begin_src sh
+bin/normalize-data.py
+#+end_src
+
+This will create an output file named "01-normalized-data.csv".
+
+## Step 3: Alive Check
+Next we want to know which domains are actually resolvable, and deliver us web content.  Which are alive, and which are dead.
+
+#+begin_src sh
+bin/alive-check.py
+#+end_src
+
+Depending on the size of your domain list, this could take a while.
+
+The alive-check script will create an output file named "02-alive-check.csv".
+
+The output file will record the original domain used for the alive check, if the domain was found to be alive or not, and also include the full url of the website that resulted from connecting to the domain.  This will be after any redirects occurred or anything.
+
+## Step 4: Split Alives into Target Files
+I like to run my scrape on multiple servers, to cover more space in more time.
+
+To that end, I usually run a script called "split-alives" which will search the "02-alive-check.csv" file created earlier for websites which are marked as alive, and then will output them into 3 files named "target1.csv", "target2.csv", and "target3.csv".
+
+I then migrate these csv's onto my poweredges and put them to work using the process-alives script afterwards.
+
+#+begin_src sh
+bin/split-alives.py
+#+end_src
+
+## Step 5: Process Alives for Pages
+The next step in the process I use here, is to scrape all the pages from all of the domains which were found to be alive.
+
+There is a script named "scrape-pages" which will scrape all of the pages from any domain we tell it to.
+
+However I call this script in parellel using another script named "process-alives".  Which when ran, will search trhough the file "targets.csv" for sites that are alive.
+
+
+The "process-alives" script will use as input a file named "targets.csv" in the project root.  I usually set this up from one of the splits I created in step 4 above.
+
+#+begin_src sh
+bin/process-alives.py
+#+end_src
+
+This will take quite a while depending on the size of your list.
+
+It is going to create a directory off of the project root named "scrapes".  And inside this directory will becreated a sub-directory for every website being processed.
+
+Inside those sub-directories we will find the following csv files getting created:
+
+- email-addresses.csv
+- external-links.csv
+- meta-info.csv
+- pages.csv
+- phone-numbers.csv
+- social-media-links.csv
+
+That's going to be the main meat and potatoes of our scrape.  Pages, external links, phone numbers, emails, and social media.
+
+## Step 6: Extract External Domains
+This is where we close the loop and scrape all of the "external-links.csv" files for new domains to process.  Just like we did with this set.
+
+The extract external domains script will harvest a domain and website url from all of the external-links.csv files into a csv file named "extracted-external-domains.csv".  This list will also be sorted and deduplicated.
+
+#+begin_src sh
+bin/extract-external-domains.py
+#+end_src
+
+Bonus: I should probably add this to the script, but to extract only the domains from the output file in this list you can use awk as in this example below:
+
+#+begin_src sh
+awk -F',' 'NR > 1 {print $2}' extracted-external-domains.csv >domains.csv
+#+end_src
+
+## Step 7: Extract Contact Forms
+This will not do as thorough a job as some tools I've used to scrape contact forms, but for now it does a pretty good job.
+
+This stage looks through all of the "pages.csv" files we scraped earlier for URLs that could be contact us forms.
+
+These forms can be quite useful.
+
+#+begin_src sh
+bin/extract-contact-us-forms.py
+#+end_src

bin/alive-check.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+import os
+import csv
+import aiohttp
+import asyncio
+import ssl
+import certifi
+import time
+import logging
+import socket
+from aiohttp import ClientSession, TCPConnector, ClientTimeout
+from yarl import URL
+
+# Global variables for progress tracking
+total_domains = 0
+processed_domains = 0
+
+# Custom logging formatter
+class ProgressFormatter(logging.Formatter):
+    def format(self, record):
+        global total_domains, processed_domains
+        progress = (processed_domains / total_domains) * 100 if total_domains > 0 else 0
+        record.progress = f"[{progress:.2f}% - {processed_domains}/{total_domains}]"
+        return super().format(record)
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+handler = logging.StreamHandler()
+handler.setFormatter(ProgressFormatter('%(progress)s : %(message)s'))
+logger.handlers.clear()
+logger.addHandler(handler)
+
+# Determine project root dynamically
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.dirname(SCRIPT_DIR)
+
+INPUT_FILE = os.path.join(PROJECT_ROOT, "01-normalized-data.csv")
+OUTPUT_FILE = os.path.join(PROJECT_ROOT, "02-alive-check.csv")
+
+MAX_CONCURRENT = 100
+CHUNK_SIZE = 500
+
+# Timeout settings
+CONNECT_TIMEOUT = 10  # seconds
+TOTAL_TIMEOUT = 30  # seconds
+
+# Retry settings
+CHUNK_RETRIES = 3
+CHUNK_RETRY_DELAY = 60  # seconds
+
+# User agent string
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+
+# Status codes that should result in immediate "no"
+NO_RETRY_STATUS_CODES = {403}
+
+# Create a SSL context that ignores certificate validation
+ssl_context = ssl.create_default_context(cafile=certifi.where())
+ssl_context.check_hostname = False
+ssl_context.verify_mode = ssl.CERT_NONE
+
+class CSVManager:
+    def __init__(self, filename, mode='w'):
+        self.filename = filename
+        self.file = open(filename, mode, newline='', encoding='utf-8')
+        self.writer = csv.writer(self.file)
+        if mode == 'w':
+            self.writer.writerow(["domain", "alive", "website"])
+        self.lock = asyncio.Lock()
+
+    async def write_row(self, row):
+        async with self.lock:
+            self.writer.writerow(row)
+            self.file.flush()
+
+    def close(self):
+        self.file.close()
+
+async def check_website(session, domain, csv_manager):
+    global processed_domains
+    await csv_manager.write_row([domain, "checking", ""])
+    logging.info(f"Checking: {domain}")
+
+    for protocol in ['https://', 'http://']:
+        url = URL(f"{protocol}{domain}")
+        try:
+            timeout = ClientTimeout(total=TOTAL_TIMEOUT, connect=CONNECT_TIMEOUT)
+            async with session.get(url, timeout=timeout, allow_redirects=True, ssl=ssl_context) as response:
+                if response.status < 400:
+                    final_url = str(response.url)
+                    await csv_manager.write_row([domain, "yes", final_url])
+                    logging.info(f"Success: {domain} -> {final_url}")
+                    processed_domains += 1
+                    return
+                elif response.status in NO_RETRY_STATUS_CODES:
+                    await csv_manager.write_row([domain, "no", ""])
+                    logging.info(f"No retry status code {response.status}: {domain}")
+                    processed_domains += 1
+                    return
+                else:
+                    logging.info(f"HTTP Error: {url} - Status {response.status}")
+        except asyncio.TimeoutError:
+            logging.info(f"Timeout: {url}")
+        except Exception as e:
+            logging.info(f"Error: {url} - {str(e)}")
+
+    await csv_manager.write_row([domain, "no", ""])
+    logging.info(f"Failed: {domain} - Both HTTPS and HTTP attempts failed")
+    processed_domains += 1
+
+async def process_chunk(domains, csv_manager, chunk_number):
+    for retry in range(CHUNK_RETRIES):
+        try:
+            connector = TCPConnector(limit=MAX_CONCURRENT, force_close=True, enable_cleanup_closed=True, ssl=ssl_context, family=socket.AF_INET)
+            timeout = ClientTimeout(total=TOTAL_TIMEOUT)
+            async with ClientSession(connector=connector, timeout=timeout, headers={"User-Agent": USER_AGENT}) as session:
+                tasks = [check_website(session, domain, csv_manager) for domain in domains]
+                await asyncio.gather(*tasks)
+            logging.info(f"Completed chunk {chunk_number}")
+            return
+        except Exception as e:
+            logging.error(f"Error processing chunk {chunk_number}: {str(e)}")
+            if retry < CHUNK_RETRIES - 1:
+                logging.info(f"Retrying chunk {chunk_number} in {CHUNK_RETRY_DELAY} seconds...")
+                await asyncio.sleep(CHUNK_RETRY_DELAY)
+            else:
+                logging.error(f"Failed to process chunk {chunk_number} after {CHUNK_RETRIES} attempts")
+
+async def process_domains(domains, processed_domains_set):
+    global processed_domains
+    csv_manager = CSVManager(OUTPUT_FILE, mode='a')
+    
+    domains_to_process = [domain for domain in domains if domain not in processed_domains_set]
+    total_chunks = len(domains_to_process) // CHUNK_SIZE + (1 if len(domains_to_process) % CHUNK_SIZE > 0 else 0)
+
+    for i in range(0, len(domains_to_process), CHUNK_SIZE):
+        chunk = domains_to_process[i:i+CHUNK_SIZE]
+        chunk_number = i//CHUNK_SIZE + 1
+        logging.info(f"Processing chunk {chunk_number} of {total_chunks}")
+        await process_chunk(chunk, csv_manager, chunk_number)
+        logging.info(f"Pausing for 5 seconds before next chunk")
+        await asyncio.sleep(5)  # Add a small pause between chunks
+
+    csv_manager.close()
+
+def read_processed_domains():
+    processed_domains = set()
+    domains_to_recheck = set()
+    if os.path.exists(OUTPUT_FILE):
+        with open(OUTPUT_FILE, 'r', newline='', encoding='utf-8') as csvfile:
+            reader = csv.reader(csvfile)
+            next(reader)  # Skip header
+            for row in reader:
+                if len(row) >= 2:
+                    domain, status = row[0], row[1]
+                    if status in ['yes', 'no']:
+                        processed_domains.add(domain)
+                    elif status == 'checking':
+                        domains_to_recheck.add(domain)
+    return processed_domains, domains_to_recheck
+
+async def main():
+    global total_domains, processed_domains
+    logging.info("Starting alive-check.py")
+    
+    logging.info(f"Reading domains from {INPUT_FILE}")
+    with open(INPUT_FILE, 'r', encoding='utf-8') as infile:
+        reader = csv.reader(infile)
+        all_domains = [row[0] for row in reader if row]
+
+    processed_domains_set, domains_to_recheck = read_processed_domains()
+    domains_to_process = set(all_domains) - processed_domains_set
+    domains_to_process.update(domains_to_recheck)
+
+    total_domains = len(all_domains)
+    processed_domains = len(processed_domains_set)
+    logging.info(f"Loaded {total_domains} domains")
+    logging.info(f"Already processed: {processed_domains}")
+    logging.info(f"Domains to process: {len(domains_to_process)}")
+
+    start_time = time.time()
+    await process_domains(list(domains_to_process), processed_domains_set)
+    elapsed_time = time.time() - start_time
+
+    with open(OUTPUT_FILE, 'r', encoding='utf-8') as outfile:
+        reader = csv.reader(outfile)
+        next(reader)  # Skip header
+        results = list(reader)
+
+    alive_count = sum(1 for row in results if row[1] == "yes")
+    speed = len(domains_to_process) / elapsed_time if elapsed_time > 0 else 0
+    logging.info(f"Processed {len(domains_to_process)} domains in {elapsed_time:.2f} seconds")
+    logging.info(f"Found {alive_count} alive websites")
+    logging.info(f"Average speed: {speed:.2f} domains/second")
+
+    logging.info("alive-check.py completed")
+
+if __name__ == "__main__":
+    asyncio.run(main())

bin/extract-contact-us-forms.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+import os
+import pandas as pd
+from tqdm import tqdm
+import re
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+# Define the input and output directories
+INPUT_DIR = "sorted-and-deduplicated"
+OUTPUT_FILE = "extracted-contact-forms.csv"
+
+# Get the project root directory (parent directory of the bin directory)
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+INPUT_PATH = os.path.join(PROJECT_ROOT, INPUT_DIR)
+OUTPUT_PATH = os.path.join(PROJECT_ROOT, OUTPUT_FILE)
+
+# Common patterns for contact pages
+CONTACT_PATTERNS = [
+    re.compile(r'contact$', re.IGNORECASE),
+    re.compile(r'contact-us$', re.IGNORECASE),
+    re.compile(r'contactus$', re.IGNORECASE),
+    re.compile(r'contact_me$', re.IGNORECASE),
+    re.compile(r'kontakt$', re.IGNORECASE),
+    re.compile(r'impressum$', re.IGNORECASE)
+]
+
+def is_contact_page(url):
+    for pattern in CONTACT_PATTERNS:
+        if pattern.search(url):
+            return True
+    return False
+
+def process_pages_file(file_path):
+    try:
+        df = pd.read_csv(file_path, usecols=["final url"])
+        df["is_contact_page"] = df["final url"].apply(is_contact_page)
+        contact_pages = df[df["is_contact_page"]]["final url"]
+        return contact_pages
+    except Exception as e:
+        print(f"Error processing file: {file_path}, Error: {e}")
+        return pd.Series()
+
+def process_contact_forms():
+    all_contact_pages = []
+    total_files = 0
+
+    for root, dirs, files in os.walk(INPUT_PATH):
+        for file in files:
+            if file == "pages.csv":
+                total_files += 1
+
+    progress_bar = tqdm(total=total_files, desc="Processing pages.csv files", unit="file")
+    
+    with ProcessPoolExecutor() as executor:
+        futures = []
+        for root, dirs, files in os.walk(INPUT_PATH):
+            for file in files:
+                if file == "pages.csv":
+                    file_path = os.path.join(root, file)
+                    futures.append(executor.submit(process_pages_file, file_path))
+        
+        for future in as_completed(futures):
+            contact_pages = future.result()
+            if not contact_pages.empty:
+                all_contact_pages.append(contact_pages)
+            progress_bar.update(1)
+    
+    progress_bar.close()
+    
+    if all_contact_pages:
+        combined_df = pd.concat(all_contact_pages, ignore_index=True)
+        combined_df = combined_df.drop_duplicates().sort_values()
+        combined_df.to_csv(OUTPUT_PATH, index=False, header=["contact_page"])
+
+def main():
+    print("Extracting contact pages...")
+
+    process_contact_forms()
+    
+    print("Extraction complete.")
+    print(f"Output file created at: {OUTPUT_PATH}")
+
+if __name__ == "__main__":
+    main()
+

bin/extract-email-addresses.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python
+import os
+import csv
+import re
+import json
+from tqdm import tqdm
+import multiprocessing
+from email_validator import validate_email, EmailNotValidError
+import time
+import logging
+import traceback
+import dns.resolver
+import socket
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Define the input and output directories
+INPUT_DIR = "sorted-and-deduplicated"
+OUTPUT_FILE = "extracted-emails.csv"
+TEMP_FILE = "temp-emails.csv"
+STATE_FILE = "extraction_state.json"
+
+# Get the project root directory (parent directory of the bin directory)
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+INPUT_PATH = os.path.join(PROJECT_ROOT, INPUT_DIR)
+OUTPUT_PATH = os.path.join(PROJECT_ROOT, OUTPUT_FILE)
+TEMP_PATH = os.path.join(PROJECT_ROOT, TEMP_FILE)
+STATE_PATH = os.path.join(PROJECT_ROOT, STATE_FILE)
+
+# Compile regex pattern for basic email validation (more forgiving)
+EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
+
+# DNS servers
+DNS_SERVERS = ['10.1.0.1', '10.1.0.2', '10.1.0.3']
+
+def save_state(state):
+    with open(STATE_PATH, 'w') as f:
+        json.dump(state, f)
+
+def load_state():
+    if os.path.exists(STATE_PATH):
+        with open(STATE_PATH, 'r') as f:
+            return json.load(f)
+    return {"last_completed_stage": 0, "total_emails": 0, "valid_emails": {}}
+
+def find_csv_files():
+    for root, _, files in os.walk(INPUT_PATH):
+        for file in files:
+            if file == "email-addresses.csv":
+                yield os.path.join(root, file)
+
+def process_file(file_path):
+    emails = set()
+    try:
+        with open(file_path, 'r', newline='', encoding='utf-8') as csvfile:
+            reader = csv.DictReader(csvfile)
+            if 'email address' not in reader.fieldnames:
+                logging.warning(f"'email address' column not found in {file_path}")
+                return emails
+            for row in reader:
+                email = row.get('email address')
+                if email:
+                    emails.add(email.strip().lower())
+    except Exception as e:
+        logging.error(f"Error processing file: {file_path}, Error: {e}")
+    return emails
+
+def stage1_compile_emails(state):
+    if state["last_completed_stage"] >= 1:
+        logging.info("Skipping Stage 1: Already completed")
+        return
+
+    logging.info("Stage 1: Compiling all email addresses...")
+    csv_files = list(find_csv_files())
+    all_emails = set()
+    
+    with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as pool:
+        with tqdm(total=len(csv_files), desc="Processing files", unit="file") as pbar:
+            for emails in pool.imap_unordered(process_file, csv_files):
+                all_emails.update(emails)
+                pbar.update(1)
+    
+    logging.info(f"Writing {len(all_emails)} email addresses to temporary file...")
+    with open(TEMP_PATH, 'w', newline='', encoding='utf-8') as f:
+        for email in all_emails:
+            f.write(f"{email}\n")
+    
+    state["total_emails"] = len(all_emails)
+    state["last_completed_stage"] = 1
+    save_state(state)
+
+def stage2_sort_and_deduplicate(state):
+    if state["last_completed_stage"] >= 2:
+        logging.info("Skipping Stage 2: Already completed")
+        return
+
+    logging.info("Stage 2: Sorting and deduplicating email addresses...")
+    with open(TEMP_PATH, 'r', encoding='utf-8') as f:
+        emails = f.readlines()
+    
+    unique_emails = sorted(set(email.strip() for email in emails))
+    
+    with open(OUTPUT_PATH, 'w', newline='', encoding='utf-8') as f:
+        f.write("email\n")
+        for email in unique_emails:
+            f.write(f"{email}\n")
+    
+    logging.info(f"Sorted and deduplicated {len(unique_emails)} email addresses.")
+    state["total_emails"] = len(unique_emails)
+    state["last_completed_stage"] = 2
+    save_state(state)
+
+def stage3_basic_validation(state):
+    if state["last_completed_stage"] >= 3:
+        logging.info("Skipping Stage 3: Already completed")
+        return
+
+    logging.info("Stage 3: Performing basic regex validation...")
+    valid_emails = []
+    
+    with open(OUTPUT_PATH, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # Skip header
+        total_emails = sum(1 for _ in reader)
+    
+    with open(OUTPUT_PATH, 'r', encoding='utf-8') as f:
+        reader = csv.reader(f)
+        next(reader)  # Skip header
+        for email in tqdm(reader, total=total_emails, desc="Validating", unit="email"):
+            if EMAIL_REGEX.match(email[0]):
+                valid_emails.append(email[0])
+    
+    with open(OUTPUT_PATH, 'w', newline='', encoding='utf-8') as f:
+        f.write("email\n")
+        for email in valid_emails:
+            f.write(f"{email}\n")
+    
+    logging.info(f"Kept {len(valid_emails)} emails after basic validation.")
+    state["valid_emails"]["stage3"] = len(valid_emails)
+    state["last_completed_stage"] = 3
+    save_state(state)
+
+def validate_email_address(email):
+    try:
+        validated_email = validate_email(email)
+        return validated_email.normalized
+    except EmailNotValidError:
+        return None
+    except Exception as e:
+        logging.error(f"Unexpected error validating {email}: {str(e)}")
+        return None
+
+def process_batch_stage4(batch):
+    valid_emails = []
+    for email in batch:
+        validated_email = validate_email_address(email)
+        if validated_email:
+            valid_emails.append(validated_email)
+    return valid_emails
+
+def stage4_advanced_validation(state):
+    if state["last_completed_stage"] >= 4:
+        logging.info("Skipping Stage 4: Already completed")
+        return
+
+    logging.info("Stage 4: Performing advanced email validation...")
+    
+    try:
+        with open(OUTPUT_PATH, 'r', encoding='utf-8') as f:
+            reader = csv.reader(f)
+            next(reader)  # Skip header
+            emails = [row[0] for row in reader]
+        
+        logging.info(f"Loaded {len(emails)} emails for validation")
+        
+        batch_size = 1000
+        num_processes = max(1, multiprocessing.cpu_count() - 1)  # Leave one CPU free
+        
+        valid_emails = []
+        with multiprocessing.Pool(processes=num_processes) as pool:
+            batches = [emails[i:i+batch_size] for i in range(0, len(emails), batch_size)]
+            with tqdm(total=len(emails), desc="Validating", unit="email") as pbar:
+                for result in pool.imap_unordered(process_batch_stage4, batches):
+                    valid_emails.extend(result)
+                    pbar.update(len(result))
+        
+        logging.info(f"Writing {len(valid_emails)} validated emails to file...")
+        with open(OUTPUT_PATH, 'w', newline='', encoding='utf-8') as f:
+            f.write("email\n")
+            for email in valid_emails:
+                f.write(f"{email}\n")
+        
+        logging.info(f"Kept {len(valid_emails)} emails after advanced validation.")
+        state["valid_emails"]["stage4"] = len(valid_emails)
+        state["last_completed_stage"] = 4
+        save_state(state)
+    
+    except Exception as e:
+        logging.error(f"Unexpected error in stage 4: {str(e)}")
+        logging.error(traceback.format_exc())
+
+def check_mx_records(domain):
+    resolver = dns.resolver.Resolver()
+    for _ in range(2):  # Try each server twice
+        for server in DNS_SERVERS:
+            resolver.nameservers = [server]
+            try:
+                mx_records = resolver.resolve(domain, 'MX')
+                if mx_records:
+                    return True
+            except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.exception.Timeout):
+                continue
+            except Exception:
+                continue
+    return False
+
+def validate_email_address_with_mx(email):
+    try:
+        validated_email = validate_email(email)
+        domain = email.split('@')[-1]
+        if check_mx_records(domain):
+            return validated_email.normalized
+        return None
+    except EmailNotValidError:
+        return None
+    except Exception as e:
+        logging.error(f"Unexpected error validating {email} with MX: {str(e)}")
+        return None
+
+def process_batch_stage5(batch):
+    valid_emails = []
+    invalid_domains = set()
+    processed_count = 0
+    for email in batch:
+        processed_count += 1
+        result = validate_email_address_with_mx(email)
+        if result:
+            valid_emails.append(result)
+        else:
+            invalid_domains.add(email.split('@')[-1])
+    
+    # Remove emails with invalid domains
+    valid_emails = [email for email in valid_emails if email.split('@')[-1] not in invalid_domains]
+    
+    return valid_emails, processed_count
+
+def stage5_advanced_validation_with_mx(state):
+    if state["last_completed_stage"] >= 5:
+        logging.info("Skipping Stage 5: Already completed")
+        return
+
+    logging.info("Stage 5: Performing advanced email validation with MX checks...")
+    
+    try:
+        with open(OUTPUT_PATH, 'r', encoding='utf-8') as f:
+            reader = csv.reader(f)
+            next(reader)  # Skip header
+            emails = [row[0] for row in reader]
+        
+        logging.info(f"Loaded {len(emails)} emails for MX validation")
+        
+        batch_size = 100  # Reduced batch size for more frequent updates
+        num_processes = multiprocessing.cpu_count()  # Use all available CPU cores
+        
+        valid_emails = []
+        with multiprocessing.Pool(processes=num_processes) as pool:
+            batches = [emails[i:i+batch_size] for i in range(0, len(emails), batch_size)]
+            with tqdm(total=len(emails), desc="Validating MX", unit="email") as pbar:
+                for result, processed_count in pool.imap_unordered(process_batch_stage5, batches):
+                    valid_emails.extend(result)
+                    pbar.update(processed_count)
+                    
+                    # Periodically log progress
+                    if len(valid_emails) % 1000 == 0:
+                        logging.info(f"Processed {pbar.n} emails, {len(valid_emails)} valid so far")
+        
+        logging.info(f"Writing {len(valid_emails)} validated emails to file...")
+        with open(OUTPUT_PATH, 'w', newline='', encoding='utf-8') as f:
+            f.write("email\n")
+            for email in valid_emails:
+                f.write(f"{email}\n")
+        
+        logging.info(f"Kept {len(valid_emails)} emails after advanced validation with MX checks.")
+        state["valid_emails"]["stage5"] = len(valid_emails)
+        state["last_completed_stage"] = 5
+        save_state(state)
+    
+    except Exception as e:
+        logging.error(f"Unexpected error in stage 5: {str(e)}")
+        logging.error(traceback.format_exc())
+
+def main():
+    state = load_state()
+    
+    if state["last_completed_stage"] > 0:
+        logging.info("Resuming from previous execution")
+        logging.info(f"Last completed stage: {state['last_completed_stage']}")
+        logging.info(f"Total emails processed: {state['total_emails']}")
+        for stage, count in state['valid_emails'].items():
+            logging.info(f"Valid emails after {stage}: {count}")
+    
+    try:
+        stage1_compile_emails(state)
+        stage2_sort_and_deduplicate(state)
+        stage3_basic_validation(state)
+        stage4_advanced_validation(state)
+        stage5_advanced_validation_with_mx(state)
+        
+        logging.info("Email extraction and validation complete.")
+        logging.info(f"Final output file: {OUTPUT_PATH}")
+        
+        # Clean up temporary files
+        if os.path.exists(TEMP_PATH):
+            os.remove(TEMP_PATH)
+        if os.path.exists(STATE_PATH):
+            os.remove(STATE_PATH)
+    
+    except Exception as e:
+        logging.error(f"An unexpected error occurred: {str(e)}")
+        logging.error(traceback.format_exc())
+
+if __name__ == "__main__":
+    main()

bin/extract-external-domains.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+import os
+import pandas as pd
+from urllib.parse import urlparse, urlunparse, unquote
+from tqdm import tqdm
+import tldextract
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+# Define the input and output directories
+INPUT_DIR = "sorted-and-deduplicated"
+OUTPUT_FILE = "extracted-external-domains.csv"
+
+# Get the project root directory (parent directory of the bin directory)
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+INPUT_PATH = os.path.join(PROJECT_ROOT, INPUT_DIR)
+OUTPUT_PATH = os.path.join(PROJECT_ROOT, OUTPUT_FILE)
+
+def extract_root_domain(url):
+    try:
+        # Remove URL encoding
+        url = unquote(url)
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme:
+            parsed_url = parsed_url._replace(scheme="http")
+        clean_url = urlunparse(parsed_url)
+
+        # Extract domain using tldextract
+        extracted = tldextract.extract(clean_url)
+        if extracted.domain and extracted.suffix:
+            return f"{parsed_url.scheme}://{extracted.domain}.{extracted.suffix}/"
+        else:
+            return None
+    except Exception as e:
+        print(f"Error parsing URL: {url}, Error: {e}")
+        return None
+
+def clean_string(s):
+    if pd.isna(s):
+        return None
+    s = s.replace("%20", "").replace(" ", "").strip()
+    return s
+
+def process_external_links_file(file_path):
+    try:
+        df = pd.read_csv(file_path, usecols=["external link", "linking domain"])
+        df["website"] = df["external link"].apply(extract_root_domain)
+        df["website"] = df["website"].apply(clean_string)
+        df["linking domain"] = df["linking domain"].apply(clean_string)
+        df = df[["website", "linking domain"]]
+        df.columns = ["website", "domain"]
+        return df
+    except Exception as e:
+        print(f"Error processing file: {file_path}, Error: {e}")
+        return pd.DataFrame(columns=["website", "domain"])
+
+def process_external_links():
+    all_data = []
+    total_files = 0
+
+    for root, dirs, files in os.walk(INPUT_PATH):
+        for file in files:
+            if file == "external-links.csv":
+                total_files += 1
+
+    progress_bar = tqdm(total=total_files, desc="Processing external-links.csv files", unit="file")
+    
+    with ProcessPoolExecutor() as executor:
+        futures = []
+        for root, dirs, files in os.walk(INPUT_PATH):
+            for file in files:
+                if file == "external-links.csv":
+                    file_path = os.path.join(root, file)
+                    futures.append(executor.submit(process_external_links_file, file_path))
+        
+        for future in as_completed(futures):
+            df = future.result()
+            if not df.empty:
+                all_data.append(df)
+            progress_bar.update(1)
+    
+    progress_bar.close()
+    
+    return all_data
+
+def save_combined_data(all_data):
+    combined_df = pd.concat(all_data)
+    initial_count = combined_df.shape[0]
+    combined_df.drop_duplicates(subset=["website"], inplace=True)
+    final_count = combined_df.shape[0]
+    duplicates_removed = initial_count - final_count
+
+    combined_df.sort_values(by=["website"], inplace=True)
+    combined_df.to_csv(OUTPUT_PATH, index=False)
+
+    return final_count, duplicates_removed
+
+def main():
+    print("Extracting external domains...")
+
+    all_data = process_external_links()
+    total_domains, duplicates_removed = save_combined_data(all_data)
+    
+    print("Extraction complete.")
+    print(f"Output file created at: {OUTPUT_PATH}")
+    print(f"Total domains extracted: {total_domains}")
+    print(f"Total duplicates removed: {duplicates_removed}")
+
+if __name__ == "__main__":
+    main()
+

bin/normalize-data.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+import os
+import csv
+import sys
+from urllib.parse import urlparse
+import validators
+
+# Increase CSV field size limit
+csv.field_size_limit(sys.maxsize)
+
+# Determine project root dynamically
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.dirname(SCRIPT_DIR)
+
+INPUT_FILE = os.path.join(PROJECT_ROOT, "input-domains.csv")
+OUTPUT_FILE = os.path.join(PROJECT_ROOT, "01-normalized-data.csv")
+INVALID_DOMAINS_FILE = os.path.join(PROJECT_ROOT, "invalid-domains.csv")
+
+def normalize_url(url):
+    parsed = urlparse(url)
+    return parsed.netloc or parsed.path.strip('/')
+
+def is_valid_domain(domain):
+    return validators.domain(domain)
+
+def process_data():
+    print(f"Reading data from {INPUT_FILE}")
+    with open(INPUT_FILE, 'r', encoding='utf-8') as infile:
+        reader = csv.reader(infile)
+        data = list(reader)
+    
+    print("Normalizing and validating URLs...", end="", flush=True)
+    normalized_data = set()
+    invalid_domains = set()
+    for row in data:
+        if row:  # Skip empty rows
+            normalized_url = normalize_url(row[0])
+            if normalized_url:
+                if is_valid_domain(normalized_url):
+                    normalized_data.add(normalized_url)
+                else:
+                    invalid_domains.add(normalized_url)
+    print(" Done")
+    
+    print("Sorting and deduplicating data")
+    sorted_data = sorted(normalized_data)
+    
+    print(f"Writing normalized data to {OUTPUT_FILE}")
+    with open(OUTPUT_FILE, 'w', newline='', encoding='utf-8') as outfile:
+        writer = csv.writer(outfile)
+        for url in sorted_data:
+            writer.writerow([url])
+    
+    print(f"Writing invalid domains to {INVALID_DOMAINS_FILE}")
+    with open(INVALID_DOMAINS_FILE, 'w', newline='', encoding='utf-8') as outfile:
+        writer = csv.writer(outfile)
+        for url in sorted(invalid_domains):
+            writer.writerow([url])
+    
+    print(f"Processed {len(data)} input rows")
+    print(f"Wrote {len(sorted_data)} valid, unique, normalized domains to output file")
+    print(f"Found {len(invalid_domains)} invalid domains")
+
+def main():
+    print("Starting normalize-data.py")
+    process_data()
+    print("normalize-data.py completed")
+
+if __name__ == "__main__":
+    main()

bin/process-alives.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+import os
+import csv
+import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from urllib.parse import urlparse
+from halo import Halo
+import logging
+import psutil
+import time
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+PROJECT_ROOT = os.path.expanduser("~/proj/external-links-scrape")
+INPUT_FILE = os.path.join(PROJECT_ROOT, "targets.csv")
+SCRAPES_DIR = os.path.join(PROJECT_ROOT, "scrapes")
+MAX_WORKERS = 1000
+REQUIRED_FREE_MEMORY_GB = 15
+
+def create_directories(websites):
+    os.makedirs(SCRAPES_DIR, exist_ok=True)
+    for website in websites:
+        domain = urlparse(website).netloc
+        os.makedirs(os.path.join(SCRAPES_DIR, domain), exist_ok=True)
+
+def run_scrape_pages(website):
+    domain = urlparse(website).netloc
+    output_dir = os.path.join(SCRAPES_DIR, domain)
+    scrape_pages_script = os.path.join(PROJECT_ROOT, "bin", "scrape-pages.py")
+    
+    command = [scrape_pages_script, "-o", output_dir, website]
+    
+    try:
+        subprocess.run(command, check=True, capture_output=True, text=True)
+        logging.info(f"Completed scraping {website}")
+        return True
+    except subprocess.CalledProcessError as e:
+        logging.error(f"Error scraping {website}: {e}")
+        return False
+
+def check_memory():
+    memory = psutil.virtual_memory()
+    free_memory_gb = memory.available / (1024 * 1024 * 1024)  # Convert to GB
+    return free_memory_gb >= REQUIRED_FREE_MEMORY_GB
+
+def process_websites():
+    with open(INPUT_FILE, 'r') as f:
+        reader = csv.DictReader(f)
+        websites = [row['website'] for row in reader]  # Assuming 'website' column exists
+    
+    logging.info(f"Found {len(websites)} websites to process")
+    
+    create_directories(websites)
+    
+    completed = 0
+    spinner = Halo(text='Processing...', spinner='dots')
+    spinner.start()
+    
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        future_to_website = {}
+        for website in websites:
+            while not check_memory():
+                logging.info("Waiting for memory to be available...")
+                time.sleep(60)  # Wait for 1 minute before checking again
+            future = executor.submit(run_scrape_pages, website)
+            future_to_website[future] = website
+
+        for future in as_completed(future_to_website):
+            website = future_to_website[future]
+            try:
+                success = future.result()
+                if success:
+                    completed += 1
+                    spinner.text = f'Processing... Completed: {completed}/{len(websites)}'
+            except Exception as e:
+                logging.error(f"Error processing {website}: {e}")
+    
+    spinner.stop()
+    logging.info(f"Completed scraping {completed} out of {len(websites)} websites")
+
+def main():
+    logging.info("Starting process-alives.py")
+    process_websites()
+    logging.info("process-alives.py completed")
+
+if __name__ == "__main__":
+    main()

bin/process-all.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+import os
+import subprocess
+from halo import Halo
+
+PROJECT_ROOT = os.path.expanduser("~/proj/external-links-scrape")
+
+def run_script(script_name):
+    script_path = os.path.join(PROJECT_ROOT, "bin", script_name)
+    print(f"Running {script_name}...", end="", flush=True)
+    
+    spinner = Halo(text='Processing', spinner='dots')
+    spinner.start()
+    
+    try:
+        result = subprocess.run([script_path], check=True, capture_output=True, text=True)
+        spinner.succeed(f"{script_name} completed successfully")
+        print(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        spinner.fail(f"{script_name} failed")
+        print(f"Error: {e}")
+        print(e.stdout.strip())
+        print(e.stderr.strip())
+
+def main():
+    print("Starting process-all.py")
+    
+    scripts = [
+        "normalize-data.py",
+        "alive-check.py",
+        "process-alives.py"
+    ]
+    
+    for script in scripts:
+        run_script(script)
+    
+    print("process-all.py completed")
+
+if __name__ == "__main__":
+    main()

bin/scrape-pages.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python
+
+import sys
+import re
+import os
+import requests
+import phonenumbers
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse, urldefrag
+import csv
+from collections import deque
+import spacy
+import argparse
+from xml.etree import ElementTree as ET
+from datetime import datetime
+import itertools
+import gc
+import time
+import validators
+import tldextract
+
+# Load spaCy model for NER
+nlp = spacy.load("en_core_web_sm")
+
+# List of social media domains and their URL shorteners
+SOCIAL_MEDIA_DOMAINS = [
+    'twitter.com', 't.co', 'x.com', 'facebook.com', 'fb.com', 'youtube.com',
+    'youtu.be', 'instagram.com', 'pinterest.com', 'linkedin.com', 'lnkd.in', 'tiktok.com'
+]
+
+# Regular expressions for email addresses
+EMAIL_REGEX = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')
+
+META_INFO_FILE = 'meta-info.csv'
+STATUS_FILE = '.status'
+
+# Network request timeout and retry settings
+TIMEOUT = 60  # 1 minute timeout
+MAX_RETRIES = 3
+
+def is_valid_url(url):
+    return validators.url(url)
+
+def get_root_domain(url):
+    return tldextract.extract(url).registered_domain
+
+def is_social_media_url(url):
+    root_domain = get_root_domain(url)
+    return any(domain in root_domain for domain in SOCIAL_MEDIA_DOMAINS)
+
+def fetch_with_retry(url, timeout=TIMEOUT, max_retries=MAX_RETRIES):
+    for attempt in range(max_retries):
+        try:
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+            }
+            response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True)
+            response.raise_for_status()
+            return response
+        except requests.RequestException as e:
+            print(f"Attempt {attempt + 1} failed: {e}")
+            if attempt == max_retries - 1:
+                raise
+
+def fetch_sitemaps(base_url):
+    sitemaps = set()
+    
+    robots_url = urljoin(base_url, '/robots.txt')
+    print(f"Fetching {robots_url}")
+    try:
+        response = fetch_with_retry(robots_url)
+        for line in response.text.split('\n'):
+            if line.strip().lower().startswith('sitemap:'):
+                sitemap_url = line.split(':', 1)[1].strip()
+                if is_valid_url(sitemap_url):
+                    print(f"Found sitemap in robots.txt: {sitemap_url}")
+                    sitemaps.add(sitemap_url)
+    except requests.RequestException as e:
+        print(f"Error fetching robots.txt: {e}")
+
+    common_sitemap_url = urljoin(base_url, '/sitemap.xml')
+    print(f"Checking common location for sitemap: {common_sitemap_url}")
+    try:
+        response = fetch_with_retry(common_sitemap_url)
+        print(f"Found sitemap at common location: {common_sitemap_url}")
+        sitemaps.add(common_sitemap_url)
+    except requests.RequestException as e:
+        print(f"Error fetching sitemap.xml: {e}")
+
+    return sitemaps
+
+def parse_sitemap(sitemap_url):
+    print(f"Parsing sitemap: {sitemap_url}")
+    try:
+        response = fetch_with_retry(sitemap_url)
+        root = ET.fromstring(response.content)
+        for url in root.findall('.//{http://www.sitemaps.org/schemas/sitemap/0.9}loc'):
+            if url is not None and is_valid_url(url.text):
+                yield url.text
+                print(f"Found URL in sitemap: {url.text}")
+    except requests.RequestException as e:
+        print(f"Error fetching sitemap: {e}")
+    except ET.ParseError as e:
+        print(f"Error parsing sitemap: {e}")
+
+def get_all_links_from_sitemaps(base_url):
+    sitemap_urls = fetch_sitemaps(base_url)
+    all_urls = set()
+    
+    for sitemap_url in sitemap_urls:
+        all_urls.update(parse_sitemap(sitemap_url))
+    
+    print(f"Total URLs found in sitemaps: {len(all_urls)}")
+    return all_urls
+
+def get_seo_meta(soup, base_url):
+    title = soup.title.string if soup.title else ''
+    meta_description = ''
+    meta_robots = ''
+    meta_keywords = ''
+    canonical = ''
+    google_analytics = 'no'
+    google_ads_tag = 'no'
+    google_tag_manager = 'no'
+    bing_analytics = 'no'
+    schema_found = 'no'
+    first_h1 = ''
+    outbound_external = set()
+    outbound_internal = set()
+    word_count = 0
+    phone_numbers = []
+    email_addresses = []
+
+    for meta in soup.find_all('meta'):
+        if meta.get('name') == 'description':
+            meta_description = meta.get('content', '')
+        if meta.get('name') == 'robots':
+            meta_robots = meta.get('content', '')
+        if meta.get('name') == 'keywords':
+            meta_keywords = meta.get('content', '')
+
+    canonical_link = soup.find('link', rel='canonical')
+    if canonical_link:
+        canonical = canonical_link.get('href', '')
+
+    for script in soup.find_all('script'):
+        if script.string:
+            script_content = script.string.lower()
+            if 'gtag(' in script_content and 'ua-' in script_content:
+                google_analytics = 'yes'
+            if 'gtag(' in script_content and 'aw-' in script_content:
+                google_ads_tag = 'yes'
+            if 'googletagmanager.com' in script_content:
+                google_tag_manager = 'yes'
+            if 'bat.bing.com' in script_content or 'bing.com/tag.js' in script_content:
+                bing_analytics = 'yes'
+            if 'schema.org' in script_content:
+                schema_found = 'yes'
+
+    h1_tag = soup.find('h1')
+    if h1_tag:
+        first_h1 = h1_tag.get_text(strip=True)
+
+    body_content = soup.body.get_text(separator=' ') if soup.body else ''
+    word_count = len(body_content.split())
+
+    for match in phonenumbers.PhoneNumberMatcher(body_content, "US"):
+        phone_number = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
+        anchor_text = ''
+        paragraph_text = ''
+        span_text = ''
+
+        anchor = soup.find('a', string=re.compile(re.escape(match.raw_string)))
+        if anchor:
+            anchor_text = anchor.get_text()
+
+        paragraph = soup.find('p', string=re.compile(re.escape(match.raw_string)))
+        if paragraph:
+            paragraph_text = paragraph.get_text()
+
+        span = soup.find('span', string=re.compile(re.escape(match.raw_string)))
+        if span:
+            span_text = span.get_text()
+
+        phone_numbers.append((phone_number, anchor_text, paragraph_text, span_text))
+
+    for email in EMAIL_REGEX.findall(body_content):
+        email_domain = email.split('@')[-1]
+        email_addresses.append((email, email_domain))
+
+    for link in soup.body.find_all('a', href=True) if soup.body else []:
+        full_link = urldefrag(urljoin(base_url, link['href']))[0]
+        if is_valid_url(full_link):
+            if get_root_domain(full_link) != get_root_domain(base_url):
+                outbound_external.add(full_link)
+            else:
+                outbound_internal.add(full_link)
+
+    return (title, meta_description, meta_robots, meta_keywords, google_analytics, google_ads_tag, 
+            google_tag_manager, bing_analytics, schema_found, canonical, first_h1, word_count, len(outbound_external), 
+            len(outbound_internal), phone_numbers, email_addresses)
+
+def get_all_links(url, base_url):
+    print(f"Fetching {url}")
+    try:
+        response = fetch_with_retry(url)
+        final_url = response.url
+        status_code = response.status_code
+        print(f"Status Code: {status_code} for {url}")
+
+        if status_code == 200:
+            if url.endswith('.xml'):
+                return parse_sitemap(url)
+            else:
+                soup = BeautifulSoup(response.content, "html.parser")
+                links = [a.get('href') for a in soup.find_all('a', href=True)]
+                print(f"Found {len(links)} links on {url}")
+
+                (title, meta_description, meta_robots, meta_keywords, google_analytics, google_ads_tag, 
+                 google_tag_manager, bing_analytics, schema_found, canonical, first_h1, word_count, 
+                 outbound_external, outbound_internal, phone_numbers, 
+                 email_addresses) = get_seo_meta(soup, base_url)
+                
+                internal_links = set()
+                external_links = []
+                social_media_links = []
+                for link in links:
+                    full_link = urldefrag(urljoin(base_url, link))[0]
+                    if is_valid_url(full_link):
+                        if get_root_domain(full_link) == get_root_domain(base_url):
+                            internal_links.add(full_link)
+                        else:
+                            if is_social_media_url(full_link):
+                                social_media_links.append((full_link.lower(), get_root_domain(full_link), url))
+                            else:
+                                external_links.append((full_link.lower(), get_root_domain(full_link), url))
+                        
+                print(f"Filtered to {len(internal_links)} internal links on the same domain")
+                print(f"Found {len(external_links)} external links")
+                print(f"Found {len(social_media_links)} social media links")
+                
+                return (internal_links, external_links, social_media_links, url, status_code, final_url, title, 
+                        meta_description, meta_robots, meta_keywords, google_analytics, google_ads_tag, 
+                        google_tag_manager, bing_analytics, schema_found, canonical, first_h1, word_count, 
+                        outbound_external, outbound_internal, phone_numbers, email_addresses)
+        else:
+            return set(), [], [], url, status_code, final_url, '', '', '', '', 'no', 'no', 'no', 'no', 'no', '', '', 0, 0, 0, [], []
+
+    except requests.RequestException as e:
+        print(f"Error fetching {url}: {e}")
+        return set(), [], [], url, "Error", url, '', '', '', '', 'no', 'no', 'no', 'no', 'no', '', '', 0, 0, 0, [], []
+
+def initialize_meta_info(output_dir):
+    meta_info_path = os.path.join(output_dir, META_INFO_FILE)
+    if os.path.exists(meta_info_path):
+        with open(meta_info_path, mode='r', newline='') as file:
+            reader = csv.DictReader(file)
+            return {row['url']: row for row in reader}
+    else:
+        with open(meta_info_path, mode='w', newline='') as file:
+            writer = csv.writer(file)
+            writer.writerow(['url', 'status', 'found_date', 'last_visited_date', 'retry_count'])
+    return {}
+
+def update_meta_info(meta_info, output_dir):
+    meta_info_path = os.path.join(output_dir, META_INFO_FILE)
+    with open(meta_info_path, mode='w', newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow(['url', 'status', 'found_date', 'last_visited_date', 'retry_count'])
+        for url, info in meta_info.items():
+            writer.writerow([url, info['status'], info['found_date'], info['last_visited_date'], info['retry_count']])
+
+def add_url_to_meta_info(meta_info, url, output_dir):
+    if url not in meta_info:
+        meta_info[url] = {
+            'url': url,
+            'status': 'pending',
+            'found_date': datetime.now().isoformat(),
+            'last_visited_date': '',
+            'retry_count': 0
+        }
+        meta_info_path = os.path.join(output_dir, META_INFO_FILE)
+        with open(meta_info_path, mode='a', newline='') as file:
+            writer = csv.writer(file)
+            writer.writerow([url, 'pending', datetime.now().isoformat(), '', 0])
+
+def mark_url_visited(meta_info, url, output_dir):
+    if url in meta_info:
+        meta_info[url]['status'] = 'visited'
+        meta_info[url]['last_visited_date'] = datetime.now().isoformat()
+        update_meta_info(meta_info, output_dir)
+
+def mark_url_error(meta_info, url, output_dir):
+    if url in meta_info:
+        meta_info[url]['status'] = 'error'
+        meta_info[url]['retry_count'] += 1
+        meta_info[url]['last_visited_date'] = datetime.now().isoformat()
+        update_meta_info(meta_info, output_dir)
+
+def check_status_file(output_dir):
+    status_file_path = os.path.join(output_dir, STATUS_FILE)
+    if os.path.exists(status_file_path):
+        with open(status_file_path, 'r') as file:
+            content = file.read().strip()
+            if content == "completed":
+                return True
+    return False
+
+def mark_as_completed(output_dir):
+    status_file_path = os.path.join(output_dir, STATUS_FILE)
+    with open(status_file_path, 'w') as file:
+        file.write("completed")
+
+def print_meta_info_summary(meta_info):
+    pending_count = sum(1 for info in meta_info.values() if info['status'] == 'pending')
+    visited_count = sum(1 for info in meta_info.values() if info['status'] == 'visited')
+    error_count = sum(1 for info in meta_info.values() if info['status'] == 'error')
+
+    print(f"Found {pending_count} pending pages.")
+    print(f"Found {visited_count} visited pages.")
+    print(f"Found {error_count} pages with errors.")
+    print("Resuming from previous state...")
+
+def scrape_website(base_url, output_dir):
+    if check_status_file(output_dir):
+        print("Scrape already completed for this domain.")
+        return
+
+    visited = set()
+    to_visit = deque()
+
+    meta_info = initialize_meta_info(output_dir)
+    print_meta_info_summary(meta_info)
+    
+    if not meta_info:
+        initial_urls = get_all_links_from_sitemaps(base_url)
+        for url in initial_urls:
+            add_url_to_meta_info(meta_info, url, output_dir)
+
+    for url, info in meta_info.items():
+        if info['status'] == 'pending':
+            to_visit.append(url)
+        elif info['status'] == 'visited':
+            visited.add(url)
+
+    internal_csv_file = os.path.join(output_dir, 'pages.csv')
+    external_csv_file = os.path.join(output_dir, 'external-links.csv')
+    social_media_csv_file = os.path.join(output_dir, 'social-media-links.csv')
+    phone_csv_file = os.path.join(output_dir, 'phone-numbers.csv')
+    email_csv_file = os.path.join(output_dir, 'email-addresses.csv')
+
+    # Initialize CSV files with headers
+    csv_files = [
+        (internal_csv_file, ['found url', 'status', 'final url', 'canonical', 'title', 'first h1',
+                             'meta description', 'meta robots', 'meta keywords', 'google analytics', 
+                             'google ads tag', 'google tag manager', 'bing analytics', 'schema found',
+                             'word count', 'outbound external', 'outbound internal']),
+        (external_csv_file, ['external link', 'linking domain', 'linking page']),
+        (social_media_csv_file, ['external link', 'linking domain', 'linking page']),
+        (phone_csv_file, ['phone number', 'anchor', 'paragraph', 'span', 'found on page']),
+        (email_csv_file, ['email address', 'email domain', 'found on page'])
+    ]
+
+    for file_path, header in csv_files:
+        if not os.path.exists(file_path):
+            with open(file_path, 'w', newline='') as file:
+                writer = csv.writer(file)
+                writer.writerow(header)
+
+    processed_count = 0
+    while to_visit:
+        current_url = to_visit.popleft()
+        if current_url not in visited:
+            visited.add(current_url)
+            print(f"Visiting: {current_url}")
+
+            if current_url.endswith('.xml'):
+                internal_links = parse_sitemap(current_url)
+                for link in internal_links:
+                    if link not in meta_info:
+                        add_url_to_meta_info(meta_info, link, output_dir)
+                        to_visit.append(link)
+            else:
+                (internal_links, external_links, social_media_links, found_url, status_code, final_url, title, 
+                    meta_description, meta_robots, meta_keywords, google_analytics, google_ads_tag, 
+                    google_tag_manager, bing_analytics, schema_found, canonical, first_h1, word_count, 
+                    outbound_external, outbound_internal, phone_numbers, email_addresses) = get_all_links(current_url, base_url)
+                
+                mark_url_visited(meta_info, current_url, output_dir)
+
+                new_internal_links = internal_links - visited
+                if new_internal_links:
+                    print(f"Adding {len(new_internal_links)} new internal links to visit")
+                    to_visit.extend(new_internal_links)
+                else:
+                    print(f"No new internal links found on {current_url}")
+
+                # Stream write to CSV files
+                with open(internal_csv_file, 'a', newline='') as file:
+                    writer = csv.writer(file)
+                    writer.writerow([found_url, status_code, final_url, canonical, title, first_h1, meta_description, 
+                                     meta_robots, meta_keywords, google_analytics, google_ads_tag, google_tag_manager, 
+                                     bing_analytics, schema_found, word_count, outbound_external, outbound_internal])
+
+                with open(external_csv_file, 'a', newline='') as file:
+                    writer = csv.writer(file)
+                    for ext_link, domain, page in external_links:
+                        writer.writerow([ext_link, domain, page])
+
+                with open(social_media_csv_file, 'a', newline='') as file:
+                    writer = csv.writer(file)
+                    for soc_link, domain, page in social_media_links:
+                        writer.writerow([soc_link, domain, page])
+
+                with open(phone_csv_file, 'a', newline='') as file:
+                    writer = csv.writer(file)
+                    for phone, anchor, paragraph, span in phone_numbers:
+                        writer.writerow([phone, anchor, paragraph, span, found_url])
+
+                with open(email_csv_file, 'a', newline='') as file:
+                    writer = csv.writer(file)
+                    for email, domain in email_addresses:
+                        writer.writerow([email, domain, found_url])
+
+            processed_count += 1
+            if processed_count % 100 == 0:
+                print(f"Processed {processed_count} pages. Clearing memory...")
+                gc.collect()  # Manual garbage collection
+
+    mark_as_completed(output_dir)
+    update_meta_info(meta_info, output_dir)
+    print("No more links to visit. Finished scraping.")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Scrape website pages and extract information.')
+    parser.add_argument('base_url', help='The base URL of the website to scrape')
+    parser.add_argument('-o', '--output', help='The directory to save the CSV files', default='.')
+    
+    args = parser.parse_args()
+    
+    if not args.base_url or not is_valid_url(args.base_url):
+        print("Please provide a valid URL.")
+        parser.print_help()
+        sys.exit(1)
+
+    base_url = args.base_url
+    output_dir = args.output
+    
+    print(f"Starting to scrape {base_url}")
+    scrape_website(base_url, output_dir)
+    print("Finished scraping.")

bin/split-alives.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+import os
+import csv
+import math
+import sys
+
+PROJECT_ROOT = os.path.expanduser("~/proj/external-links-scrape")
+INPUT_FILE = os.path.join(PROJECT_ROOT, "02-alive-check.csv")
+OUTPUT_FILES = [os.path.join(PROJECT_ROOT, f"target{i}.csv") for i in range(1, 4)]
+
+def split_alives():
+    print(f"Reading input file: {INPUT_FILE}")
+
+    if not os.path.exists(INPUT_FILE):
+        print(f"Error: Input file {INPUT_FILE} does not exist.")
+        sys.exit(1)
+
+    # Read alive websites
+    alive_websites = []
+    try:
+        with open(INPUT_FILE, 'r') as f:
+            reader = csv.DictReader(f)
+            headers = reader.fieldnames
+            for row in reader:
+                if 'alive' not in row:
+                    print("Error: 'alive' column not found in the input file.")
+                    sys.exit(1)
+                if row['alive'].lower() == 'yes':
+                    alive_websites.append(row)
+    except Exception as e:
+        print(f"Error reading input file: {e}")
+        sys.exit(1)
+
+    total_alive = len(alive_websites)
+    print(f"Total alive websites: {total_alive}")
+
+    if total_alive == 0:
+        print("No alive websites found. Exiting.")
+        sys.exit(0)
+
+    # Calculate split sizes
+    base_size = math.ceil(total_alive / 3)
+    split_sizes = [base_size] * 2 + [total_alive - base_size * 2]
+
+    # Write split files
+    start_index = 0
+    for i, output_file in enumerate(OUTPUT_FILES):
+        end_index = start_index + split_sizes[i]
+        try:
+            with open(output_file, 'w', newline='') as f:
+                writer = csv.DictWriter(f, fieldnames=headers)
+                writer.writeheader()
+                for row in alive_websites[start_index:end_index]:
+                    writer.writerow(row)
+            print(f"Written {split_sizes[i]} rows to {output_file}")
+        except Exception as e:
+            print(f"Error writing to {output_file}: {e}")
+        start_index = end_index
+
+def main():
+    print("Starting split-alives.py")
+    split_alives()
+    print("split-alives.py completed")
+
+if __name__ == "__main__":
+    main()
+