87 lines
2.7 KiB
Python
Executable file
87 lines
2.7 KiB
Python
Executable file
#!/usr/bin/env python
|
|
|
|
import os
|
|
import pandas as pd
|
|
from tqdm import tqdm
|
|
import re
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
|
|
# Define the input and output directories
|
|
INPUT_DIR = "sorted-and-deduplicated"
|
|
OUTPUT_FILE = "extracted-contact-forms.csv"
|
|
|
|
# Get the project root directory (parent directory of the bin directory)
|
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
|
INPUT_PATH = os.path.join(PROJECT_ROOT, INPUT_DIR)
|
|
OUTPUT_PATH = os.path.join(PROJECT_ROOT, OUTPUT_FILE)
|
|
|
|
# Common patterns for contact pages
|
|
CONTACT_PATTERNS = [
|
|
re.compile(r'contact$', re.IGNORECASE),
|
|
re.compile(r'contact-us$', re.IGNORECASE),
|
|
re.compile(r'contactus$', re.IGNORECASE),
|
|
re.compile(r'contact_me$', re.IGNORECASE),
|
|
re.compile(r'kontakt$', re.IGNORECASE),
|
|
re.compile(r'impressum$', re.IGNORECASE)
|
|
]
|
|
|
|
def is_contact_page(url):
|
|
for pattern in CONTACT_PATTERNS:
|
|
if pattern.search(url):
|
|
return True
|
|
return False
|
|
|
|
def process_pages_file(file_path):
|
|
try:
|
|
df = pd.read_csv(file_path, usecols=["final url"])
|
|
df["is_contact_page"] = df["final url"].apply(is_contact_page)
|
|
contact_pages = df[df["is_contact_page"]]["final url"]
|
|
return contact_pages
|
|
except Exception as e:
|
|
print(f"Error processing file: {file_path}, Error: {e}")
|
|
return pd.Series()
|
|
|
|
def process_contact_forms():
|
|
all_contact_pages = []
|
|
total_files = 0
|
|
|
|
for root, dirs, files in os.walk(INPUT_PATH):
|
|
for file in files:
|
|
if file == "pages.csv":
|
|
total_files += 1
|
|
|
|
progress_bar = tqdm(total=total_files, desc="Processing pages.csv files", unit="file")
|
|
|
|
with ProcessPoolExecutor() as executor:
|
|
futures = []
|
|
for root, dirs, files in os.walk(INPUT_PATH):
|
|
for file in files:
|
|
if file == "pages.csv":
|
|
file_path = os.path.join(root, file)
|
|
futures.append(executor.submit(process_pages_file, file_path))
|
|
|
|
for future in as_completed(futures):
|
|
contact_pages = future.result()
|
|
if not contact_pages.empty:
|
|
all_contact_pages.append(contact_pages)
|
|
progress_bar.update(1)
|
|
|
|
progress_bar.close()
|
|
|
|
if all_contact_pages:
|
|
combined_df = pd.concat(all_contact_pages, ignore_index=True)
|
|
combined_df = combined_df.drop_duplicates().sort_values()
|
|
combined_df.to_csv(OUTPUT_PATH, index=False, header=["contact_page"])
|
|
|
|
def main():
|
|
print("Extracting contact pages...")
|
|
|
|
process_contact_forms()
|
|
|
|
print("Extraction complete.")
|
|
print(f"Output file created at: {OUTPUT_PATH}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|