71 lines
2.2 KiB
Python
Executable file
71 lines
2.2 KiB
Python
Executable file
#!/usr/bin/env python
|
|
import os
|
|
import csv
|
|
import sys
|
|
from urllib.parse import urlparse
|
|
import validators
|
|
|
|
# Increase CSV field size limit
|
|
csv.field_size_limit(sys.maxsize)
|
|
|
|
# Determine project root dynamically
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
PROJECT_ROOT = os.path.dirname(SCRIPT_DIR)
|
|
|
|
INPUT_FILE = os.path.join(PROJECT_ROOT, "input-domains.csv")
|
|
OUTPUT_FILE = os.path.join(PROJECT_ROOT, "01-normalized-data.csv")
|
|
INVALID_DOMAINS_FILE = os.path.join(PROJECT_ROOT, "invalid-domains.csv")
|
|
|
|
def normalize_url(url):
|
|
parsed = urlparse(url)
|
|
return parsed.netloc or parsed.path.strip('/')
|
|
|
|
def is_valid_domain(domain):
|
|
return validators.domain(domain)
|
|
|
|
def process_data():
|
|
print(f"Reading data from {INPUT_FILE}")
|
|
with open(INPUT_FILE, 'r', encoding='utf-8') as infile:
|
|
reader = csv.reader(infile)
|
|
data = list(reader)
|
|
|
|
print("Normalizing and validating URLs...", end="", flush=True)
|
|
normalized_data = set()
|
|
invalid_domains = set()
|
|
for row in data:
|
|
if row: # Skip empty rows
|
|
normalized_url = normalize_url(row[0])
|
|
if normalized_url:
|
|
if is_valid_domain(normalized_url):
|
|
normalized_data.add(normalized_url)
|
|
else:
|
|
invalid_domains.add(normalized_url)
|
|
print(" Done")
|
|
|
|
print("Sorting and deduplicating data")
|
|
sorted_data = sorted(normalized_data)
|
|
|
|
print(f"Writing normalized data to {OUTPUT_FILE}")
|
|
with open(OUTPUT_FILE, 'w', newline='', encoding='utf-8') as outfile:
|
|
writer = csv.writer(outfile)
|
|
for url in sorted_data:
|
|
writer.writerow([url])
|
|
|
|
print(f"Writing invalid domains to {INVALID_DOMAINS_FILE}")
|
|
with open(INVALID_DOMAINS_FILE, 'w', newline='', encoding='utf-8') as outfile:
|
|
writer = csv.writer(outfile)
|
|
for url in sorted(invalid_domains):
|
|
writer.writerow([url])
|
|
|
|
print(f"Processed {len(data)} input rows")
|
|
print(f"Wrote {len(sorted_data)} valid, unique, normalized domains to output file")
|
|
print(f"Found {len(invalid_domains)} invalid domains")
|
|
|
|
def main():
|
|
print("Starting normalize-data.py")
|
|
process_data()
|
|
print("normalize-data.py completed")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|