87 lines
3 KiB
Bash
Executable file
87 lines
3 KiB
Bash
Executable file
#!/usr/bin/env python
|
|
|
|
import sys
|
|
import re
|
|
import os
|
|
import csv
|
|
from urllib.parse import urlparse
|
|
|
|
def load_tld_list(project_root, filename):
|
|
tld_file = os.path.join(project_root, 'conf.d', filename)
|
|
tlds = set()
|
|
with open(tld_file, 'r') as f:
|
|
reader = csv.DictReader(f)
|
|
for row in reader:
|
|
tlds.add(row['tld'].lower().strip('.'))
|
|
return tlds
|
|
|
|
def filter_domains(input_file):
|
|
# Get the project root directory
|
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
|
project_root = os.path.dirname(script_dir)
|
|
|
|
# Load the positive and negative TLD lists
|
|
positive_tlds = load_tld_list(project_root, 'positive-tlds.csv')
|
|
negative_tlds = load_tld_list(project_root, 'negative-tlds.csv')
|
|
|
|
# Generate output filenames in the data directory
|
|
data_dir = os.path.join(project_root, 'data')
|
|
base_name = os.path.splitext(os.path.basename(input_file))[0]
|
|
output_file = os.path.join(data_dir, f"{base_name}-no-bad-doms.csv")
|
|
removed_file = os.path.join(data_dir, f"{base_name}-removed-urls.csv")
|
|
|
|
# Compile the regex patterns for efficiency
|
|
domain_pattern = re.compile(r'^https?://[^:/]*\.(gov|uk)([/:]|$)')
|
|
edu_govt_pattern = re.compile(r'\.(edu|govt)\.')
|
|
|
|
def should_keep(url):
|
|
parsed_url = urlparse(url)
|
|
domain_parts = parsed_url.netloc.lower().split('.')
|
|
return any(part in positive_tlds for part in domain_parts)
|
|
|
|
def should_remove(url):
|
|
if domain_pattern.search(url) or edu_govt_pattern.search(url):
|
|
return True
|
|
|
|
parsed_url = urlparse(url)
|
|
domain_parts = parsed_url.netloc.lower().split('.')
|
|
return any(part in negative_tlds for part in domain_parts)
|
|
|
|
# Ensure the full path to the input file is in the data directory
|
|
input_file_full_path = os.path.join(data_dir, os.path.basename(input_file))
|
|
|
|
with open(input_file_full_path, 'r') as infile, \
|
|
open(output_file, 'w', newline='') as outfile, \
|
|
open(removed_file, 'w', newline='') as removedfile:
|
|
|
|
# Read and write the header
|
|
header = next(infile)
|
|
outfile.write(header)
|
|
removedfile.write(header)
|
|
|
|
for line in infile:
|
|
# Strip whitespace but keep quotes
|
|
url = line.strip()
|
|
|
|
# Extract URL without quotes for pattern matching
|
|
url_no_quotes = url.strip('"')
|
|
|
|
# Check if the URL should be kept or removed
|
|
if should_keep(url_no_quotes):
|
|
outfile.write(line)
|
|
elif should_remove(url_no_quotes):
|
|
removedfile.write(line)
|
|
else:
|
|
outfile.write(line) # If it's neither in positive nor negative list, we keep it
|
|
|
|
print(f"Filtered lines have been saved to {output_file}")
|
|
print(f"Removed URLs have been saved to {removed_file}")
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 2:
|
|
print("Usage: python filter_domains.py <input_file>")
|
|
sys.exit(1)
|
|
|
|
input_file = sys.argv[1]
|
|
filter_domains(input_file)
|