Initial commit.

.gitignore

@@ -0,0 +1 @@
+*.csv

bin/extract-targets.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+import csv
+import sys
+import os
+from datetime import datetime
+
+def get_output_filename(input_filename, chunk_number):
+    base, ext = os.path.splitext(input_filename)
+    return f"{base}-chunk-{chunk_number}{ext}"
+
+def get_history_filename(input_filename):
+    base, ext = os.path.splitext(input_filename)
+    return f"{base}-history{ext}"
+
+def read_csv(filename):
+    with open(filename, 'r', newline='') as csvfile:
+        reader = csv.reader(csvfile)
+        return list(reader)
+
+def write_csv(filename, data):
+    with open(filename, 'w', newline='') as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerows(data)
+
+def update_history(history_filename, new_data, chunk_number):
+    if os.path.exists(history_filename):
+        history_data = read_csv(history_filename)
+        headers = history_data[0]
+        if 'extraction' not in headers:
+            headers.append('extraction')
+            for row in history_data[1:]:
+                row.append('chunk-1')
+    else:
+        headers = new_data[0] + ['extraction']
+        history_data = [headers]
+
+    for row in new_data[1:]:
+        history_data.append(row + [f'chunk-{chunk_number}'])
+
+    write_csv(history_filename, history_data)
+
+def get_extracted_rows(history_filename):
+    if not os.path.exists(history_filename):
+        return set()
+
+    history_data = read_csv(history_filename)
+    extracted_rows = set()
+    for row in history_data[1:]:  # Skip header
+        extracted_rows.add(tuple(row[:-1]))  # Exclude the 'extraction' column
+    return extracted_rows
+
+def extract_targets(input_filename, num_rows):
+    history_filename = get_history_filename(input_filename)
+    extracted_rows = get_extracted_rows(history_filename)
+    
+    input_data = read_csv(input_filename)
+    headers = input_data[0]
+    
+    new_chunk = [headers]
+    chunk_number = len(set(row[-1] for row in read_csv(history_filename)[1:])) + 1 if os.path.exists(history_filename) else 1
+    
+    rows_added = 0
+    for row in input_data[1:]:
+        if tuple(row) not in extracted_rows and rows_added < num_rows:
+            new_chunk.append(row)
+            rows_added += 1
+        
+        if rows_added == num_rows:
+            break
+    
+    if rows_added == 0:
+        print("No new rows to extract.")
+        return
+    
+    output_filename = get_output_filename(input_filename, chunk_number)
+    write_csv(output_filename, new_chunk)
+    update_history(history_filename, new_chunk, chunk_number)
+    
+    print(f"Extracted {rows_added} rows to {output_filename}")
+    print(f"Updated history file: {history_filename}")
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python extract-targets.py <input_filename> <num_rows>")
+        sys.exit(1)
+
+    input_filename = sys.argv[1]
+    num_rows = int(sys.argv[2])
+
+    if not os.path.exists(input_filename):
+        print(f"Error: Input file '{input_filename}' not found.")
+        sys.exit(1)
+
+    extract_targets(input_filename, num_rows)

bin/filter-bad-tlds.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Check if the correct number of arguments is provided
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <input_file> <output_file>"
+  exit 1
+fi
+
+input_file="$1"
+output_file="$2"
+
+# Use awk to filter the lines and save to the output file
+awk '
+BEGIN { FS = OFS = "\"" }
+{
+    url = $2
+    if (url !~ /^https?:\/\/[^:\/]*\.(gov|uk)([\/:]|$)/ && url !~ /\.govt\./) {
+        print $0
+    }
+}' "$input_file" > "$output_file"
+
+echo "Filtered lines have been saved to $output_file"

bin/remove-bad-doms.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+import sys
+import re
+import os
+import csv
+
+def load_tld_list(project_root):
+    tld_file = os.path.join(project_root, 'tld-list.csv')
+    tlds = set()
+    with open(tld_file, 'r') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            tlds.add(row['tld'].lower())
+    return tlds
+
+def filter_domains(input_file):
+    # Get the project root directory
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    project_root = os.path.dirname(script_dir)
+
+    # Load the TLD list
+    tld_list = load_tld_list(project_root)
+
+    # Generate output filenames in the project root
+    base_name = os.path.splitext(os.path.basename(input_file))[0]
+    output_file = os.path.join(project_root, f"{base_name}-no-bad-doms.csv")
+    removed_file = os.path.join(project_root, f"{base_name}-removed-urls.csv")
+
+    # Compile the regex patterns for efficiency
+    domain_pattern = re.compile(r'^https?://[^:/]*\.(gov|uk)([/:]|$)')
+    edu_govt_pattern = re.compile(r'\.(edu|govt)\.')
+    tld_pattern = re.compile(r'\.([a-z]{2,})([/:]|$)')
+
+    with open(input_file, 'r') as infile, \
+         open(output_file, 'w', newline='') as outfile, \
+         open(removed_file, 'w', newline='') as removedfile:
+        
+        # Read and write the header
+        header = next(infile)
+        outfile.write(header)
+        removedfile.write(header)
+
+        for line in infile:
+            # Strip whitespace but keep quotes
+            url = line.strip()
+            
+            # Extract URL without quotes for pattern matching
+            url_no_quotes = url.strip('"')
+            
+            # Check if the URL matches any of the patterns to be filtered
+            if (not domain_pattern.search(url_no_quotes) and 
+                not edu_govt_pattern.search(url_no_quotes) and
+                not any(url_no_quotes.endswith(tld) for tld in tld_list)):
+                # If it doesn't match, write to the main output file
+                outfile.write(line)
+            else:
+                # If it matches, write to the removed URLs file
+                removedfile.write(line)
+
+    print(f"Filtered lines have been saved to {output_file}")
+    print(f"Removed URLs have been saved to {removed_file}")
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python filter_domains.py <input_file>")
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    filter_domains(input_file)

bin/wrap-quotes.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+import sys
+import os
+
+def wrap_quotes(input_filename):
+    # Generate the output filename
+    base, ext = os.path.splitext(input_filename)
+    output_filename = f"{base}-quoted{ext}"
+
+    try:
+        with open(input_filename, 'r') as infile, open(output_filename, 'w') as outfile:
+            for line in infile:
+                # Strip whitespace and wrap the line in double quotes
+                quoted_line = f'"{line.strip()}"\n'
+                outfile.write(quoted_line)
+        
+        print(f"Successfully created: {output_filename}")
+    except IOError as e:
+        print(f"Error processing file: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python wrap-quotes.py <input_filename>")
+        sys.exit(1)
+
+    input_filename = sys.argv[1]
+
+    if not os.path.exists(input_filename):
+        print(f"Error: Input file '{input_filename}' not found.")
+        sys.exit(1)
+
+    wrap_quotes(input_filename)