Adjusted the project structure and added things.

.gitignore

@@ -1 +1 @@
-*.csv
+data/*

bin/extract-targets.py

@@ -1,17 +1,21 @@
 #!/usr/bin/env python
-
 import csv
 import sys
 import os
 from datetime import datetime
 
+# Get the project root directory
+script_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.dirname(script_dir)
+data_dir = os.path.join(project_root, 'data')
+
 def get_output_filename(input_filename, chunk_number):
-    base, ext = os.path.splitext(input_filename)
-    return f"{base}-chunk-{chunk_number}{ext}"
+    base, ext = os.path.splitext(os.path.basename(input_filename))
+    return os.path.join(data_dir, f"{base}-chunk-{chunk_number}{ext}")
 
 def get_history_filename(input_filename):
-    base, ext = os.path.splitext(input_filename)
-    return f"{base}-history{ext}"
+    base, ext = os.path.splitext(os.path.basename(input_filename))
+    return os.path.join(data_dir, f"{base}-history{ext}")
 
 def read_csv(filename):
     with open(filename, 'r', newline='') as csvfile:
@@ -51,10 +55,11 @@ def get_extracted_rows(history_filename):
     return extracted_rows
 
 def extract_targets(input_filename, num_rows):
+    full_input_path = os.path.join(data_dir, input_filename)
     history_filename = get_history_filename(input_filename)
     extracted_rows = get_extracted_rows(history_filename)
     
-    input_data = read_csv(input_filename)
+    input_data = read_csv(full_input_path)
     headers = input_data[0]
     
     new_chunk = [headers]
@@ -88,8 +93,9 @@ if __name__ == "__main__":
     input_filename = sys.argv[1]
     num_rows = int(sys.argv[2])
 
-    if not os.path.exists(input_filename):
-        print(f"Error: Input file '{input_filename}' not found.")
+    full_input_path = os.path.join(data_dir, input_filename)
+    if not os.path.exists(full_input_path):
+        print(f"Error: Input file '{full_input_path}' not found.")
         sys.exit(1)
 
     extract_targets(input_filename, num_rows)

bin/filter-bad-tlds.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-# Check if the correct number of arguments is provided
-if [ "$#" -ne 2 ]; then
-  echo "Usage: $0 <input_file> <output_file>"
-  exit 1
-fi
-
-input_file="$1"
-output_file="$2"
-
-# Use awk to filter the lines and save to the output file
-awk '
-BEGIN { FS = OFS = "\"" }
-{
-    url = $2
-    if (url !~ /^https?:\/\/[^:\/]*\.(gov|uk)([\/:]|$)/ && url !~ /\.govt\./) {
-        print $0
-    }
-}' "$input_file" > "$output_file"
-
-echo "Filtered lines have been saved to $output_file"

bin/remove-bad-doms.sh

@@ -4,14 +4,15 @@ import sys
 import re
 import os
 import csv
+from urllib.parse import urlparse
 
-def load_tld_list(project_root):
-    tld_file = os.path.join(project_root, 'tld-list.csv')
+def load_tld_list(project_root, filename):
+    tld_file = os.path.join(project_root, 'conf.d', filename)
     tlds = set()
     with open(tld_file, 'r') as f:
         reader = csv.DictReader(f)
         for row in reader:
-            tlds.add(row['tld'].lower())
+            tlds.add(row['tld'].lower().strip('.'))
     return tlds
 
 def filter_domains(input_file):
@@ -19,20 +20,37 @@ def filter_domains(input_file):
     script_dir = os.path.dirname(os.path.abspath(__file__))
     project_root = os.path.dirname(script_dir)
 
-    # Load the TLD list
-    tld_list = load_tld_list(project_root)
+    # Load the positive and negative TLD lists
+    positive_tlds = load_tld_list(project_root, 'positive-tlds.csv')
+    negative_tlds = load_tld_list(project_root, 'negative-tlds.csv')
 
-    # Generate output filenames in the project root
+    # Generate output filenames in the data directory
+    data_dir = os.path.join(project_root, 'data')
     base_name = os.path.splitext(os.path.basename(input_file))[0]
-    output_file = os.path.join(project_root, f"{base_name}-no-bad-doms.csv")
-    removed_file = os.path.join(project_root, f"{base_name}-removed-urls.csv")
+    output_file = os.path.join(data_dir, f"{base_name}-no-bad-doms.csv")
+    removed_file = os.path.join(data_dir, f"{base_name}-removed-urls.csv")
 
     # Compile the regex patterns for efficiency
     domain_pattern = re.compile(r'^https?://[^:/]*\.(gov|uk)([/:]|$)')
     edu_govt_pattern = re.compile(r'\.(edu|govt)\.')
-    tld_pattern = re.compile(r'\.([a-z]{2,})([/:]|$)')
 
-    with open(input_file, 'r') as infile, \
+    def should_keep(url):
+        parsed_url = urlparse(url)
+        domain_parts = parsed_url.netloc.lower().split('.')
+        return any(part in positive_tlds for part in domain_parts)
+
+    def should_remove(url):
+        if domain_pattern.search(url) or edu_govt_pattern.search(url):
+            return True
+        
+        parsed_url = urlparse(url)
+        domain_parts = parsed_url.netloc.lower().split('.')
+        return any(part in negative_tlds for part in domain_parts)
+
+    # Ensure the full path to the input file is in the data directory
+    input_file_full_path = os.path.join(data_dir, os.path.basename(input_file))
+
+    with open(input_file_full_path, 'r') as infile, \
          open(output_file, 'w', newline='') as outfile, \
          open(removed_file, 'w', newline='') as removedfile:
         
@@ -48,15 +66,13 @@ def filter_domains(input_file):
             # Extract URL without quotes for pattern matching
             url_no_quotes = url.strip('"')
             
-            # Check if the URL matches any of the patterns to be filtered
-            if (not domain_pattern.search(url_no_quotes) and 
-                not edu_govt_pattern.search(url_no_quotes) and
-                not any(url_no_quotes.endswith(tld) for tld in tld_list)):
-                # If it doesn't match, write to the main output file
+            # Check if the URL should be kept or removed
+            if should_keep(url_no_quotes):
                 outfile.write(line)
-            else:
-                # If it matches, write to the removed URLs file
+            elif should_remove(url_no_quotes):
                 removedfile.write(line)
+            else:
+                outfile.write(line)  # If it's neither in positive nor negative list, we keep it
 
     print(f"Filtered lines have been saved to {output_file}")
     print(f"Removed URLs have been saved to {removed_file}")

bin/toclip.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Check if a file name is provided
+if [ $# -eq 0 ]; then
+    echo "Error: No input file specified."
+    echo "Usage: $0 <input_file>"
+    exit 1
+fi
+
+# Get the script's directory
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Get the project root directory (one level up from the script directory)
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+
+# Set the data directory
+DATA_DIR="$PROJECT_ROOT/data"
+
+# Get the input file name from the command line argument
+INPUT_FILE="$1"
+
+# Construct the full path to the input file
+FULL_PATH="$DATA_DIR/$INPUT_FILE"
+
+# Check if the file exists
+if [ ! -f "$FULL_PATH" ]; then
+    echo "Error: File '$FULL_PATH' not found."
+    exit 1
+fi
+
+echo "Processing file: $FULL_PATH"
+
+# Use tail to skip the first line (header) and pipe to copyq
+tail -n +2 "$FULL_PATH" | copyq copy -
+
+echo "Contents of $INPUT_FILE (excluding header) have been copied to clipboard."

conf.d/negative-tlds.csv

@@ -0,0 +1,252 @@
+tld
+.ac
+.ad
+.ae
+.af
+.ag
+.al
+.am
+.an
+.ao
+.aq
+.ar
+.as
+.at
+.au
+.aw
+.ax
+.az
+.ba
+.bb
+.bd
+.be
+.bf
+.bg
+.bh
+.bi
+.bj
+.bl
+.bm
+.bn
+.bo
+.bq
+.br
+.bs
+.bt
+.bv
+.bw
+.by
+.bz
+.ca
+.cc
+.cd
+.cf
+.cg
+.ch
+.ci
+.ck
+.cl
+.cm
+.cn
+.cr
+.cu
+.cv
+.cw
+.cx
+.cy
+.cz
+.de
+.dj
+.dk
+.dm
+.do
+.dz
+.ec
+.ee
+.eg
+.eh
+.er
+.es
+.et
+.eu
+.fi
+.fj
+.fk
+.fm
+.fo
+.fr
+.ga
+.gb
+.gd
+.ge
+.gf
+.gg
+.gh
+.gi
+.gl
+.gm
+.gn
+.gp
+.gq
+.gr
+.gs
+.gt
+.gu
+.gw
+.gy
+.hk
+.hm
+.hn
+.hr
+.ht
+.hu
+.id
+.ie
+.il
+.im
+.in
+.iq
+.ir
+.is
+.it
+.je
+.jm
+.jo
+.jp
+.ke
+.kg
+.kh
+.ki
+.km
+.kn
+.kp
+.kr
+.kw
+.ky
+.kz
+.la
+.lb
+.lc
+.li
+.lk
+.lr
+.ls
+.lt
+.lu
+.lv
+.ly
+.ma
+.mc
+.md
+.me
+.mf
+.mg
+.mh
+.mk
+.ml
+.mm
+.mn
+.mo
+.mp
+.mq
+.mr
+.ms
+.mt
+.mu
+.mv
+.mw
+.mx
+.my
+.mz
+.na
+.nc
+.ne
+.nf
+.ng
+.ni
+.nl
+.no
+.np
+.nr
+.nu
+.nz
+.om
+.pa
+.pe
+.pf
+.pg
+.ph
+.pk
+.pl
+.pm
+.pn
+.pr
+.ps
+.pt
+.pw
+.py
+.qa
+.re
+.ro
+.rs
+.ru
+.rw
+.sa
+.sb
+.sc
+.sd
+.se
+.sg
+.sh
+.si
+.sj
+.sk
+.sl
+.sm
+.sn
+.so
+.sr
+.ss
+.st
+.su
+.sv
+.sx
+.sy
+.sz
+.tc
+.td
+.tf
+.tg
+.th
+.tj
+.tk
+.tl
+.tm
+.tn
+.to
+.tp
+.tr
+.tt
+.tv
+.tw
+.tz
+.ua
+.ug
+.uk
+.um
+.uy
+.uz
+.va
+.vc
+.ve
+.vg
+.vi
+.vn
+.vu
+.wf
+.ws
+.ye
+.yt
+.za
+.zm
+.zw

conf.d/positive-tlds.csv

@@ -0,0 +1,10 @@
+tld
+.com
+.net
+.org
+.io
+.biz
+.agency
+.legal
+.co
+.us