84 lines
2.9 KiB
Python
Executable file
84 lines
2.9 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# Script Name: strip-neg-terms-from-unknown-cols
|
|
|
|
import os
|
|
import csv
|
|
from pyfiglet import Figlet
|
|
from halo import Halo
|
|
|
|
# Define the paths based on the project details
|
|
PROJECT_ROOT = "/home/ld/mgk-scrapes"
|
|
DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data")
|
|
STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4")
|
|
BLACKLIST_FILE = os.path.join(DATA_DIRECTORY, 'unknown-blacklist.txt')
|
|
|
|
def read_blacklist(file_path):
|
|
"""Read the blacklist terms from a file."""
|
|
if not os.path.isfile(file_path):
|
|
print("Error: The unknown-blacklist.txt file is missing.")
|
|
return []
|
|
|
|
with open(file_path, 'r') as file:
|
|
blacklist = [line.strip().lower() for line in file if line.strip()]
|
|
return blacklist
|
|
|
|
def process_csv_file(file_path, blacklist):
|
|
"""Process a CSV file to remove blacklist terms from unknown columns."""
|
|
with open(file_path, 'r') as file:
|
|
reader = csv.reader(file)
|
|
rows = list(reader)
|
|
|
|
if not rows:
|
|
return 0
|
|
|
|
headers = rows[0]
|
|
data_rows = rows[1:]
|
|
modified_rows = [headers]
|
|
terms_removed = 0
|
|
|
|
for row in data_rows:
|
|
for i, header in enumerate(headers):
|
|
if header.startswith("unknown-") and row[i].strip().lower() in blacklist:
|
|
row[i] = '' # Clear the cell if it contains a blacklist term
|
|
terms_removed += 1
|
|
modified_rows.append(row)
|
|
|
|
with open(file_path, 'w', newline='') as file:
|
|
writer = csv.writer(file)
|
|
writer.writerows(modified_rows)
|
|
|
|
return terms_removed
|
|
|
|
def strip_negative_terms():
|
|
"""Strip negative terms from all CSV files in the stage 4 directory and tally the results."""
|
|
figlet = Figlet(font='slant')
|
|
print(figlet.renderText('Strip Neg Terms'))
|
|
|
|
blacklist = read_blacklist(BLACKLIST_FILE)
|
|
if not blacklist:
|
|
return
|
|
|
|
total_terms_removed = 0
|
|
|
|
for state_dir in os.listdir(STAGE_4_DIRECTORY):
|
|
state_path = os.path.join(STAGE_4_DIRECTORY, state_dir)
|
|
if os.path.isdir(state_path):
|
|
spinner = Halo(text=f'Processing {state_dir}', spinner='dots')
|
|
spinner.start()
|
|
state_terms_removed = 0
|
|
for county_dir in os.listdir(state_path):
|
|
county_path = os.path.join(state_path, county_dir)
|
|
if os.path.isdir(county_path):
|
|
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
|
for file in csv_files:
|
|
file_path = os.path.join(county_path, file)
|
|
state_terms_removed += process_csv_file(file_path, blacklist)
|
|
total_terms_removed += state_terms_removed
|
|
spinner.succeed(f'Finished processing {state_dir}. Terms removed: {state_terms_removed}')
|
|
|
|
print(f"Total Terms Removed: {total_terms_removed}")
|
|
|
|
if __name__ == "__main__":
|
|
strip_negative_terms()
|
|
|