118 lines
4.2 KiB
Python
Executable file
118 lines
4.2 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# Script Name: remove-obviously-bad-columns
|
|
|
|
import os
|
|
import csv
|
|
import sys
|
|
from pyfiglet import Figlet
|
|
import halo
|
|
|
|
# Define the paths based on the project details
|
|
PROJECT_ROOT = "/home/ld/mgk-scrapes"
|
|
CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data")
|
|
DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data")
|
|
STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1")
|
|
|
|
def process_csv_file(file_path):
|
|
"""
|
|
Process a CSV file to remove columns containing specific unwanted patterns.
|
|
|
|
Parameters:
|
|
file_path (str): The path to the CSV file to be processed.
|
|
|
|
Returns:
|
|
int: The number of columns removed from the CSV file.
|
|
"""
|
|
with open(file_path, 'r') as file:
|
|
reader = csv.reader(file)
|
|
rows = list(reader)
|
|
|
|
if not rows:
|
|
return 0
|
|
|
|
num_columns = len(rows[0])
|
|
columns_to_delete = [False] * num_columns
|
|
|
|
for col in range(num_columns - 1, -1, -1):
|
|
for row in rows:
|
|
cell_value = row[col]
|
|
if (
|
|
"googleusercontent" in cell_value or
|
|
"googleapis" in cell_value or
|
|
"www.google.com/maps/vt/data" in cell_value or
|
|
" Opens " in cell_value or
|
|
"geocode" in cell_value or
|
|
"gstatic.com" in cell_value or
|
|
cell_value.startswith("tel:+") or
|
|
cell_value.startswith("⋅ ") or
|
|
cell_value in ["Open", "Quote", "Share", "Closed", "Closes soon", "Open 24 hours",
|
|
"Directions", "Booking", "Website", "https://www.google.com/#", "Call",
|
|
"Provides:", "Online estimates", "Onsite services", "No ratings or reviews"] or
|
|
cell_value.startswith('"')
|
|
):
|
|
columns_to_delete[col] = True
|
|
break
|
|
|
|
modified_rows = []
|
|
for row in rows:
|
|
modified_row = [cell for idx, cell in enumerate(row) if not columns_to_delete[idx]]
|
|
modified_rows.append(modified_row)
|
|
|
|
with open(file_path, 'w', newline='') as file:
|
|
writer = csv.writer(file)
|
|
writer.writerows(modified_rows)
|
|
|
|
return sum(columns_to_delete)
|
|
|
|
def process_csv_files(stage_directory):
|
|
"""
|
|
Process all CSV files in the given stage directory to remove unwanted columns.
|
|
|
|
Parameters:
|
|
stage_directory (str): The path to the stage directory containing the CSV files to be processed.
|
|
"""
|
|
total_bad_columns = 0
|
|
for state_dir in os.listdir(stage_directory):
|
|
state_path = os.path.join(stage_directory, state_dir)
|
|
if os.path.isdir(state_path):
|
|
print(f"Removing obviously bad columns in {state_dir}...")
|
|
spinner = halo.Halo(spinner='dots', color='green')
|
|
spinner.start()
|
|
|
|
state_bad_columns = 0
|
|
for county_dir in os.listdir(state_path):
|
|
county_path = os.path.join(state_path, county_dir)
|
|
if os.path.isdir(county_path):
|
|
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
|
for file in csv_files:
|
|
file_path = os.path.join(county_path, file)
|
|
state_bad_columns += process_csv_file(file_path)
|
|
|
|
total_bad_columns += state_bad_columns
|
|
spinner.succeed(f"Removed bad columns in {state_dir}. Bad columns found: {state_bad_columns}")
|
|
|
|
return total_bad_columns
|
|
|
|
def main():
|
|
figlet = Figlet(font='slant')
|
|
script_name = "remove-obviously-bad-columns".replace("-", " ").title()
|
|
print(figlet.renderText(script_name))
|
|
|
|
stage_directory = STAGE_1_DIRECTORY
|
|
|
|
if not os.path.exists(stage_directory):
|
|
print(f"Error: The directory {stage_directory} does not exist.")
|
|
sys.exit(1)
|
|
|
|
print(f"Removing obviously bad columns in {stage_directory}...")
|
|
total_bad_columns = process_csv_files(stage_directory)
|
|
|
|
# Final success message with a single colored checkmark
|
|
final_spinner = halo.Halo(spinner='dots', color='green')
|
|
final_spinner.start()
|
|
final_spinner.succeed(f"Total number of obviously bad columns deleted: {total_bad_columns}")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|