103 lines
3.5 KiB
Python
Executable file
103 lines
3.5 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# Script Name: standardize-location-data
|
|
|
|
import os
|
|
import csv
|
|
from pyfiglet import Figlet
|
|
from halo import Halo
|
|
|
|
# Define the paths based on the project details
|
|
PROJECT_ROOT = "/home/ld/mgk-scrapes"
|
|
DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data")
|
|
STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5")
|
|
|
|
def process_csv_file_standardize_location(file_path):
|
|
"""Process a CSV file to standardize location data."""
|
|
with open(file_path, 'r') as file:
|
|
reader = csv.reader(file)
|
|
rows = list(reader)
|
|
|
|
if not rows:
|
|
return 0
|
|
|
|
headers = rows[0]
|
|
data_rows = rows[1:]
|
|
|
|
# Ensure the necessary columns exist
|
|
if 'GBP Location' not in headers:
|
|
return 0
|
|
|
|
# Create new columns for municipality and state if they don't exist
|
|
if 'GBP Location Municipality' not in headers:
|
|
headers.append('GBP Location Municipality')
|
|
if 'GBP Location State' not in headers:
|
|
headers.append('GBP Location State')
|
|
|
|
municipality_index = headers.index('GBP Location Municipality')
|
|
state_index = headers.index('GBP Location State')
|
|
location_index = headers.index('GBP Location')
|
|
|
|
changes = 0
|
|
for row in data_rows:
|
|
while len(row) < len(headers):
|
|
row.append('')
|
|
|
|
location = row[location_index]
|
|
|
|
# Standardize location data by removing ", United States"
|
|
if ', United States' in location:
|
|
location = location.replace(', United States', '').strip()
|
|
|
|
# Split location into municipality and state
|
|
if ', ' in location:
|
|
parts = location.split(', ')
|
|
if len(parts) == 2:
|
|
municipality, state = parts
|
|
else:
|
|
municipality = ', '.join(parts[:-1])
|
|
state = parts[-1]
|
|
row[municipality_index] = municipality
|
|
row[state_index] = state
|
|
row[location_index] = ''
|
|
changes += 1
|
|
else:
|
|
row[municipality_index] = ''
|
|
row[state_index] = ''
|
|
|
|
# Write the modified rows back to the CSV file
|
|
with open(file_path, 'w', newline='') as file:
|
|
writer = csv.writer(file)
|
|
writer.writerow(headers)
|
|
writer.writerows(data_rows)
|
|
|
|
return changes
|
|
|
|
def standardize_location_data():
|
|
"""Standardize location data in all CSV files in the stage 5 directory and tally the results."""
|
|
figlet = Figlet(font='slant')
|
|
print(figlet.renderText('Standardize Location Data'))
|
|
|
|
total_changes = 0
|
|
|
|
for state_dir in os.listdir(STAGE_5_DIRECTORY):
|
|
state_path = os.path.join(STAGE_5_DIRECTORY, state_dir)
|
|
if os.path.isdir(state_path):
|
|
spinner = Halo(text=f'Processing {state_dir}', spinner='dots')
|
|
spinner.start()
|
|
state_changes = 0
|
|
for county_dir in os.listdir(state_path):
|
|
county_path = os.path.join(state_path, county_dir)
|
|
if os.path.isdir(county_path):
|
|
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
|
for file in csv_files:
|
|
file_path = os.path.join(county_path, file)
|
|
state_changes += process_csv_file_standardize_location(file_path)
|
|
total_changes += state_changes
|
|
spinner.succeed(f'Finished processing {state_dir}. Changes made: {state_changes}')
|
|
|
|
print(f"Total Changes Made: {total_changes}")
|
|
|
|
if __name__ == "__main__":
|
|
standardize_location_data()
|
|
|