66 lines
2.2 KiB
Python
Executable file
66 lines
2.2 KiB
Python
Executable file
#!/usr/bin/env python
|
|
|
|
import os
|
|
import csv
|
|
import re
|
|
|
|
# Define the paths based on the project details
|
|
PROJECT_ROOT = "/home/ld/mgk-scrapes"
|
|
CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data")
|
|
DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data")
|
|
STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2")
|
|
|
|
# Regular expression to match the pattern "-dd"
|
|
pattern = re.compile(r'^-\d{2}$')
|
|
|
|
def check_csv_for_pattern(file_path):
|
|
"""Check if the CSV file contains a column with mostly strings that match the pattern."""
|
|
with open(file_path, 'r') as file:
|
|
reader = csv.reader(file)
|
|
rows = list(reader)
|
|
|
|
if not rows:
|
|
return False
|
|
|
|
column_counts = [0] * len(rows[0])
|
|
|
|
for row in rows:
|
|
for idx, cell in enumerate(row):
|
|
if pattern.match(cell):
|
|
column_counts[idx] += 1
|
|
|
|
# Consider a column with mostly matching patterns if more than 50% of its cells match
|
|
threshold = len(rows) / 2
|
|
return any(count > threshold for count in column_counts)
|
|
|
|
def find_csv_files_with_pattern(stage_directory):
|
|
"""Find all CSV files in the stage directory that contain columns with the specified pattern."""
|
|
matching_files = []
|
|
|
|
for state_dir in os.listdir(stage_directory):
|
|
state_path = os.path.join(stage_directory, state_dir)
|
|
if os.path.isdir(state_path):
|
|
for county_dir in os.listdir(state_path):
|
|
county_path = os.path.join(state_path, county_dir)
|
|
if os.path.isdir(county_path):
|
|
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
|
for file in csv_files:
|
|
file_path = os.path.join(county_path, file)
|
|
if check_csv_for_pattern(file_path):
|
|
matching_files.append(file_path)
|
|
return matching_files
|
|
|
|
def main():
|
|
matching_files = find_csv_files_with_pattern(STAGE_2_DIRECTORY)
|
|
|
|
if matching_files:
|
|
print("CSV files containing columns with '-dd' patterns:")
|
|
for file in matching_files:
|
|
print(file)
|
|
else:
|
|
print("No CSV files with the '-dd' pattern found.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|