Fixed the set to accomodate for Kittys new data.
This commit is contained in:
parent
03ab193552
commit
9e81ea008f
69
bin/list-unknown-terms.py
Executable file
69
bin/list-unknown-terms.py
Executable file
|
|
@ -0,0 +1,69 @@
|
|||
#!/usr/bin/env python
|
||||
# Script Name: list-unknown-terms.py
|
||||
|
||||
import os
|
||||
import csv
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from tqdm import tqdm
|
||||
from pyfiglet import Figlet
|
||||
from halo import Halo
|
||||
|
||||
def process_csv_file(file_path):
|
||||
unknown_terms = defaultdict(set)
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as file:
|
||||
reader = csv.DictReader(file)
|
||||
for row in reader:
|
||||
for header, value in row.items():
|
||||
if header.startswith('unknown-') and value.strip():
|
||||
unknown_terms[header].add(value.strip().lower())
|
||||
except Exception as e:
|
||||
print(f"Error processing file {file_path}: {str(e)}")
|
||||
return unknown_terms
|
||||
|
||||
def main(input_file):
|
||||
figlet = Figlet(font='slant')
|
||||
print(figlet.renderText("List Unknown Terms"))
|
||||
|
||||
all_unknown_terms = set()
|
||||
|
||||
spinner = Halo(text='Reading input file', spinner='dots')
|
||||
spinner.start()
|
||||
|
||||
try:
|
||||
with open(input_file, 'r') as file:
|
||||
csv_files = [row[0] for row in csv.reader(file)]
|
||||
except Exception as e:
|
||||
spinner.fail(f"Error reading input file: {str(e)}")
|
||||
sys.exit(1)
|
||||
|
||||
spinner.succeed('Input file read successfully')
|
||||
|
||||
print("Processing CSV files...")
|
||||
for file_path in tqdm(csv_files, unit='file'):
|
||||
unknown_terms = process_csv_file(file_path)
|
||||
for terms in unknown_terms.values():
|
||||
all_unknown_terms.update(terms)
|
||||
|
||||
spinner = Halo(text='Sorting and deduplicating terms', spinner='dots')
|
||||
spinner.start()
|
||||
|
||||
sorted_terms = sorted(all_unknown_terms)
|
||||
|
||||
output_file = 'list-of-unknowns.txt'
|
||||
try:
|
||||
with open(output_file, 'w', encoding='utf-8') as file:
|
||||
for term in sorted_terms:
|
||||
file.write(f"{term}\n")
|
||||
spinner.succeed(f"Terms saved to {output_file}")
|
||||
except Exception as e:
|
||||
spinner.fail(f"Error writing to {output_file}: {str(e)}")
|
||||
|
||||
print(f"Total unique unknown terms found: {len(sorted_terms)}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python list-unknown-terms.py <input_csv_file>")
|
||||
sys.exit(1)
|
||||
main(sys.argv[1])
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
#/usr/bin/env python
|
||||
#!/usr/bin/env python
|
||||
# Script Name: process-stage-6
|
||||
|
||||
import os
|
||||
|
|
|
|||
83
bin/scan-columns-for.py
Executable file
83
bin/scan-columns-for.py
Executable file
|
|
@ -0,0 +1,83 @@
|
|||
#!/usr/bin/env python
|
||||
# Script Name: scan-columns-for.py
|
||||
|
||||
import os
|
||||
import csv
|
||||
import sys
|
||||
import glob
|
||||
from tqdm import tqdm
|
||||
from pyfiglet import Figlet
|
||||
from halo import Halo
|
||||
|
||||
def scan_csv_file(file_path, expected_column, search_term):
|
||||
matches = []
|
||||
search_term_lower = search_term.lower()
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as file:
|
||||
reader = csv.DictReader(file)
|
||||
for row in reader:
|
||||
for column, value in row.items():
|
||||
if column != expected_column and value.lower() == search_term_lower:
|
||||
matches.append({
|
||||
'CSV File': file_path,
|
||||
'Expected Column': expected_column,
|
||||
'Search Term': search_term,
|
||||
'Found In Column': column
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"Error processing file {file_path}: {str(e)}")
|
||||
return matches
|
||||
|
||||
def main(args):
|
||||
if len(args) < 3:
|
||||
print("Usage: python scan-columns-for.py <csv_file_pattern> <expected_column> <search_term>")
|
||||
sys.exit(1)
|
||||
|
||||
csv_file_pattern = args[0]
|
||||
expected_column = args[1]
|
||||
search_term = ' '.join(args[2:]) # Join all remaining arguments as the search term
|
||||
|
||||
figlet = Figlet(font='slant')
|
||||
print(figlet.renderText("Scan Columns For"))
|
||||
|
||||
all_matches = []
|
||||
|
||||
spinner = Halo(text='Preparing to scan files', spinner='dots')
|
||||
spinner.start()
|
||||
|
||||
# Expand the file pattern
|
||||
csv_file_paths = glob.glob(csv_file_pattern)
|
||||
|
||||
if not csv_file_paths:
|
||||
spinner.fail(f"No CSV files found matching the pattern: {csv_file_pattern}")
|
||||
sys.exit(1)
|
||||
|
||||
spinner.succeed(f'Found {len(csv_file_paths)} CSV files to scan')
|
||||
|
||||
print(f"Scanning CSV files for exact match of '{search_term}' outside of '{expected_column}'...")
|
||||
for file_path in tqdm(csv_file_paths, unit='file'):
|
||||
matches = scan_csv_file(file_path, expected_column, search_term)
|
||||
all_matches.extend(matches)
|
||||
|
||||
spinner = Halo(text='Writing results', spinner='dots')
|
||||
spinner.start()
|
||||
|
||||
output_file = 'located-matches.csv'
|
||||
try:
|
||||
with open(output_file, 'w', encoding='utf-8', newline='') as file:
|
||||
if all_matches:
|
||||
fieldnames = ['CSV File', 'Expected Column', 'Search Term', 'Found In Column']
|
||||
writer = csv.DictWriter(file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for match in all_matches:
|
||||
writer.writerow(match)
|
||||
spinner.succeed(f"Results saved to {output_file}")
|
||||
else:
|
||||
spinner.info("No matches found")
|
||||
except Exception as e:
|
||||
spinner.fail(f"Error writing to {output_file}: {str(e)}")
|
||||
|
||||
print(f"Total matches found: {len(all_matches)}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
60
bin/stage-1/prepare-data-and-configs.py
Executable file
60
bin/stage-1/prepare-data-and-configs.py
Executable file
|
|
@ -0,0 +1,60 @@
|
|||
#!/usr/bin/env python
|
||||
# Script Name: prepare-data-and-configs
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from pyfiglet import Figlet
|
||||
from halo import Halo
|
||||
|
||||
def find_project_root(current_path):
|
||||
"""Find the project root by locating the 'bin' directory."""
|
||||
while current_path != os.path.dirname(current_path):
|
||||
if os.path.basename(current_path) == 'bin':
|
||||
return os.path.dirname(current_path)
|
||||
current_path = os.path.dirname(current_path)
|
||||
raise FileNotFoundError("Could not find 'bin' directory in the path hierarchy.")
|
||||
|
||||
# Define the paths based on the project details
|
||||
script_path = os.path.abspath(__file__)
|
||||
PROJECT_ROOT = find_project_root(script_path)
|
||||
CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data")
|
||||
DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data")
|
||||
CONF_D_DIRECTORY = os.path.join(PROJECT_ROOT, "conf.d")
|
||||
|
||||
def copy_config_files():
|
||||
"""Copy *.txt files from conf.d to .data if they do not already exist."""
|
||||
txt_files = [f for f in os.listdir(CONF_D_DIRECTORY) if f.endswith('.txt')]
|
||||
|
||||
if not txt_files:
|
||||
print("No .txt files found in conf.d directory.")
|
||||
return
|
||||
|
||||
for txt_file in txt_files:
|
||||
src_file = os.path.join(CONF_D_DIRECTORY, txt_file)
|
||||
dest_file = os.path.join(DATA_DIRECTORY, txt_file)
|
||||
|
||||
if not os.path.exists(dest_file):
|
||||
print(f"Copying {txt_file} to .data directory...")
|
||||
shutil.copy2(src_file, dest_file)
|
||||
else:
|
||||
print(f"{txt_file} already exists in .data directory, skipping copy.")
|
||||
|
||||
def main():
|
||||
figlet = Figlet(font='slant')
|
||||
script_name = "prepare-data-and-configs".replace("-", " ").title()
|
||||
print(figlet.renderText(script_name))
|
||||
|
||||
print("Preparing data and configs...")
|
||||
|
||||
# Ensure the .data directory exists
|
||||
os.makedirs(DATA_DIRECTORY, exist_ok=True)
|
||||
|
||||
# Copy config files
|
||||
spinner = Halo(text='Copying config files', spinner='dots')
|
||||
spinner.start()
|
||||
copy_config_files()
|
||||
spinner.succeed("Config files copied or already present.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
|
@ -39,7 +39,8 @@ def process_csv_file(file_path):
|
|||
return
|
||||
|
||||
def count_matches(column_data):
|
||||
matches = sum(1 for cell in column_data if re.match(r'^[\w\s]+, [A-Z]{2}$', cell) or re.match(r'^[\w\s]+, [A-Z]{2}, United States$', cell))
|
||||
pattern = r'^[\w\s\.\'-]+, [A-Z]{2}(, United States)?$'
|
||||
matches = sum(1 for cell in column_data if re.match(pattern, cell))
|
||||
return matches
|
||||
|
||||
# Evaluate all columns
|
||||
|
|
@ -96,4 +97,3 @@ def main():
|
|||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
|
|
|||
153
bin/stage-3/discover-categories-and-services.py
Executable file
153
bin/stage-3/discover-categories-and-services.py
Executable file
|
|
@ -0,0 +1,153 @@
|
|||
#!/usr/bin/env python
|
||||
# Script Name: discover-categories-and-services
|
||||
|
||||
import os
|
||||
import csv
|
||||
from pyfiglet import Figlet
|
||||
from halo import Halo
|
||||
|
||||
def find_project_root(current_path):
|
||||
"""Find the project root by locating the 'bin' directory."""
|
||||
while current_path != os.path.dirname(current_path):
|
||||
if os.path.basename(current_path) == 'bin':
|
||||
return os.path.dirname(current_path)
|
||||
current_path = os.path.dirname(current_path)
|
||||
raise FileNotFoundError("Could not find 'bin' directory in the path hierarchy.")
|
||||
|
||||
# Define the paths based on the project details
|
||||
script_path = os.path.abspath(__file__)
|
||||
PROJECT_ROOT = find_project_root(script_path)
|
||||
DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data")
|
||||
STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3")
|
||||
|
||||
DISCOVERED_CATEGORIES_FILE = os.path.join(DATA_DIRECTORY, 'discovered-categories.txt')
|
||||
DISCOVERED_SERVICES_FILE = os.path.join(DATA_DIRECTORY, 'discovered-services.txt')
|
||||
|
||||
GBP_CATEGORIES_FILE = os.path.join(DATA_DIRECTORY, 'gbp-business-categories.txt')
|
||||
GBP_SERVICES_FILE = os.path.join(DATA_DIRECTORY, 'gbp-matching-services.txt')
|
||||
|
||||
NEW_CATEGORIES_FILE = os.path.join(DATA_DIRECTORY, 'new-categories.txt')
|
||||
NEW_SERVICES_FILE = os.path.join(DATA_DIRECTORY, 'new-services.txt')
|
||||
|
||||
BAD_MATCHING_SERVICES_FILE = os.path.join(DATA_DIRECTORY, 'bad-matching-services.txt')
|
||||
|
||||
def check_and_delete_file(file_path):
|
||||
"""Check if a file exists and delete it if it does."""
|
||||
if os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
print(f"Deleted existing file: {file_path}")
|
||||
else:
|
||||
print(f"No existing file found: {file_path}")
|
||||
|
||||
def discover_entries():
|
||||
categories = set()
|
||||
services = set()
|
||||
|
||||
for state_dir in os.listdir(STAGE_3_DIRECTORY):
|
||||
state_path = os.path.join(STAGE_3_DIRECTORY, state_dir)
|
||||
if os.path.isdir(state_path):
|
||||
for county_dir in os.listdir(state_path):
|
||||
county_path = os.path.join(state_path, county_dir)
|
||||
if os.path.isdir(county_path):
|
||||
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
||||
for file in csv_files:
|
||||
file_path = os.path.join(county_path, file)
|
||||
with open(file_path, 'r') as csv_file:
|
||||
reader = csv.DictReader(csv_file)
|
||||
rows = list(reader)
|
||||
|
||||
if not rows:
|
||||
continue
|
||||
|
||||
headers = rows[0].keys()
|
||||
modified_rows = []
|
||||
|
||||
for row in rows:
|
||||
if 'GBP Business Category' in row:
|
||||
category = row['GBP Business Category'].strip().lower()
|
||||
row['GBP Business Category'] = category # Transform in the CSV
|
||||
if category:
|
||||
categories.add(category)
|
||||
if 'GBP Matching Service' in row:
|
||||
service = row['GBP Matching Service'].strip().lower()
|
||||
row['GBP Matching Service'] = service # Transform in the CSV
|
||||
if service:
|
||||
services.add(service)
|
||||
modified_rows.append(row)
|
||||
|
||||
# Write the transformed data back to the CSV file
|
||||
with open(file_path, 'w', newline='') as csv_file:
|
||||
writer = csv.DictWriter(csv_file, fieldnames=headers)
|
||||
writer.writeheader()
|
||||
writer.writerows(modified_rows)
|
||||
|
||||
return categories, services
|
||||
|
||||
def write_to_file(file_path, entries):
|
||||
with open(file_path, 'w') as file:
|
||||
for entry in sorted(entries):
|
||||
file.write(f"{entry}\n")
|
||||
|
||||
def load_existing_entries(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
return set()
|
||||
with open(file_path, 'r') as file:
|
||||
return {line.strip().lower() for line in file if line.strip()}
|
||||
|
||||
def load_bad_matching_services(file_path):
|
||||
"""Load bad matching services from a file."""
|
||||
if not os.path.exists(file_path):
|
||||
print(f"Warning: {file_path} not found. Proceeding without bad matching services.")
|
||||
return set()
|
||||
with open(file_path, 'r') as file:
|
||||
return {line.strip().lower() for line in file if line.strip()}
|
||||
|
||||
def find_new_entries(discovered, existing, bad_entries, existing_services):
|
||||
"""Find new entries, excluding existing entries, bad entries, and existing services."""
|
||||
return discovered - existing - bad_entries - existing_services
|
||||
|
||||
def main():
|
||||
figlet = Figlet(font='slant')
|
||||
script_name = "discover-categories-and-services".replace("-", " ").title()
|
||||
print(figlet.renderText(script_name))
|
||||
|
||||
# Check and delete existing files
|
||||
check_and_delete_file(DISCOVERED_CATEGORIES_FILE)
|
||||
check_and_delete_file(DISCOVERED_SERVICES_FILE)
|
||||
check_and_delete_file(NEW_CATEGORIES_FILE)
|
||||
check_and_delete_file(NEW_SERVICES_FILE)
|
||||
|
||||
spinner = Halo(text='Discovering categories and services', spinner='dots')
|
||||
spinner.start()
|
||||
|
||||
categories, services = discover_entries()
|
||||
|
||||
write_to_file(DISCOVERED_CATEGORIES_FILE, categories)
|
||||
write_to_file(DISCOVERED_SERVICES_FILE, services)
|
||||
|
||||
spinner.succeed("Discovery complete.")
|
||||
|
||||
print(f"Total unique categories discovered: {len(categories)}")
|
||||
print(f"Total unique services discovered: {len(services)}")
|
||||
|
||||
spinner = Halo(text='Finding new categories and services', spinner='dots')
|
||||
spinner.start()
|
||||
|
||||
existing_categories = load_existing_entries(GBP_CATEGORIES_FILE)
|
||||
existing_services = load_existing_entries(GBP_SERVICES_FILE)
|
||||
bad_matching_services = load_bad_matching_services(BAD_MATCHING_SERVICES_FILE)
|
||||
|
||||
new_categories = find_new_entries(categories, existing_categories, bad_matching_services, existing_services)
|
||||
new_services = find_new_entries(services, existing_services, bad_matching_services, set())
|
||||
|
||||
write_to_file(NEW_CATEGORIES_FILE, new_categories)
|
||||
write_to_file(NEW_SERVICES_FILE, new_services)
|
||||
|
||||
spinner.succeed("New entries determination complete.")
|
||||
|
||||
print(f"Total new categories found: {len(new_categories)}")
|
||||
print(f"Total new services found: {len(new_services)}")
|
||||
print(f"Total bad matching services excluded: {len(bad_matching_services)}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -5,6 +5,8 @@ import os
|
|||
import csv
|
||||
import re
|
||||
import hashlib
|
||||
import validators
|
||||
from urllib.parse import urlparse
|
||||
from pyfiglet import Figlet
|
||||
from halo import Halo
|
||||
|
||||
|
|
@ -40,10 +42,11 @@ GBP_MATCHING_SERVICES = read_terms(os.path.join(DATA_DIRECTORY, 'gbp-matching-se
|
|||
UNKNOWN_BLACKLIST = read_terms(os.path.join(DATA_DIRECTORY, 'unknown-blacklist.txt'))
|
||||
|
||||
# Patterns for different data types
|
||||
LOCATION_PATTERN = re.compile(r'^[\w\s]+, [A-Z]{2}(, United States)?$', re.IGNORECASE)
|
||||
LOCATION_PATTERN = re.compile(r'^[\w\s\.-]+,\s[A-Z]{2}(,\sUnited States)?$', re.IGNORECASE)
|
||||
YIB_PATTERN = re.compile(r'^\d+\+ years in business$', re.IGNORECASE)
|
||||
REVIEW_RATING_PATTERN = re.compile(r'^[1-5]\.\d$', re.IGNORECASE)
|
||||
REVIEW_COUNT_PATTERN = re.compile(r'^\(\d+\)$', re.IGNORECASE)
|
||||
URL_START_PATTERN = re.compile(r'^https?://', re.IGNORECASE)
|
||||
|
||||
def calculate_checksum(file_path):
|
||||
"""Calculate the MD5 checksum of a file."""
|
||||
|
|
@ -53,6 +56,14 @@ def calculate_checksum(file_path):
|
|||
hash_md5.update(chunk)
|
||||
return hash_md5.hexdigest()
|
||||
|
||||
def is_valid_url(url):
|
||||
"""Check if a given string is a valid URL."""
|
||||
try:
|
||||
result = validators.url(url)
|
||||
return result
|
||||
except validators.ValidationFailure:
|
||||
return False
|
||||
|
||||
def process_csv_file(file_path):
|
||||
"""Process a CSV file to reshift terms and return the number of terms shifted."""
|
||||
with open(file_path, 'r') as file:
|
||||
|
|
@ -62,47 +73,108 @@ def process_csv_file(file_path):
|
|||
if not rows:
|
||||
return 0
|
||||
|
||||
headers = reader.fieldnames
|
||||
headers = list(reader.fieldnames) # Convert to list to allow modification
|
||||
terms_shifted = 0
|
||||
|
||||
# Check for missing columns and add them if necessary
|
||||
expected_columns = ['GBP Location', 'YiB', 'GBP Review Rating', 'GBP Review Count',
|
||||
'GBP Business Category', 'GBP Matching Service', 'GBP Business Website']
|
||||
for column in expected_columns:
|
||||
if column not in headers:
|
||||
headers.append(column)
|
||||
for row in rows:
|
||||
row[column] = ''
|
||||
|
||||
for row in rows:
|
||||
# New logic for GBP Business Category and GBP Matching Service
|
||||
if 'GBP Business Category' in row and row['GBP Business Category']:
|
||||
category = row['GBP Business Category'].strip().lower()
|
||||
if category not in GBP_BUSINESS_CATEGORIES:
|
||||
if 'GBP Matching Service' in row:
|
||||
if not row['GBP Matching Service']:
|
||||
row['GBP Matching Service'] = category
|
||||
row['GBP Business Category'] = ''
|
||||
terms_shifted += 1
|
||||
else:
|
||||
service = row['GBP Matching Service'].strip().lower()
|
||||
if service in GBP_BUSINESS_CATEGORIES:
|
||||
# Swap the contents
|
||||
row['GBP Business Category'] = service
|
||||
row['GBP Matching Service'] = category
|
||||
terms_shifted += 1
|
||||
else:
|
||||
# Append with separator
|
||||
row['GBP Matching Service'] += f" - {category}"
|
||||
row['GBP Business Category'] = ''
|
||||
terms_shifted += 1
|
||||
|
||||
# Existing logic for unknown columns
|
||||
for header in headers:
|
||||
if header.startswith('unknown-'):
|
||||
cell = row[header].strip().lower()
|
||||
cell = row[header].strip()
|
||||
|
||||
# New URL handling logic
|
||||
if URL_START_PATTERN.match(cell):
|
||||
if is_valid_url(cell):
|
||||
if not row['GBP Business Website']:
|
||||
row['GBP Business Website'] = cell
|
||||
row[header] = ''
|
||||
terms_shifted += 1
|
||||
else:
|
||||
row[header] = '' # Remove the URL from unknown column if it can't be copied
|
||||
terms_shifted += 1
|
||||
continue
|
||||
|
||||
cell_lower = cell.lower() # Convert to lowercase for other checks
|
||||
|
||||
if LOCATION_PATTERN.match(cell):
|
||||
if not row['GBP Location']:
|
||||
row['GBP Location'] = row[header]
|
||||
row['GBP Location'] = cell # Use the original cell value, not lowercase
|
||||
row[header] = ''
|
||||
terms_shifted += 1
|
||||
elif YIB_PATTERN.match(cell):
|
||||
else:
|
||||
row[header] = '' # Remove the location from unknown column if it can't be copied
|
||||
terms_shifted += 1
|
||||
elif YIB_PATTERN.match(cell_lower):
|
||||
if not row['YiB']:
|
||||
row['YiB'] = row[header]
|
||||
row['YiB'] = cell
|
||||
row[header] = ''
|
||||
terms_shifted += 1
|
||||
elif REVIEW_RATING_PATTERN.match(cell):
|
||||
else:
|
||||
row[header] = '' # Remove the YiB from unknown column if it can't be copied
|
||||
terms_shifted += 1
|
||||
elif REVIEW_RATING_PATTERN.match(cell_lower):
|
||||
if not row['GBP Review Rating']:
|
||||
row['GBP Review Rating'] = row[header]
|
||||
row['GBP Review Rating'] = cell
|
||||
row[header] = ''
|
||||
terms_shifted += 1
|
||||
elif REVIEW_COUNT_PATTERN.match(cell):
|
||||
else:
|
||||
row[header] = '' # Remove the review rating from unknown column if it can't be copied
|
||||
terms_shifted += 1
|
||||
elif REVIEW_COUNT_PATTERN.match(cell_lower):
|
||||
if not row['GBP Review Count']:
|
||||
row['GBP Review Count'] = row[header]
|
||||
row['GBP Review Count'] = cell
|
||||
row[header] = ''
|
||||
terms_shifted += 1
|
||||
elif cell in GBP_BUSINESS_CATEGORIES:
|
||||
else:
|
||||
row[header] = '' # Remove the review count from unknown column if it can't be copied
|
||||
terms_shifted += 1
|
||||
elif cell_lower in GBP_BUSINESS_CATEGORIES:
|
||||
if not row['GBP Business Category']:
|
||||
row['GBP Business Category'] = row[header]
|
||||
row['GBP Business Category'] = cell
|
||||
row[header] = ''
|
||||
terms_shifted += 1
|
||||
elif cell in GBP_MATCHING_SERVICES:
|
||||
if row['GBP Matching Service']: # Check if the column is not empty
|
||||
if row['GBP Matching Service'].strip().lower() in UNKNOWN_BLACKLIST:
|
||||
row['GBP Matching Service'] = '' # Clear the cell
|
||||
if not row['GBP Matching Service']: # Now check if it's empty again
|
||||
row['GBP Matching Service'] = row[header]
|
||||
else:
|
||||
row[header] = '' # Remove the business category from unknown column if it can't be copied
|
||||
terms_shifted += 1
|
||||
elif cell_lower in GBP_MATCHING_SERVICES:
|
||||
if not row['GBP Matching Service']:
|
||||
row['GBP Matching Service'] = cell
|
||||
row[header] = ''
|
||||
terms_shifted += 1
|
||||
else:
|
||||
row[header] = '' # Remove the matching service from unknown column if it can't be copied
|
||||
terms_shifted += 1
|
||||
|
||||
# Write the modified rows back to the CSV file
|
||||
with open(file_path, 'w', newline='') as file:
|
||||
|
|
@ -118,8 +190,6 @@ def reshift_terms():
|
|||
print(figlet.renderText('Reshift Terms'))
|
||||
|
||||
total_terms_shifted = 0
|
||||
before_checksums = {}
|
||||
after_checksums = {}
|
||||
|
||||
for state_dir in os.listdir(STAGE_4_DIRECTORY):
|
||||
state_path = os.path.join(STAGE_4_DIRECTORY, state_dir)
|
||||
|
|
@ -133,19 +203,11 @@ def reshift_terms():
|
|||
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
||||
for file in csv_files:
|
||||
file_path = os.path.join(county_path, file)
|
||||
before_checksums[file_path] = calculate_checksum(file_path)
|
||||
state_terms_shifted += process_csv_file(file_path)
|
||||
after_checksums[file_path] = calculate_checksum(file_path)
|
||||
total_terms_shifted += state_terms_shifted
|
||||
spinner.succeed(f'Finished processing {state_dir}. Terms shifted: {state_terms_shifted}')
|
||||
|
||||
# Verification step
|
||||
for file_path in before_checksums:
|
||||
if before_checksums[file_path] != after_checksums[file_path]:
|
||||
print(f"Changes detected in: {file_path}")
|
||||
|
||||
print(f"Total Terms Shifted: {total_terms_shifted}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
reshift_terms()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
#!/usr/bin/env python
|
||||
# Script Name: strip-neg-patterns-from-unknown-cols
|
||||
|
||||
import os
|
||||
import csv
|
||||
import re
|
||||
|
|
@ -23,8 +22,22 @@ STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4")
|
|||
|
||||
# Define the pattern for foreign phone numbers
|
||||
foreign_phone_patterns = [
|
||||
re.compile(r'^\+\d{1,3} \d{3,4} \d{3} \d{3,4}$'),
|
||||
re.compile(r'^\+\d{2} \d{9,10}$') # New pattern for +61 4239543977
|
||||
re.compile(r'^\+\d{1,3}\s?\d{1,4}[\s.-]?\d{1,4}[\s.-]?\d{1,4}$'), # General pattern for most international numbers
|
||||
re.compile(r'^\+\d{2,3}\s\d{1,2}\s\d{2,4}\s\d{2,4}$'), # Pattern for numbers with multiple spaces
|
||||
re.compile(r'^\+\d{2,3}\s\d{3,4}-\d{3,4}-\d{3,4}$'), # Pattern for numbers with hyphens
|
||||
re.compile(r'^\+\d{2,3}\s\d{2}\s\d{3}\s\d{2}\s\d{2}$'), # Pattern for numbers with multiple groups
|
||||
re.compile(r'^\+\d{2,3}\s\d{3,4}\s\d{5,6}$'), # Pattern for numbers with two groups after country code
|
||||
re.compile(r'^\+\d{1,3}\s\d{9,11}$'), # Pattern for numbers with country code and 9-11 digits
|
||||
re.compile(r'^\+\d{2,3}\s\d{1,2}\s\d{4}\s\d{4}$'), # Pattern for numbers like +xx x xxxx xxxx
|
||||
re.compile(r'^\+\d{2}\s\d\s\d{2}\s\d{2}\s\d{2}\s\d{2}$'), # Pattern for +33 1 72 67 69 14
|
||||
re.compile(r'^\+\d{2}\s\d{3}\s\d{2}\s\d{2}\s\d{2}$'), # Pattern for +34 651 32 11 25
|
||||
re.compile(r'^\+\d{2}\s\d{2}\s\d{2}\s\d{2}\s\d{2}$'), # Pattern for +46 31 23 06 50
|
||||
re.compile(r'^\+91\s\d{5}\s\d{5}$'), # Pattern for Indian numbers like +91 62917 98998
|
||||
]
|
||||
|
||||
# Define the pattern for websites to remove
|
||||
website_patterns = [
|
||||
re.compile(r'https://www\.google\.com/maps')
|
||||
]
|
||||
|
||||
def process_csv_file(file_path):
|
||||
|
|
@ -34,29 +47,38 @@ def process_csv_file(file_path):
|
|||
rows = list(reader)
|
||||
|
||||
if not rows:
|
||||
return 0
|
||||
return 0, 0
|
||||
|
||||
headers = rows[0]
|
||||
data_rows = rows[1:]
|
||||
|
||||
modified_rows = [headers]
|
||||
terms_removed = 0
|
||||
websites_removed = 0
|
||||
|
||||
for row in data_rows:
|
||||
for i, header in enumerate(headers):
|
||||
if header.startswith("unknown-"):
|
||||
# Check for phone numbers
|
||||
for pattern in foreign_phone_patterns:
|
||||
if pattern.match(row[i]):
|
||||
row[i] = '' # Clear the cell if it contains a foreign phone number
|
||||
terms_removed += 1
|
||||
break
|
||||
|
||||
# Check for websites
|
||||
for pattern in website_patterns:
|
||||
if pattern.search(row[i]):
|
||||
row[i] = '' # Clear the cell if it contains a matching website
|
||||
websites_removed += 1
|
||||
break
|
||||
|
||||
modified_rows.append(row)
|
||||
|
||||
with open(file_path, 'w', newline='') as file:
|
||||
writer = csv.writer(file)
|
||||
writer.writerows(modified_rows)
|
||||
|
||||
return terms_removed
|
||||
return terms_removed, websites_removed
|
||||
|
||||
def strip_negative_patterns():
|
||||
"""Strip negative patterns from all CSV files in the stage 4 directory and tally the results."""
|
||||
|
|
@ -64,6 +86,7 @@ def strip_negative_patterns():
|
|||
print(figlet.renderText('Strip Negative Patterns'))
|
||||
|
||||
total_terms_removed = 0
|
||||
total_websites_removed = 0
|
||||
|
||||
for state_dir in os.listdir(STAGE_4_DIRECTORY):
|
||||
state_path = os.path.join(STAGE_4_DIRECTORY, state_dir)
|
||||
|
|
@ -71,18 +94,22 @@ def strip_negative_patterns():
|
|||
spinner = Halo(text=f'Processing {state_dir}', spinner='dots')
|
||||
spinner.start()
|
||||
state_terms_removed = 0
|
||||
state_websites_removed = 0
|
||||
for county_dir in os.listdir(state_path):
|
||||
county_path = os.path.join(state_path, county_dir)
|
||||
if os.path.isdir(county_path):
|
||||
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
||||
for file in csv_files:
|
||||
file_path = os.path.join(county_path, file)
|
||||
state_terms_removed += process_csv_file(file_path)
|
||||
terms, websites = process_csv_file(file_path)
|
||||
state_terms_removed += terms
|
||||
state_websites_removed += websites
|
||||
total_terms_removed += state_terms_removed
|
||||
spinner.succeed(f'Finished processing {state_dir}. Terms removed: {state_terms_removed}')
|
||||
total_websites_removed += state_websites_removed
|
||||
spinner.succeed(f'Finished processing {state_dir}. Terms removed: {state_terms_removed}, Websites removed: {state_websites_removed}')
|
||||
|
||||
print(f"Total Terms Removed: {total_terms_removed}")
|
||||
print(f"Total Websites Removed: {total_websites_removed}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
strip_negative_patterns()
|
||||
|
||||
|
|
|
|||
|
|
@ -20,19 +20,20 @@ PROJECT_ROOT = find_project_root(script_path)
|
|||
DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data")
|
||||
STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4")
|
||||
BLACKLIST_FILE = os.path.join(DATA_DIRECTORY, 'unknown-blacklist.txt')
|
||||
BAD_MATCHING_SERVICES_FILE = os.path.join(DATA_DIRECTORY, 'bad-matching-services.txt')
|
||||
|
||||
def read_blacklist(file_path):
|
||||
"""Read the blacklist terms from a file."""
|
||||
def read_terms(file_path):
|
||||
"""Read terms from a file."""
|
||||
if not os.path.isfile(file_path):
|
||||
print("Error: The unknown-blacklist.txt file is missing.")
|
||||
print(f"Error: The {os.path.basename(file_path)} file is missing.")
|
||||
return []
|
||||
|
||||
with open(file_path, 'r') as file:
|
||||
blacklist = [line.strip().lower() for line in file if line.strip()]
|
||||
return blacklist
|
||||
terms = [line.strip().lower() for line in file if line.strip()]
|
||||
return terms
|
||||
|
||||
def process_csv_file(file_path, blacklist):
|
||||
"""Process a CSV file to remove blacklist terms from unknown columns."""
|
||||
def process_csv_file(file_path, term_list):
|
||||
"""Process a CSV file to remove specified terms from unknown columns."""
|
||||
with open(file_path, 'r') as file:
|
||||
reader = csv.reader(file)
|
||||
rows = list(reader)
|
||||
|
|
@ -47,8 +48,8 @@ def process_csv_file(file_path, blacklist):
|
|||
|
||||
for row in data_rows:
|
||||
for i, header in enumerate(headers):
|
||||
if header.startswith("unknown-") and row[i].strip().lower() in blacklist:
|
||||
row[i] = '' # Clear the cell if it contains a blacklist term
|
||||
if header.startswith("unknown-") and row[i].strip().lower() in term_list:
|
||||
row[i] = '' # Clear the cell if it contains a specified term
|
||||
terms_removed += 1
|
||||
modified_rows.append(row)
|
||||
|
||||
|
|
@ -63,8 +64,11 @@ def strip_negative_terms():
|
|||
figlet = Figlet(font='slant')
|
||||
print(figlet.renderText('Strip Neg Terms'))
|
||||
|
||||
blacklist = read_blacklist(BLACKLIST_FILE)
|
||||
if not blacklist:
|
||||
blacklist = read_terms(BLACKLIST_FILE)
|
||||
bad_matching_services = read_terms(BAD_MATCHING_SERVICES_FILE)
|
||||
|
||||
if not blacklist and not bad_matching_services:
|
||||
print("No terms to process. Exiting.")
|
||||
return
|
||||
|
||||
total_terms_removed = 0
|
||||
|
|
@ -81,7 +85,10 @@ def strip_negative_terms():
|
|||
csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')]
|
||||
for file in csv_files:
|
||||
file_path = os.path.join(county_path, file)
|
||||
state_terms_removed += process_csv_file(file_path, blacklist)
|
||||
if blacklist:
|
||||
state_terms_removed += process_csv_file(file_path, blacklist)
|
||||
if bad_matching_services:
|
||||
state_terms_removed += process_csv_file(file_path, bad_matching_services)
|
||||
total_terms_removed += state_terms_removed
|
||||
spinner.succeed(f'Finished processing {state_dir}. Terms removed: {state_terms_removed}')
|
||||
|
||||
|
|
@ -89,4 +96,3 @@ def strip_negative_terms():
|
|||
|
||||
if __name__ == "__main__":
|
||||
strip_negative_terms()
|
||||
|
||||
|
|
|
|||
15
bin/unify.sh
Executable file
15
bin/unify.sh
Executable file
|
|
@ -0,0 +1,15 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# This script is to help make combining lists of things quick and easy, and
|
||||
# quite dirty.
|
||||
|
||||
_target="$1"
|
||||
|
||||
# Lowercase all text
|
||||
sed -i 's/.*/\L&/' "${_target}"
|
||||
|
||||
# Remove trailing spaces.
|
||||
sed -i 's/[[:space:]]*$//' "${_target}"
|
||||
|
||||
|
||||
sort "${_target}" | uniq > "${_target}.tmp" && mv "${_target}.tmp" "${_target}"
|
||||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -1,13 +1,34 @@
|
|||
a
|
||||
appliance
|
||||
a pressure washer for
|
||||
book online
|
||||
business
|
||||
cleaner
|
||||
companies
|
||||
company
|
||||
concrete
|
||||
crawl space waterproofing in williamsburg, va
|
||||
driveway
|
||||
driveway seal
|
||||
for
|
||||
for a
|
||||
for washer
|
||||
in
|
||||
machine
|
||||
my
|
||||
near
|
||||
of
|
||||
of a
|
||||
of cleaning
|
||||
of power
|
||||
power
|
||||
pressure
|
||||
repair
|
||||
seal
|
||||
sealing
|
||||
to
|
||||
repair
|
||||
Crawl Space Waterproofing In Williamsburg, VA
|
||||
companies
|
||||
company
|
||||
to schedule
|
||||
in
|
||||
driveway seal
|
||||
washer
|
||||
washing
|
||||
with
|
||||
St Augustine
|
||||
|
|
|
|||
Loading…
Reference in a new issue