instant-data-scraper-utils/bin/process-stage-3.py

65 lines
2.5 KiB
Python
Executable file

#!/usr/bin/env python
# Script Name: process-stage-3
import os
import subprocess
from pyfiglet import Figlet
from halo import Halo
def find_project_root(current_path):
"""Find the project root by locating the 'bin' directory."""
while current_path != os.path.dirname(current_path):
if os.path.basename(current_path) == 'bin':
return os.path.dirname(current_path)
current_path = os.path.dirname(current_path)
raise FileNotFoundError("Could not find 'bin' directory in the path hierarchy.")
# Define the paths based on the project details
script_path = os.path.abspath(__file__)
PROJECT_ROOT = find_project_root(script_path)
BIN_STAGE_3_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-3')
BIN_DIR = os.path.join(PROJECT_ROOT, 'bin')
def run_scripts():
"""Run the specified scripts in order."""
scripts = [
os.path.join(BIN_STAGE_3_DIR, "prepare-stage-3.py"),
os.path.join(BIN_STAGE_3_DIR, "column-search_business-name.py"), # Moved up
os.path.join(BIN_STAGE_3_DIR, "column-search_review-count.py"),
os.path.join(BIN_STAGE_3_DIR, "column-search_review-rating.py"),
os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-location.py"),
os.path.join(BIN_STAGE_3_DIR, "column-search_yib.py"),
os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-business-category.py"),
os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-matching-services.py"),
(os.path.join(BIN_DIR, "remove-empty-columns.py"), "stage-3"),
os.path.join(BIN_STAGE_3_DIR, "column-search_unknowns.py"),
os.path.join(BIN_DIR, "report-unknowns.py"), # Moved up
os.path.join(BIN_STAGE_3_DIR, "discover-categories-and-services.py") # Added here
]
# Run scripts that do not require arguments
for script in scripts[:-4]:
subprocess.run(["python", script], check=True)
# Run scripts that require the 'stage-3' argument
subprocess.run(["python", scripts[-4][0], scripts[-4][1]], check=True)
subprocess.run(["python", scripts[-3], "stage-3"], check=True)
subprocess.run(["python", scripts[-2], "stage-3"], check=True)
subprocess.run(["python", scripts[-1]], check=True) # Added script
def main():
figlet = Figlet(font='slant')
script_name = "process-stage-3".replace("-", " ").title()
print(figlet.renderText(script_name))
print("Running scripts for Stage 3...\n")
run_scripts()
final_spinner = Halo(spinner='dots', color='green')
final_spinner.start()
final_spinner.succeed("All scripts completed.")
if __name__ == '__main__':
main()