52 lines
1.8 KiB
Python
Executable file
52 lines
1.8 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# Script Name: process-stage-1
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
from pyfiglet import Figlet
|
|
from halo import Halo
|
|
|
|
# Define the paths based on the project details
|
|
PROJECT_ROOT = "/home/ld/mgk-scrapes"
|
|
CURRENT_DATA_DIR = os.path.join(PROJECT_ROOT, 'current-data')
|
|
STAGE_1_DIR = os.path.join(CURRENT_DATA_DIR, '.data', 'stage-1')
|
|
BIN_STAGE_1_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-1')
|
|
|
|
def run_scripts():
|
|
"""Run the specified scripts in order."""
|
|
scripts = [
|
|
os.path.join(BIN_STAGE_1_DIR, "prepare-stage-1.py"),
|
|
os.path.join(PROJECT_ROOT, "bin", "delete-empty-counties.py"),
|
|
os.path.join(BIN_STAGE_1_DIR, "remove-obviously-bad-columns.py"),
|
|
os.path.join(PROJECT_ROOT, "bin", "remove-empty-columns.py"),
|
|
os.path.join(BIN_STAGE_1_DIR, "delete-malformed-csvs.py"),
|
|
os.path.join(PROJECT_ROOT, "bin", "delete-empty-counties.py"),
|
|
os.path.join(BIN_STAGE_1_DIR, "md5-the-scrapes.py") # Add the new script here
|
|
]
|
|
|
|
subprocess.run(["python", scripts[0]], check=True)
|
|
subprocess.run(["python", scripts[2]], check=True)
|
|
subprocess.run(["python", scripts[4]], check=True)
|
|
|
|
subprocess.run(["python", scripts[1], "stage-1"], check=True)
|
|
subprocess.run(["python", scripts[3], "stage-1"], check=True)
|
|
subprocess.run(["python", scripts[5], "stage-1"], check=True)
|
|
subprocess.run(["python", scripts[6]], check=True) # Run the new script here
|
|
|
|
def main():
|
|
figlet = Figlet(font='slant')
|
|
script_name = "process-stage-1".replace("-", " ").title()
|
|
print(figlet.renderText(script_name))
|
|
|
|
print("Running scripts for Stage 1...\n")
|
|
run_scripts()
|
|
|
|
final_spinner = Halo(spinner='dots', color='green')
|
|
final_spinner.start()
|
|
final_spinner.succeed("All scripts completed.")
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
|