71 lines
2.2 KiB
Python
Executable file
71 lines
2.2 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# Script Name: prepare-stage-4
|
|
|
|
import os
|
|
import shutil
|
|
import halo
|
|
from tqdm import tqdm
|
|
from pyfiglet import Figlet
|
|
|
|
# Define the paths based on the project details
|
|
PROJECT_ROOT = "/home/ld/mgk-scrapes"
|
|
CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data")
|
|
DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data")
|
|
STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3")
|
|
STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4")
|
|
|
|
def create_stage_4_directory():
|
|
"""Create the stage-4 directory, replacing it if it already exists."""
|
|
if os.path.exists(STAGE_4_DIRECTORY):
|
|
print("Existing Stage 4 found, deleting and replacing...")
|
|
shutil.rmtree(STAGE_4_DIRECTORY)
|
|
os.makedirs(STAGE_4_DIRECTORY)
|
|
print("Stage 4 directory created.")
|
|
|
|
def copy_data_to_stage_4():
|
|
"""Copy data from stage-3 to stage-4."""
|
|
for item in os.listdir(STAGE_3_DIRECTORY):
|
|
s = os.path.join(STAGE_3_DIRECTORY, item)
|
|
d = os.path.join(STAGE_4_DIRECTORY, item)
|
|
if os.path.isdir(s):
|
|
shutil.copytree(s, d)
|
|
else:
|
|
shutil.copy2(s, d)
|
|
|
|
def verify_stage_4_data():
|
|
"""Verify that the data in stage-4 matches the data in stage-3."""
|
|
for root, dirs, files in os.walk(STAGE_3_DIRECTORY):
|
|
for file in files:
|
|
src_file = os.path.join(root, file)
|
|
dest_file = src_file.replace(STAGE_3_DIRECTORY, STAGE_4_DIRECTORY)
|
|
if not os.path.exists(dest_file):
|
|
return False
|
|
return True
|
|
|
|
def main():
|
|
figlet = Figlet(font='slant')
|
|
script_name = "prepare-stage-4".replace("-", " ").title()
|
|
print(figlet.renderText(script_name))
|
|
|
|
print("Preparing Stage 4...")
|
|
|
|
create_stage_4_directory()
|
|
|
|
print("Copying data to Stage 4...")
|
|
spinner = halo.Halo(text='Copying data', spinner='dots')
|
|
spinner.start()
|
|
copy_data_to_stage_4()
|
|
spinner.succeed("Data copied.")
|
|
|
|
print("Verifying Stage 4 data...")
|
|
is_valid = verify_stage_4_data()
|
|
|
|
if is_valid:
|
|
print("Data verification successful. Stage 4 data is valid.")
|
|
else:
|
|
print("Data verification failed. Stage 4 data is not valid.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|