mgk/domain-scraping-utils
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fixed process-alives.

Browse source
This commit is contained in:
Lord_Devi 2024-07-15 23:12:34 -04:00
parent da93d0b067
commit 2b4f51ba5d
2 changed files with 10 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
*.csv
*.backup
scrapes

14
bin/process-alives.py
View file

@ -17,7 +17,7 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
# Get the project root dynamically
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_ROOT = os.path.dirname(os.path.dirname(SCRIPT_DIR))
PROJECT_ROOT = os.path.dirname(SCRIPT_DIR) # Changed this line
SCRAPES_DIR = os.path.join(PROJECT_ROOT, "scrapes")
MAX_WORKERS = 1000
@ -25,6 +25,9 @@ REQUIRED_FREE_MEMORY_GB = 15
MAX_SCRAPE_TIME = 24 * 60 * 60 # 24 hours in seconds
SCRAPE_STATUS_FILE = os.path.join(SCRAPES_DIR, "scrape_status.json")
# Define the path to scrape-pages.py
SCRAPE_PAGES_SCRIPT = os.path.join(PROJECT_ROOT, "bin", "scrape-pages.py")
def parse_arguments():
parser = argparse.ArgumentParser(description="Process websites from a CSV file.")
parser.add_argument("-t", "--targets", required=True, help="Path to the CSV file containing target websites")
@ -39,8 +42,7 @@ def create_directories(websites):
def run_scrape_pages(website):
domain = urlparse(website).netloc
output_dir = os.path.join(SCRAPES_DIR, domain)
scrape_pages_script = os.path.join(PROJECT_ROOT, "bin", "scrape-pages.py")
command = [scrape_pages_script, "-o", output_dir, website]
command = [SCRAPE_PAGES_SCRIPT, "-o", output_dir, website]
try:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -93,7 +95,7 @@ def process_websites(input_file):
websites_to_scrape = [w for w in websites if scrape_status.get(w) != "completed"]
logging.info(f"{len(websites_to_scrape)} websites need scraping")
completed = 0
spinner = Halo(text='Processing...', spinner='dots')
spinner.start()
@ -127,7 +129,9 @@ def process_websites(input_file):
def main():
args = parse_arguments()
logging.info("Starting process-alives.py")
logging.info(f"Starting process-alives.py from {SCRIPT_DIR}")
logging.info(f"Project root: {PROJECT_ROOT}")
logging.info(f"scrape-pages.py path: {SCRAPE_PAGES_SCRIPT}")
process_websites(args.targets)
logging.info("process-alives.py completed")