mirror of
https://github.com/Skallwar/suckit.git
synced 2026-09-10 07:16:19 -04:00
137 lines
3.4 KiB
Python
Executable file
137 lines
3.4 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
from termcolor import colored
|
|
from tabulate import tabulate
|
|
from threading import Thread
|
|
|
|
import time
|
|
import argparse
|
|
import sys
|
|
import errno
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import signal
|
|
|
|
# Output directory for the downloaded files. Cleaned afterwards
|
|
# Be extra careful when changing that value !
|
|
OUTPUT_DIR = "/tmp/suckit_bench"
|
|
|
|
# Set the running time for both benchmarks
|
|
RUN_TIME = 120
|
|
|
|
# Keep track of the current PID to SIGINT it
|
|
CUR_PID = 0
|
|
|
|
# Path to the suckit binary
|
|
SUCKIT_CMD = "suckit"
|
|
|
|
# URL to start scraping from
|
|
URL = "http://books.toscrape.com"
|
|
|
|
def print_info():
|
|
info = """
|
|
This benchmark aims to bench suckit against other, popular website
|
|
downloaders such as httrack
|
|
"""
|
|
|
|
time_str = """
|
|
Each program will run for {} seconds
|
|
""".format(RUN_TIME)
|
|
|
|
print(f"{colored(info, 'blue')}")
|
|
print(time_str)
|
|
|
|
def bench_worker(dir_name, cmd):
|
|
global CUR_PID
|
|
|
|
# Handle the case where the directory exists already
|
|
try:
|
|
os.mkdir(dir_name)
|
|
except OSError as exc:
|
|
if exc.errno != errno.EEXIST:
|
|
raise
|
|
pass
|
|
|
|
os.chdir(dir_name)
|
|
|
|
CUR_PID = subprocess.Popen([cmd, URL],
|
|
stdout = open("/dev/null", "w"), shell = False).pid
|
|
|
|
def bench(dir_name, cmd):
|
|
thread = Thread(target = bench_worker, args = (dir_name, cmd, ))
|
|
thread.start()
|
|
|
|
# Let the benched program run for a certain amount of time
|
|
time.sleep(RUN_TIME)
|
|
|
|
thread.join()
|
|
os.kill(CUR_PID, signal.SIGINT)
|
|
|
|
# Count the number of files it downloaded
|
|
count = sum([len(files) for r, d, files in os.walk(".")])
|
|
|
|
# Go back to /tmp
|
|
os.chdir(OUTPUT_DIR)
|
|
|
|
return count
|
|
|
|
def flush_output(res):
|
|
print(tabulate(res, headers = ["name", "pages downloaded"]))
|
|
|
|
def main():
|
|
global OUTPUT_DIR
|
|
global RUN_TIME
|
|
global SUCKIT_CMD
|
|
global URL
|
|
|
|
parser = argparse.ArgumentParser(description = "SuckIT benchmark")
|
|
|
|
parser.add_argument("-o", "--output", action = "store", type = str, help = f"benchmark output directory (default_value = '{OUTPUT_DIR}')")
|
|
parser.add_argument("-t", "--time", action = "store", type = int, help = f"time given to each binary in seconds (default_value = RUN_TIME)")
|
|
parser.add_argument("-s", "--suckit", action = "store", type = str, help = f"path to the suckit binary (default_value = '{SUCKIT_CMD}')")
|
|
parser.add_argument("-u", "--url", action = "store", type = str, help = f"url to start scraping from (default_value = {URL})")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.output:
|
|
OUTPUT_DIR = args.output
|
|
|
|
if args.time:
|
|
RUN_TIME = args.time
|
|
|
|
if args.suckit:
|
|
SUCKIT_CMD = os.path.abspath(args.suckit)
|
|
|
|
if args.url:
|
|
URL = os.path.abspath(args.url)
|
|
|
|
print_info()
|
|
|
|
# Handle the case where the directory exists already
|
|
try:
|
|
os.mkdir(OUTPUT_DIR)
|
|
except OSError as exc:
|
|
err ="""
|
|
You're trying to use an already existing directory as your
|
|
output directory. Since the directory will be counted and
|
|
removed after the benchmark, I can't let you do that !
|
|
"""
|
|
print(f"{colored(err, 'red')}")
|
|
raise
|
|
|
|
os.chdir(OUTPUT_DIR)
|
|
|
|
results = []
|
|
|
|
results.append(["suckit", bench("suckit", SUCKIT_CMD)])
|
|
results.append(["httrack", bench("httrack", "httrack")])
|
|
|
|
flush_output(results)
|
|
|
|
# Clean benchmark output
|
|
shutil.rmtree(OUTPUT_DIR)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|