Added extract-alive-domains-from-alive-list.

This commit is contained in:
Lord_Devi 2024-07-19 00:32:56 -04:00
parent 4d73ef30c7
commit 0d3cd1c855

View file

@ -65,9 +65,12 @@ class CSVManager:
self.filename = filename
self.file = open(filename, mode, newline='', encoding='utf-8')
self.writer = csv.writer(self.file)
if mode == 'w':
self.writer.writerow(["domain", "alive", "website"])
self.lock = asyncio.Lock()
# Write header if file is empty
if self.file.tell() == 0:
self.writer.writerow(["domain", "alive", "website"])
self.file.flush()
async def write_row(self, row):
async with self.lock:
@ -129,8 +132,8 @@ async def process_chunk(domains, csv_manager, chunk_number):
async def process_domains(domains, processed_domains_set):
global processed_domains
csv_manager = CSVManager(OUTPUT_FILE, mode='a')
csv_manager = CSVManager(OUTPUT_FILE, mode='w') # Changed to 'w' mode
domains_to_process = [domain for domain in domains if domain not in processed_domains_set]
total_chunks = len(domains_to_process) // CHUNK_SIZE + (1 if len(domains_to_process) % CHUNK_SIZE > 0 else 0)
@ -163,7 +166,7 @@ def read_processed_domains():
async def main():
global total_domains, processed_domains
logging.info("Starting alive-check.py")
logging.info(f"Reading domains from {INPUT_FILE}")
with open(INPUT_FILE, 'r', encoding='utf-8') as infile:
reader = csv.reader(infile)