From 1056a0994512000da75d361632f6ca15a9fbc772 Mon Sep 17 00:00:00 2001 From: Lord_Devi Date: Tue, 16 Jul 2024 18:08:36 -0400 Subject: [PATCH] Initial commit. --- .gitignore | 2 + README.org | 9 + bin/delete-empty-counties.md | 317 ++++ bin/delete-empty-counties.py | 107 ++ bin/fetch-census-columns.py | 27 + bin/fetch-census-examples.py | 32 + bin/find-bad-review-data.py | 65 + bin/process-all.md | 56 + bin/process-all.py | 45 + bin/process-stage-1.md | 328 ++++ bin/process-stage-1.py | 51 + bin/process-stage-2.md | 360 ++++ bin/process-stage-2.py | 46 + bin/process-stage-3.md | 478 +++++ bin/process-stage-3.py | 53 + bin/process-stage-4.md | 175 ++ bin/process-stage-4.py | 48 + bin/process-stage-5.md | 116 ++ bin/process-stage-5.py | 41 + bin/process-stage-6.md | 1026 +++++++++++ bin/process-stage-6.py | 63 + bin/remove-empty-columns.md | 477 +++++ bin/remove-empty-columns.py | 103 ++ bin/report-unknowns.md | 224 +++ bin/report-unknowns.py | 74 + bin/reshift-terms.py | 110 ++ bin/scrape-census.py | 40 + bin/scrape-simplemaps.py | 51 + bin/stage-1/delete-malformed-csvs.md | 427 +++++ bin/stage-1/delete-malformed-csvs.py | 76 + bin/stage-1/md5-the-scrapes.py | 116 ++ bin/stage-1/prepare-stage-1.md | 325 ++++ bin/stage-1/prepare-stage-1.py | 87 + bin/stage-1/remove-obviously-bad-columns.md | 573 ++++++ bin/stage-1/remove-obviously-bad-columns.py | 117 ++ bin/stage-2/prepare-stage-2.md | 160 ++ bin/stage-2/prepare-stage-2.py | 70 + bin/stage-2/remove-utm.md | 418 +++++ bin/stage-2/remove-utm.py | 78 + bin/stage-2/sanitize-gbp-business-website.md | 196 ++ bin/stage-2/sanitize-gbp-business-website.py | 84 + bin/stage-2/sanitize-phone-data.md | 207 +++ bin/stage-2/sanitize-phone-data.py | 95 + bin/stage-2/sanitize-review-count.md | 197 ++ bin/stage-2/sanitize-review-count.py | 84 + bin/stage-3/column-search_business-name.md | 178 ++ bin/stage-3/column-search_business-name.py | 75 + .../column-search_gbp-business-category.md | 215 +++ .../column-search_gbp-business-category.py | 108 ++ bin/stage-3/column-search_gbp-location.md | 195 ++ bin/stage-3/column-search_gbp-location.py | 90 + .../column-search_gbp-matching-services.md | 221 +++ .../column-search_gbp-matching-services.py | 120 ++ bin/stage-3/column-search_review-count.md | 197 ++ bin/stage-3/column-search_review-count.py | 92 + bin/stage-3/column-search_review-rating.md | 196 ++ bin/stage-3/column-search_review-rating.py | 91 + bin/stage-3/column-search_unknowns.md | 182 ++ bin/stage-3/column-search_unknowns.py | 83 + bin/stage-3/column-search_yib.md | 251 +++ bin/stage-3/column-search_yib.py | 91 + bin/stage-3/prepare-stage-3.md | 116 ++ bin/stage-3/prepare-stage-3.py | 70 + bin/stage-4/delete-sponsored-listings.md | 181 ++ bin/stage-4/delete-sponsored-listings.py | 67 + bin/stage-4/prepare-stage-4.md | 116 ++ bin/stage-4/prepare-stage-4.py | 70 + bin/stage-4/reshift-terms.md | 245 +++ bin/stage-4/reshift-terms.py | 138 ++ .../strip-neg-patterns-from-unknown-cols.md | 185 ++ .../strip-neg-patterns-from-unknown-cols.py | 73 + .../strip-neg-terms-from-unknown-cols.md | 318 ++++ .../strip-neg-terms-from-unknown-cols.py | 83 + bin/stage-5/prepare-stage-5.md | 115 ++ bin/stage-5/prepare-stage-5.py | 70 + bin/stage-5/retrieve-root-domain.md | 276 +++ bin/stage-5/retrieve-root-domain.py | 111 ++ bin/stage-5/standardize-location-data.md | 221 +++ bin/stage-5/standardize-location-data.py | 102 + .../strip-brackets-from-review-count.md | 290 +++ .../strip-brackets-from-review-count.py | 116 ++ bin/stage-6/business-name-strip-structures.md | 107 ++ bin/stage-6/business-name-strip-structures.py | 73 + bin/stage-6/business-names-with-locations.md | 99 + bin/stage-6/business-names-with-locations.py | 70 + bin/stage-6/delete-bad-matching-services.md | 169 ++ bin/stage-6/delete-bad-matching-services.py | 70 + bin/stage-6/fill-in-missing-data.md | 58 + bin/stage-6/fill-in-missing-data.py | 83 + .../find-unknown-categories-and-services.md | 126 ++ .../find-unknown-categories-and-services.py | 75 + bin/stage-6/initial-deduplication.md | 50 + bin/stage-6/initial-deduplication.py | 84 + bin/stage-6/old-initial-dedup.py | 170 ++ bin/stage-6/package-lock.json | 1640 +++++++++++++++++ bin/stage-6/package.json | 5 + bin/stage-6/pre-sort.md | 134 ++ bin/stage-6/pre-sort.py | 51 + bin/stage-6/prepare-stage-6.md | 197 ++ bin/stage-6/prepare-stage-6.py | 85 + bin/stage-6/process-stage-3.md | 166 ++ bin/stage-6/re-order-columns.md | 86 + bin/stage-6/re-order-columns.py | 80 + bin/stage-6/services-gbp-optimization.md | 49 + bin/stage-6/services-gbp-optimization.py | 75 + .../services-gbp-reputation-management.md | 161 ++ .../services-gbp-reputation-management.py | 172 ++ bin/stage-6/services-needs-website.md | 49 + bin/stage-6/services-needs-website.py | 73 + bin/stage-6/services-website-audit.md | 127 ++ bin/stage-6/services-website-audit.py | 213 +++ bin/stage-6/standardize-capital-letters.md | 166 ++ bin/stage-6/standardize-capital-letters.py | 61 + bin/stage-6/standardize-yib.mb | 61 + bin/stage-6/standardize-yib.py | 89 + bin/stage-6/tally-initial-deduplication.py | 41 + bin/stage-6/trim-invalid-rows-from-merger.md | 172 ++ bin/stage-6/trim-invalid-rows-from-merger.py | 66 + bin/stage-6/uniqify-pass-1.md | 567 ++++++ bin/stage-6/uniqify-pass-1.py | 147 ++ bin/stage-6/uniqify-pass-2.md | 16 + bin/stage-6/uniqify-pass-2.py | 131 ++ .../uuid-business-1-by-gbp-business-phone.md | 216 +++ .../uuid-business-1-by-gbp-business-phone.py | 109 ++ bin/stage-6/uuid-business-2-by-root-domain.md | 437 +++++ bin/stage-6/uuid-business-2-by-root-domain.py | 109 ++ .../uuid-business-3-by-business-name.md | 156 ++ .../uuid-business-3-by-business-name.py | 113 ++ ...uuid-business-4-single-location-entries.md | 48 + ...uuid-business-4-single-location-entries.py | 67 + 130 files changed, 20551 insertions(+) create mode 100644 .gitignore create mode 100644 README.org create mode 100644 bin/delete-empty-counties.md create mode 100755 bin/delete-empty-counties.py create mode 100755 bin/fetch-census-columns.py create mode 100755 bin/fetch-census-examples.py create mode 100755 bin/find-bad-review-data.py create mode 100644 bin/process-all.md create mode 100755 bin/process-all.py create mode 100644 bin/process-stage-1.md create mode 100755 bin/process-stage-1.py create mode 100644 bin/process-stage-2.md create mode 100755 bin/process-stage-2.py create mode 100644 bin/process-stage-3.md create mode 100755 bin/process-stage-3.py create mode 100644 bin/process-stage-4.md create mode 100755 bin/process-stage-4.py create mode 100644 bin/process-stage-5.md create mode 100755 bin/process-stage-5.py create mode 100644 bin/process-stage-6.md create mode 100755 bin/process-stage-6.py create mode 100644 bin/remove-empty-columns.md create mode 100755 bin/remove-empty-columns.py create mode 100644 bin/report-unknowns.md create mode 100755 bin/report-unknowns.py create mode 100755 bin/reshift-terms.py create mode 100755 bin/scrape-census.py create mode 100755 bin/scrape-simplemaps.py create mode 100644 bin/stage-1/delete-malformed-csvs.md create mode 100755 bin/stage-1/delete-malformed-csvs.py create mode 100755 bin/stage-1/md5-the-scrapes.py create mode 100644 bin/stage-1/prepare-stage-1.md create mode 100755 bin/stage-1/prepare-stage-1.py create mode 100644 bin/stage-1/remove-obviously-bad-columns.md create mode 100755 bin/stage-1/remove-obviously-bad-columns.py create mode 100644 bin/stage-2/prepare-stage-2.md create mode 100755 bin/stage-2/prepare-stage-2.py create mode 100644 bin/stage-2/remove-utm.md create mode 100755 bin/stage-2/remove-utm.py create mode 100644 bin/stage-2/sanitize-gbp-business-website.md create mode 100755 bin/stage-2/sanitize-gbp-business-website.py create mode 100644 bin/stage-2/sanitize-phone-data.md create mode 100755 bin/stage-2/sanitize-phone-data.py create mode 100644 bin/stage-2/sanitize-review-count.md create mode 100755 bin/stage-2/sanitize-review-count.py create mode 100644 bin/stage-3/column-search_business-name.md create mode 100755 bin/stage-3/column-search_business-name.py create mode 100644 bin/stage-3/column-search_gbp-business-category.md create mode 100755 bin/stage-3/column-search_gbp-business-category.py create mode 100644 bin/stage-3/column-search_gbp-location.md create mode 100755 bin/stage-3/column-search_gbp-location.py create mode 100644 bin/stage-3/column-search_gbp-matching-services.md create mode 100755 bin/stage-3/column-search_gbp-matching-services.py create mode 100644 bin/stage-3/column-search_review-count.md create mode 100755 bin/stage-3/column-search_review-count.py create mode 100644 bin/stage-3/column-search_review-rating.md create mode 100755 bin/stage-3/column-search_review-rating.py create mode 100644 bin/stage-3/column-search_unknowns.md create mode 100755 bin/stage-3/column-search_unknowns.py create mode 100644 bin/stage-3/column-search_yib.md create mode 100755 bin/stage-3/column-search_yib.py create mode 100644 bin/stage-3/prepare-stage-3.md create mode 100755 bin/stage-3/prepare-stage-3.py create mode 100644 bin/stage-4/delete-sponsored-listings.md create mode 100755 bin/stage-4/delete-sponsored-listings.py create mode 100644 bin/stage-4/prepare-stage-4.md create mode 100755 bin/stage-4/prepare-stage-4.py create mode 100644 bin/stage-4/reshift-terms.md create mode 100755 bin/stage-4/reshift-terms.py create mode 100644 bin/stage-4/strip-neg-patterns-from-unknown-cols.md create mode 100755 bin/stage-4/strip-neg-patterns-from-unknown-cols.py create mode 100644 bin/stage-4/strip-neg-terms-from-unknown-cols.md create mode 100755 bin/stage-4/strip-neg-terms-from-unknown-cols.py create mode 100644 bin/stage-5/prepare-stage-5.md create mode 100755 bin/stage-5/prepare-stage-5.py create mode 100644 bin/stage-5/retrieve-root-domain.md create mode 100755 bin/stage-5/retrieve-root-domain.py create mode 100644 bin/stage-5/standardize-location-data.md create mode 100755 bin/stage-5/standardize-location-data.py create mode 100644 bin/stage-5/strip-brackets-from-review-count.md create mode 100755 bin/stage-5/strip-brackets-from-review-count.py create mode 100644 bin/stage-6/business-name-strip-structures.md create mode 100755 bin/stage-6/business-name-strip-structures.py create mode 100644 bin/stage-6/business-names-with-locations.md create mode 100755 bin/stage-6/business-names-with-locations.py create mode 100644 bin/stage-6/delete-bad-matching-services.md create mode 100755 bin/stage-6/delete-bad-matching-services.py create mode 100644 bin/stage-6/fill-in-missing-data.md create mode 100755 bin/stage-6/fill-in-missing-data.py create mode 100644 bin/stage-6/find-unknown-categories-and-services.md create mode 100755 bin/stage-6/find-unknown-categories-and-services.py create mode 100644 bin/stage-6/initial-deduplication.md create mode 100755 bin/stage-6/initial-deduplication.py create mode 100755 bin/stage-6/old-initial-dedup.py create mode 100644 bin/stage-6/package-lock.json create mode 100644 bin/stage-6/package.json create mode 100644 bin/stage-6/pre-sort.md create mode 100755 bin/stage-6/pre-sort.py create mode 100644 bin/stage-6/prepare-stage-6.md create mode 100755 bin/stage-6/prepare-stage-6.py create mode 100644 bin/stage-6/process-stage-3.md create mode 100644 bin/stage-6/re-order-columns.md create mode 100755 bin/stage-6/re-order-columns.py create mode 100644 bin/stage-6/services-gbp-optimization.md create mode 100755 bin/stage-6/services-gbp-optimization.py create mode 100644 bin/stage-6/services-gbp-reputation-management.md create mode 100755 bin/stage-6/services-gbp-reputation-management.py create mode 100644 bin/stage-6/services-needs-website.md create mode 100755 bin/stage-6/services-needs-website.py create mode 100644 bin/stage-6/services-website-audit.md create mode 100755 bin/stage-6/services-website-audit.py create mode 100644 bin/stage-6/standardize-capital-letters.md create mode 100755 bin/stage-6/standardize-capital-letters.py create mode 100644 bin/stage-6/standardize-yib.mb create mode 100755 bin/stage-6/standardize-yib.py create mode 100755 bin/stage-6/tally-initial-deduplication.py create mode 100644 bin/stage-6/trim-invalid-rows-from-merger.md create mode 100755 bin/stage-6/trim-invalid-rows-from-merger.py create mode 100644 bin/stage-6/uniqify-pass-1.md create mode 100755 bin/stage-6/uniqify-pass-1.py create mode 100644 bin/stage-6/uniqify-pass-2.md create mode 100755 bin/stage-6/uniqify-pass-2.py create mode 100644 bin/stage-6/uuid-business-1-by-gbp-business-phone.md create mode 100755 bin/stage-6/uuid-business-1-by-gbp-business-phone.py create mode 100644 bin/stage-6/uuid-business-2-by-root-domain.md create mode 100755 bin/stage-6/uuid-business-2-by-root-domain.py create mode 100644 bin/stage-6/uuid-business-3-by-business-name.md create mode 100755 bin/stage-6/uuid-business-3-by-business-name.py create mode 100644 bin/stage-6/uuid-business-4-single-location-entries.md create mode 100755 bin/stage-6/uuid-business-4-single-location-entries.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..12cbfd8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +current-data +instant-data-scraper-data diff --git a/README.org b/README.org new file mode 100644 index 0000000..1229832 --- /dev/null +++ b/README.org @@ -0,0 +1,9 @@ +#+TITLE: Instant Data Scraper Utils +#+AUTHOR: Lord Devi + +* Instant Data Scraper Utils +These are the tools I use to sort through data that has been scraped using the chrome plugin "Instant Data Scraper". + +That data needs to be sorted, deduplicated, normalized, have everything in the correct rows, have invalid data removed, etc. + +These tools aim to make the data more useful for me. diff --git a/bin/delete-empty-counties.md b/bin/delete-empty-counties.md new file mode 100644 index 0000000..b3f0ab9 --- /dev/null +++ b/bin/delete-empty-counties.md @@ -0,0 +1,317 @@ +# Prompt 1 + +- Description: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- Project Root: "/home/ld/mgk-scrapes" +- Current Dataset: "/current-data/" +- Data Directory: "/.data/" +- Parent Data: "/"(All directories that do not begin with a period ".") +- Stage Directories: "/stage-#" (Where # is a number.) +- Stage 1: "/stage-1/" +- Stage 2: "/stage-2/" +- Stage 3: "/stage-3/" +- Stage 4: "/stage-4/" +- Stage 5: "/stage-5/" +- Stage 6: "/stage-6/" +- Binaries: "/bin/" +- Stage 1 Binaries: "/stage-1/" +- Stage 2 Binaries: "/stage-2/" +- Stage 3 Binaries: "/stage-3/" +- Stage 4 Binaries: "/stage-4/" +- Stage 5 Binaries: "/stage-5/" +- Stage 6 Binaries: "/stage-6/" +- Rule 1: All scripts need to be able to be ran from any directory. +- Rule 2: All scripts need output what they are doing, as they are doing it. +- Rule 3: All tasks should use halo to report successes and failures. +- Rule 4: The python module "tqdm" can be used to report progress when appropriate. +- Rule 5: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- Rule 6: Every script should begin with a hashbang "#!/usr/bin/env python". +- Rule 7: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- Rule 8: Scripts should generally have robust error checking. + + +I need help creating a new script. See above for metadata and information regarding our project setup, configuration, purpose, and general rules to follow. + +This script should be called "delete-empty-counties". + +This script should iterate through the sub directories in looking for empty sub-directories and then delete them. + + consists of and beneath the . + +Sometimes these are empty of CSV files. When that is the case, we want to delete those directories. + +The script should report what it is doing. + +It should report "Empty counties found:" and then on the next line, provide a comma separated list of all we found that have no CSV files which we are going to delete. + +It should then report that it is deleting the directories, and then delete them. + +At the end, report a final tally of all directories deleted. + +This script will reside in the directory, and needs to be able to be ran against different stages. + +So it will need a command line option provided to it, such as these examples: + + +delete-empty-counties.py stage-1 +delete-empty-counties.py stage-2 + + + +# Prompt 2 +I got this error: + + +Error: The directory stage-1 does not exist. + + +Please ensure that the have been referenced to setup the correct variables at the start of the script so that the script knows where to find the . + + +# Prompt 3 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "delete-empty-columns" script: + + + +This script should use pyfiglet to output the name of the script as it runs. + + +# Prompt 4 +Can the pyfiglet portion of this script be made to strip hyphens from the script name before using pyfiglet to display the name of the script? + + +# Prompt 5 +Ok I would like the hypens actually replaced with spaces, not stripped or removed. + + +# Prompt 6 +Additionally, after the hyphens have been replaced with spaces, I would like the words that are left to be capitalized before printing the pyfiglet. + + +# Prompt 7 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I wish to modify the 'delete-empty-counties' script: + + +#!/usr/bin/env python +# Script Name: delete-empty-counties + +import os +import sys +import shutil +import halo +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") + +def get_stage_directory(stage): + """Get the path for the specified stage directory.""" + return os.path.join(DATA_DIRECTORY, stage) + +def find_empty_counties(stage_directory): + """Find all empty county directories in the given stage directory.""" + empty_counties = [] + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + # Check if the county directory contains any CSV files + csv_files = [f for f in os.listdir(county_path) if f.endswith('.csv')] + if not csv_files: + empty_counties.append(county_path) + return empty_counties + +def delete_directories(directories): + """Delete the given directories.""" + for directory in directories: + shutil.rmtree(directory) + +def main(): + figlet = Figlet(font='slant') + script_name = "delete-empty-counties".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if len(sys.argv) != 2: + print("Usage: delete-empty-counties.py ") + sys.exit(1) + + stage = sys.argv[1] + stage_directory = get_stage_directory(stage) + + if not os.path.exists(stage_directory): + print(f"Error: The directory {stage_directory} does not exist.") + sys.exit(1) + + print(f"Scanning for empty counties in {stage_directory}...") + spinner = halo.Halo(text='Scanning', spinner='dots') + spinner.start() + empty_counties = find_empty_counties(stage_directory) + spinner.succeed("Scan complete.") + + if empty_counties: + print("Empty counties found:") + print(", ".join(empty_counties)) + + print("Deleting empty counties...") + spinner.start() + delete_directories(empty_counties) + spinner.succeed("Deletion complete.") + + print(f"Total directories deleted: {len(empty_counties)}") + else: + print("No empty counties found.") + +if __name__ == "__main__": + main() + + +Change 1: Right now it it deletes empty county directories, but I would like it to also delete empty state directories in the stage folder being processed. diff --git a/bin/delete-empty-counties.py b/bin/delete-empty-counties.py new file mode 100755 index 0000000..cc08f68 --- /dev/null +++ b/bin/delete-empty-counties.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# Script Name: delete-empty-counties + +import os +import sys +import shutil +import halo +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") + +def get_stage_directory(stage): + """Get the path for the specified stage directory.""" + return os.path.join(DATA_DIRECTORY, stage) + +def find_empty_counties(stage_directory): + """Find all empty county directories in the given stage directory.""" + empty_counties = [] + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + # Check if the county directory contains any CSV files + csv_files = [f for f in os.listdir(county_path) if f.endswith('.csv')] + if not csv_files: + empty_counties.append(county_path) + return empty_counties + +def find_empty_states(stage_directory): + """Find all empty state directories in the given stage directory.""" + empty_states = [] + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + county_dirs = [d for d in os.listdir(state_path) if os.path.isdir(os.path.join(state_path, d))] + if not county_dirs: + empty_states.append(state_path) + else: + # Check if all counties are empty + if all(os.path.isdir(os.path.join(state_path, county_dir)) and + not any(f.endswith('.csv') for f in os.listdir(os.path.join(state_path, county_dir))) + for county_dir in county_dirs): + empty_states.append(state_path) + return empty_states + +def delete_directories(directories): + """Delete the given directories.""" + for directory in directories: + shutil.rmtree(directory) + +def main(): + figlet = Figlet(font='slant') + script_name = "delete-empty-counties".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if len(sys.argv) != 2: + print("Usage: delete-empty-counties.py ") + sys.exit(1) + + stage = sys.argv[1] + stage_directory = get_stage_directory(stage) + + if not os.path.exists(stage_directory): + print(f"Error: The directory {stage_directory} does not exist.") + sys.exit(1) + + print(f"Scanning for empty counties and states in {stage_directory}...") + spinner = halo.Halo(text='Scanning', spinner='dots') + spinner.start() + empty_counties = find_empty_counties(stage_directory) + empty_states = find_empty_states(stage_directory) + spinner.succeed("Scan complete.") + + if empty_counties: + print("Empty counties found:") + print(", ".join(empty_counties)) + + print("Deleting empty counties...") + spinner.start() + delete_directories(empty_counties) + spinner.succeed("County deletion complete.") + + print(f"Total county directories deleted: {len(empty_counties)}") + else: + print("No empty counties found.") + + if empty_states: + print("Empty states found:") + print(", ".join(empty_states)) + + print("Deleting empty states...") + spinner.start() + delete_directories(empty_states) + spinner.succeed("State deletion complete.") + + print(f"Total state directories deleted: {len(empty_states)}") + else: + print("No empty states found.") + +if __name__ == "__main__": + main() + diff --git a/bin/fetch-census-columns.py b/bin/fetch-census-columns.py new file mode 100755 index 0000000..c634c9f --- /dev/null +++ b/bin/fetch-census-columns.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +import requests +import csv +import io + +# Send a GET request to the US Census Bureau dataset URL +url = "https://www2.census.gov/programs-surveys/popest/datasets/2010-2019/cities/totals/sub-est2019_all.csv" +response = requests.get(url) + +# Check if the request was successful +if response.status_code == 200: + # Read the CSV content from the response + csv_content = io.StringIO(response.text) + + # Create a CSV reader object + reader = csv.reader(csv_content) + + # Get the column names from the first row + column_names = next(reader) + + # Print the column names + print("Column names:") + for column_name in column_names: + print(column_name) +else: + print("Failed to fetch the dataset.") diff --git a/bin/fetch-census-examples.py b/bin/fetch-census-examples.py new file mode 100755 index 0000000..ce9367b --- /dev/null +++ b/bin/fetch-census-examples.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import requests +import csv +import io + +# Send a GET request to the US Census Bureau dataset URL +url = "https://www2.census.gov/programs-surveys/popest/datasets/2010-2019/cities/totals/sub-est2019_all.csv" +response = requests.get(url) + +# Check if the request was successful +if response.status_code == 200: + # Read the CSV content from the response + csv_content = io.StringIO(response.text) + + # Create a CSV reader object + reader = csv.DictReader(csv_content) + + # Fetch a small set of example data (e.g., first 5 rows) + example_data = [] + for _ in range(5): + row = next(reader) + example_data.append(row) + + # Print the example data for each column + for column_name in reader.fieldnames: + print(f"Column: {column_name}") + for row in example_data: + print(f" {row[column_name]}") + print() +else: + print("Failed to fetch the dataset.") diff --git a/bin/find-bad-review-data.py b/bin/find-bad-review-data.py new file mode 100755 index 0000000..98dbadc --- /dev/null +++ b/bin/find-bad-review-data.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import os +import csv +import re + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +# Regular expression to match the pattern "-dd" +pattern = re.compile(r'^-\d{2}$') + +def check_csv_for_pattern(file_path): + """Check if the CSV file contains a column with mostly strings that match the pattern.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return False + + column_counts = [0] * len(rows[0]) + + for row in rows: + for idx, cell in enumerate(row): + if pattern.match(cell): + column_counts[idx] += 1 + + # Consider a column with mostly matching patterns if more than 50% of its cells match + threshold = len(rows) / 2 + return any(count > threshold for count in column_counts) + +def find_csv_files_with_pattern(stage_directory): + """Find all CSV files in the stage directory that contain columns with the specified pattern.""" + matching_files = [] + + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + if check_csv_for_pattern(file_path): + matching_files.append(file_path) + return matching_files + +def main(): + matching_files = find_csv_files_with_pattern(STAGE_2_DIRECTORY) + + if matching_files: + print("CSV files containing columns with '-dd' patterns:") + for file in matching_files: + print(file) + else: + print("No CSV files with the '-dd' pattern found.") + +if __name__ == "__main__": + main() + diff --git a/bin/process-all.md b/bin/process-all.md new file mode 100644 index 0000000..bbc7f66 --- /dev/null +++ b/bin/process-all.md @@ -0,0 +1,56 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I would like a new script created, which would reside in [Binaries]. This script should be called 'process-all'. + +And it should basically just call 6 other scripts in order. + +Here are the scripts it should call: + + +process-stage-1.py +process-stage-2.py +process-stage-3.py +process-stage-4.py +process-stage-5.py +process-stage-6.py + + + diff --git a/bin/process-all.py b/bin/process-all.py new file mode 100755 index 0000000..ac7f253 --- /dev/null +++ b/bin/process-all.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# Script Name: process-all + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_DIR, "process-stage-1.py"), + os.path.join(BIN_DIR, "process-stage-2.py"), + os.path.join(BIN_DIR, "process-stage-3.py"), + os.path.join(BIN_DIR, "process-stage-4.py"), + os.path.join(BIN_DIR, "process-stage-5.py"), + os.path.join(BIN_DIR, "process-stage-6.py") + ] + + for script in scripts: + try: + subprocess.run(["python", script], check=True) + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + break + +def main(): + figlet = Figlet(font='slant') + script_name = "process-all".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running all stage processing scripts...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/process-stage-1.md b/bin/process-stage-1.md new file mode 100644 index 0000000..e69e0f2 --- /dev/null +++ b/bin/process-stage-1.md @@ -0,0 +1,328 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "process-stage-1" script: + + + +I need this script to run the following scripts in order: + + +[Project Root]/bin/stage-1/prepare-stage-1.py +[Project Root]/bin/delete-empty-counties.py stage-1 +[Project Root]/bin/column-count-correction.py stage-1 +[Project Root]/bin/stage-1/remove-obviously-bad-columns.py +[Project Root]/bin/remove-empty-columns.py stage-1 +[Project Root]/bin/stage-1/delete-malformed-csvs.py +[Project Root]/bin/column-count-correction.py stage-1 + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "process-stage-1" script: + + + +I need this script to run the following scripts in the order in which they are listed below: + + +[Project Root]/bin/stage-1/prepare-stage-1.py +[Project Root]/bin/delete-empty-counties.py stage-1 +[Project Root]/bin/column-count-correction.py stage-1 +[Project Root]/bin/stage-1/remove-obviously-bad-columns.py +[Project Root]/bin/remove-empty-columns.py stage-1 +[Project Root]/bin/stage-1/delete-malformed-csvs.py +[Project Root]/bin/delete-empty-counties.py stage-1 +[Project Root]/bin/column-count-correction.py stage-1 + + + +# Prompt 3 +It looks to me like the script is trying to verify the existence of Stage 2, and is deleting it, and then creating it. + +I need this to be done, but not in the process-stage-2 script. + +I need this task to be performed in the 'prepare-stage-2' script. + +Please modify the process-stage-2 script to offload the stage 2 creation task to the prepare-stage-2 script. + +After I get the process-stage-2 I will afterwards ask for the prepare-stage-2. + + +# Prompt 4 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I wish to adjust the 'process-stage-1' script: + + + +Change 1: I wish to remove the 'column-count-correction.py' script from being used in process-stage-1. Completely remove references to that script, we do not need it anymore. diff --git a/bin/process-stage-1.py b/bin/process-stage-1.py new file mode 100755 index 0000000..271cc96 --- /dev/null +++ b/bin/process-stage-1.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# Script Name: process-stage-1 + +import os +import shutil +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATA_DIR = os.path.join(PROJECT_ROOT, 'current-data') +STAGE_1_DIR = os.path.join(CURRENT_DATA_DIR, '.data', 'stage-1') +BIN_STAGE_1_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-1') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_1_DIR, "prepare-stage-1.py"), + os.path.join(PROJECT_ROOT, "bin", "delete-empty-counties.py"), + os.path.join(BIN_STAGE_1_DIR, "remove-obviously-bad-columns.py"), + os.path.join(PROJECT_ROOT, "bin", "remove-empty-columns.py"), + os.path.join(BIN_STAGE_1_DIR, "delete-malformed-csvs.py"), + os.path.join(PROJECT_ROOT, "bin", "delete-empty-counties.py"), + os.path.join(BIN_STAGE_1_DIR, "md5-the-scrapes.py") # Add the new script here + ] + + subprocess.run(["python", scripts[0]], check=True) + subprocess.run(["python", scripts[2]], check=True) + subprocess.run(["python", scripts[4]], check=True) + + subprocess.run(["python", scripts[1], "stage-1"], check=True) + subprocess.run(["python", scripts[3], "stage-1"], check=True) + subprocess.run(["python", scripts[5], "stage-1"], check=True) + subprocess.run(["python", scripts[6]], check=True) # Run the new script here + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-1".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 1...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/process-stage-2.md b/bin/process-stage-2.md new file mode 100644 index 0000000..7a22602 --- /dev/null +++ b/bin/process-stage-2.md @@ -0,0 +1,360 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I have a script called 'process-stage-2' which is similar to the 'process-stage-1' script we worked on earlier. + +I need this process-stage-2 script to be refactored to be in direct accordance with the above. + +Here is the process-stage-2 script: + + +#!/usr/bin/env python + +import os +import shutil +import subprocess +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage1_dir = os.path.join(project_root, '.data', 'stage-1') +stage2_dir = os.path.join(project_root, '.data', 'stage-2') + +def check_and_prepare_stage2(): + # Check if the stage-2 directory exists + if os.path.exists(stage2_dir): + # Remove the existing stage-2 directory + shutil.rmtree(stage2_dir) + + # Copy stage-1 directory to stage-2 + shutil.copytree(stage1_dir, stage2_dir) + +def run_scripts(): + scripts = [ + "remove-utm.py", + "sanitize-review-count.py", + "sanitize-phone-data.py", + "sanitize-gbp-business-website.py", + ("remove-empty-columns.py", "stage-2") + ] + + for script in scripts: + if isinstance(script, tuple): + script_name, arg = script + subprocess.run(["python", script_name, arg], check=True) + else: + subprocess.run(["python", script], check=True) + +if __name__ == '__main__': + check_and_prepare_stage2() + run_scripts() + + +I need this script to be refactored to be more like our 'process-stage-1' script in terms of style and reporting. + +Here ist he process-stage-1 script I am referencing: + + +#!/usr/bin/env python +# Script Name: process-stage-1 + +import os +import shutil +import subprocess + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATA_DIR = os.path.join(PROJECT_ROOT, 'current-data') +STAGE_1_DIR = os.path.join(CURRENT_DATA_DIR, '.data', 'stage-1') +BIN_STAGE_1_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-1') + +def prepare_stage_1(): + """Prepare the stage-1 directory by creating it and copying state directories into it.""" + # Check if the stage directory exists + if os.path.exists(STAGE_1_DIR): + # Remove the existing stage directory + shutil.rmtree(STAGE_1_DIR) + + # Create the stage directory + os.makedirs(STAGE_1_DIR) + + # Copy all state directories to the stage directory + for state_dir in os.listdir(CURRENT_DATA_DIR): + state_path = os.path.join(CURRENT_DATA_DIR, state_dir) + if os.path.isdir(state_path) and not state_dir.startswith('.'): + stage_state_path = os.path.join(STAGE_1_DIR, state_dir) + shutil.copytree(state_path, stage_state_path) + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_1_DIR, "prepare-stage-1.py"), + os.path.join(PROJECT_ROOT, "bin", "delete-empty-counties.py"), + os.path.join(PROJECT_ROOT, "bin", "column-count-correction.py"), + os.path.join(BIN_STAGE_1_DIR, "remove-obviously-bad-columns.py"), + os.path.join(PROJECT_ROOT, "bin", "remove-empty-columns.py"), + os.path.join(BIN_STAGE_1_DIR, "delete-malformed-csvs.py"), + os.path.join(PROJECT_ROOT, "bin", "delete-empty-counties.py"), + os.path.join(PROJECT_ROOT, "bin", "column-count-correction.py") + ] + + # Run scripts that do not require arguments + subprocess.run(["python", scripts[0]], check=True) + subprocess.run(["python", scripts[3]], check=True) + subprocess.run(["python", scripts[5]], check=True) + + # Run scripts that require the 'stage-1' argument + subprocess.run(["python", scripts[1], "stage-1"], check=True) + subprocess.run(["python", scripts[2], "stage-1"], check=True) + subprocess.run(["python", scripts[4], "stage-1"], check=True) + subprocess.run(["python", scripts[6], "stage-1"], check=True) + subprocess.run(["python", scripts[7], "stage-1"], check=True) + +if __name__ == '__main__': + prepare_stage_1() + run_scripts() + + +Currently the process-stage-2 script is running the following scripts (and paremeters): + + +remove-utm.py +sanitize-review-count.py +sanitize-phone-data.py +sanitize-gbp-business-website.py +remove-empty-columns.py stage-2 + + +This list of will need slight adjustment. + +The remove-utm.py, sanitize-review-count.py, sanitize-phone-data.py, and sanitize-gbp-business-website.py scripts will be located in [Stage 2 Binaries] (as per for how it calls its own version of 'prepare-stage-2', called 'prepare-stage-1'. + +Here is the prepare-stage-2 script below to give you a reference point: + + +#!/usr/bin/env python +# Script Name: prepare-stage-1 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +PARENT_DATA_DIRECTORY = CURRENT_DATASET + +def create_stage_1_directory(): + """Create the stage-1 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_1_DIRECTORY): + print("Existing Stage 1 found, deleting and replacing...") + shutil.rmtree(STAGE_1_DIRECTORY) + os.makedirs(STAGE_1_DIRECTORY) + print("Stage 1 directory created.") + +def find_parent_data(): + """Find all parent data directories.""" + parent_data_dirs = [ + d for d in os.listdir(PARENT_DATA_DIRECTORY) + if os.path.isdir(os.path.join(PARENT_DATA_DIRECTORY, d)) and not d.startswith('.') + ] + return parent_data_dirs + +def copy_data_to_stage_1(parent_data_dirs): + """Copy data from parent data directories to stage-1.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + shutil.copytree(src_dir, dest_dir) + +def verify_stage_1_data(parent_data_dirs): + """Verify that the data in stage-1 matches the parent data.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + + for root, dirs, files in os.walk(src_dir): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(PARENT_DATA_DIRECTORY, STAGE_1_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-1".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 1...") + + create_stage_1_directory() + + parent_data_dirs = find_parent_data() + + if parent_data_dirs: + print("Found data for the following states:") + print(", ".join(parent_data_dirs)) + + print("Copying data to Stage 1...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_1(parent_data_dirs) + spinner.succeed("Data copied.") + + print("Verifying Stage 1 data...") + is_valid = verify_stage_1_data(parent_data_dirs) + + if is_valid: + print("Data verification successful. Stage 1 data is valid.") + else: + print("Data verification failed. Stage 1 data is not valid.") + else: + print("No parent data found.") + +if __name__ == "__main__": + main() + + +Now please refactor the 'process-stage-2' script in the ways perscribed. + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I need to adjust the 'process-stage-2' script. + +The text "Running scripts" does not need a spinnder, and it should have a new line after it. + +It is causing the initial output of each script that is ran to appear on the same line as the "Running scripts" text and spinner. Which does not look good. + + + +#!/usr/bin/env python +# Script Name: process-stage-2 + +import os +import subprocess +from pyfiglet import Figlet +import halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_2_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-2') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_2_DIR, "prepare-stage-2.py"), + os.path.join(BIN_STAGE_2_DIR, "remove-utm.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-review-count.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-phone-data.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-gbp-business-website.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-1]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-2' argument + subprocess.run(["python", scripts[-1], "stage-2"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 2...") + spinner = halo.Halo(text='Running scripts', spinner='dots') + spinner.start() + run_scripts() + spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/process-stage-2.py b/bin/process-stage-2.py new file mode 100755 index 0000000..5630c8f --- /dev/null +++ b/bin/process-stage-2.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# Script Name: process-stage-2 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_2_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-2') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_2_DIR, "prepare-stage-2.py"), + os.path.join(BIN_STAGE_2_DIR, "remove-utm.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-review-count.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-phone-data.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-gbp-business-website.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-1]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-2' argument + subprocess.run(["python", scripts[-1], "stage-2"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 2...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/process-stage-3.md b/bin/process-stage-3.md new file mode 100644 index 0000000..3cbb65e --- /dev/null +++ b/bin/process-stage-3.md @@ -0,0 +1,478 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please refactor the process-stage-3 script below to be in accordance with the above. + +This script will not need to check for the existence of Stage 3, or worry about creating it. This task is handled elsewhere now. + +Please create the process-stage-3 in the same style as the process-stage-2 script below: + + +#!/usr/bin/env python +# Script Name: process-stage-2 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_2_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-2') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_2_DIR, "prepare-stage-2.py"), + os.path.join(BIN_STAGE_2_DIR, "remove-utm.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-review-count.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-phone-data.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-gbp-business-website.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-1]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-2' argument + subprocess.run(["python", scripts[-1], "stage-2"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 2...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +Here is the current version of the process-stage-3 script which needs refactoring: + + +#!/usr/bin/env python + +import os +import shutil +import subprocess +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +data_dir = os.path.join(project_root, '.data') +stage2_dir = os.path.join(data_dir, 'stage-2') +stage3_dir = os.path.join(data_dir, 'stage-3') + +def check_and_prepare_stage3(): + # Check if the stage-3 directory exists + if os.path.exists(stage3_dir): + # Remove the existing stage-3 directory + shutil.rmtree(stage3_dir) + + # Copy stage-2 directory to stage-3 + shutil.copytree(stage2_dir, stage3_dir) + +def run_scripts(): + scripts = [ + "column-search_review-count.py", + "column-search_review-rating.py", + "column-search_gbp-location.py", + "column-search_yib.py", + "column-search_gbp-business-category.py", + "column-search_gbp-matching-services.py", + "column-search_business-name.py", + ("remove-empty-columns.py", "stage-3"), + "column-search_unknowns.py" + ] + + for script in scripts: + if isinstance(script, tuple): + script_name, arg = script + subprocess.run(["python", script_name, arg], check=True) + else: + subprocess.run(["python", script], check=True) + + # Call report-unknowns.py with the "stage-3" option + subprocess.run(["python", "report-unknowns.py", "stage-3"], check=True) + +if __name__ == '__main__': + check_and_prepare_stage3() + run_scripts() + + +Item 1: The process-stage-3 script will need to call a scrip named "prepare-stage-3" script as the first script it calls. + +Item 2: The following scripts which process-stage-3 calls are located in [Stage 3 Binaries]: + + +column-search_review-count.py +column-search_review-rating.py +column-search_gbp-location.py +column-search_yib.py +column-search_gbp-business-category.py +column-search_gbp-matching-services.py +column-search_business-name.py +column-search_unknowns.py + + +The following script is located in [Binaries]. + + +remove-empty-columns.py + + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I ran into a problem from the 'process-stage-3' script: + + +#!/usr/bin/env python +# Script Name: process-stage-3 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_3_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-3') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_3_DIR, "prepare-stage-3.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_review-count.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_review-rating.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-location.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_yib.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-business-category.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-matching-services.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_business-name.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_unknowns.py"), + os.path.join(BIN_DIR, "report-unknowns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-2]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-3' argument + subprocess.run(["python", scripts[-2], "stage-3"], check=True) + subprocess.run(["python", scripts[-1], "stage-3"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-3".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 3...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +When it called the 'remove-empty-columns' script: + + +#!/usr/bin/env python +# Script Name: remove-empty-columns + +import os +import csv +import sys +from pyfiglet import Figlet +import halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") + +def print_help_message(): + """Prints the help message for script usage.""" + print("Usage: remove-empty-columns.py ") + print("Example: remove-empty-columns.py stage-1") + print("The provided stage directory must exist within the .data directory of the project root.") + +def delete_empty_columns(file_path): + """Deletes empty columns from a CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + + # Ensure each row has the same length as the headers + for row in data_rows: + while len(row) < len(headers): + row.append('') + + # Identify non-empty columns + non_empty_columns = [i for i in range(len(headers)) if any(row[i].strip() for row in data_rows)] + empty_column_count = len(headers) - len(non_empty_columns) + + # Create a new list to store the modified rows without empty columns + modified_rows = [[headers[i] for i in non_empty_columns]] + for row in data_rows: + modified_rows.append([row[i] for i in non_empty_columns]) + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + + return empty_column_count + +def remove_empty_columns(stage_directory): + """Remove empty columns from all CSV files in the given stage directory.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + print(f"Searching for empty columns to delete in {state_dir}...") + spinner = halo.Halo(spinner='dots', color='green') + spinner.start() + + total_removed_columns = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + total_removed_columns += delete_empty_columns(file_path) + + spinner.succeed(f"Removed columns from {state_dir}. Columns removed: {total_removed_columns}") + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-empty-columns".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if len(sys.argv) != 2: + print("Error: Incorrect number of arguments.") + print_help_message() + sys.exit(1) + + stage_arg = sys.argv[1] + if not stage_arg.startswith('stage-'): + print("Error: Invalid stage directory format.") + print_help_message() + sys.exit(1) + + stage_dir = os.path.join(DATA_DIRECTORY, stage_arg) + if not os.path.exists(stage_dir): + print(f"Error: The directory '{stage_dir}' does not exist.") + sys.exit(1) + + print(f"Removing empty columns in {stage_arg}...") + remove_empty_columns(stage_dir) + + final_spinner = halo.Halo(spinner='dots', color='green') + final_spinner.succeed("Finished removing empty columns.") + +if __name__ == "__main__": + main() + + +The problem was an error I got: + + + +Error: Incorrect number of arguments. +Usage: remove-empty-columns.py +Example: remove-empty-columns.py stage-1 +The provided stage directory must exist within the .data directory of the project root. +Traceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/./process-stage-3.py", line 51, in + main() + File "/home/ld/mgk-scrapes/bin/./process-stage-3.py", line 44, in main + run_scripts() + File "/home/ld/mgk-scrapes/bin/./process-stage-3.py", line 32, in run_scripts + subprocess.run(["python", script], check=True) + File "/usr/lib64/python3.12/subprocess.py", line 571, in run + raise CalledProcessError(retcode, process.args, +subprocess.CalledProcessError: Command '['python', '/home/ld/mgk-scrapes/bin/remove-empty-columns.py']' returned non-zero exit status 1. + + + +If I run 'remove-empty-columns' manually such as : + + +./remove-empty-columns stage3 + + +It runs correctly. + + +# Prompt 3 +I got the following error: + + +Error: Incorrect number of arguments. +Usage: remove-empty-columns.py +Example: remove-empty-columns.py stage-1 +The provided stage directory must exist within the .data directory of the project root. +Traceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/./process-stage-3.py", line 52, in + main() + File "/home/ld/mgk-scrapes/bin/./process-stage-3.py", line 45, in main + run_scripts() + File "/home/ld/mgk-scrapes/bin/./process-stage-3.py", line 32, in run_scripts + subprocess.run(["python", script], check=True) + File "/usr/lib64/python3.12/subprocess.py", line 571, in run + raise CalledProcessError(retcode, process.args, +subprocess.CalledProcessError: Command '['python', '/home/ld/mgk-scrapes/bin/remove-empty-columns.py']' returned non-zero exit status 1. + + +Are you sure the script is trying to run the command "remove-empty-columns.py stage-3"? + +Here is the working copy of the script: + + diff --git a/bin/process-stage-3.py b/bin/process-stage-3.py new file mode 100755 index 0000000..11fe15a --- /dev/null +++ b/bin/process-stage-3.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Script Name: process-stage-3 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_3_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-3') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_3_DIR, "prepare-stage-3.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_review-count.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_review-rating.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-location.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_yib.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-business-category.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_gbp-matching-services.py"), + os.path.join(BIN_STAGE_3_DIR, "column-search_business-name.py"), + (os.path.join(BIN_DIR, "remove-empty-columns.py"), "stage-3"), + os.path.join(BIN_STAGE_3_DIR, "column-search_unknowns.py"), + os.path.join(BIN_DIR, "report-unknowns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-3]: + subprocess.run(["python", script], check=True) + + # Run scripts that require the 'stage-3' argument + subprocess.run(["python", scripts[-3][0], scripts[-3][1]], check=True) + subprocess.run(["python", scripts[-2], "stage-3"], check=True) + subprocess.run(["python", scripts[-1], "stage-3"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-3".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 3...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/process-stage-4.md b/bin/process-stage-4.md new file mode 100644 index 0000000..4163dad --- /dev/null +++ b/bin/process-stage-4.md @@ -0,0 +1,175 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please refactor the process-stage-4 script below to be in accordance with the above. + +This script will not need to check for the existence of Stage 4, or worry about creating it. This task is handled elsewhere now. Remove any function that checks for Stage 4, creates it or recreates it. + +Please create the process-stage-4 in the same style as the process-stage-2 script below: + + +#!/usr/bin/env python +# Script Name: process-stage-2 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_2_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-2') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_2_DIR, "prepare-stage-2.py"), + os.path.join(BIN_STAGE_2_DIR, "remove-utm.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-review-count.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-phone-data.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-gbp-business-website.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-1]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-2' argument + subprocess.run(["python", scripts[-1], "stage-2"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 2...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +Here is the current version of the process-stage-4 script which needs refactoring: + + +#!/usr/bin/env python + +import os +import shutil +import subprocess +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +data_dir = os.path.join(project_root, '.data') +stage2_dir = os.path.join(data_dir, 'stage-2') +stage3_dir = os.path.join(data_dir, 'stage-3') + +def check_and_prepare_stage3(): + # Check if the stage-3 directory exists + if os.path.exists(stage3_dir): + # Remove the existing stage-3 directory + shutil.rmtree(stage3_dir) + + # Copy stage-2 directory to stage-3 + shutil.copytree(stage2_dir, stage3_dir) + +def run_scripts(): + scripts = [ + "column-search_review-count.py", + "column-search_review-rating.py", + "column-search_gbp-location.py", + "column-search_yib.py", + "column-search_gbp-business-category.py", + "column-search_gbp-matching-services.py", + "column-search_business-name.py", + ("remove-empty-columns.py", "stage-3"), + "column-search_unknowns.py" + ] + + for script in scripts: + if isinstance(script, tuple): + script_name, arg = script + subprocess.run(["python", script_name, arg], check=True) + else: + subprocess.run(["python", script], check=True) + + # Call report-unknowns.py with the "stage-3" option + subprocess.run(["python", "report-unknowns.py", "stage-3"], check=True) + +if __name__ == '__main__': + check_and_prepare_stage3() + run_scripts() + + +Item 1: The process-stage-4 script will need to call a script named "prepare-stage-4" script as the first script it calls. This script will need to be ran first by process-stage-4, and will be located in [Stage 4 Binaries] + +Item 2: The following scripts which process-stage-4 calls are located in [Stage 4 Binaries]: + + +reshift-terms.py +strip-neg-terms-from-unknown-cols.py +strip-neg-patterns-from-unknown-cols.py +delete-sponsored-listings.py + + +The following script is located in [Binaries]. + + +remove-empty-columns.py +report-unknowns.py + + +The scripts need to be ran in the following order: + + +reshift-terms.py +strip-neg-terms-from-unknown-cols.py +strip-neg-patterns-from-unknown-cols.py +delete-sponsored-listings.py +remove-empty-columns.py +report-unknowns.py + + diff --git a/bin/process-stage-4.py b/bin/process-stage-4.py new file mode 100755 index 0000000..c567ade --- /dev/null +++ b/bin/process-stage-4.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# Script Name: process-stage-4 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_4_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-4') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_4_DIR, "prepare-stage-4.py"), + os.path.join(BIN_STAGE_4_DIR, "reshift-terms.py"), + os.path.join(BIN_STAGE_4_DIR, "strip-neg-terms-from-unknown-cols.py"), + os.path.join(BIN_STAGE_4_DIR, "strip-neg-patterns-from-unknown-cols.py"), + os.path.join(BIN_STAGE_4_DIR, "delete-sponsored-listings.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py"), + os.path.join(BIN_DIR, "report-unknowns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-2]: + subprocess.run(["python", script], check=True) + + # Run scripts that require the 'stage-4' argument + subprocess.run(["python", scripts[-2], "stage-4"], check=True) + subprocess.run(["python", scripts[-1], "stage-4"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-4".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 4...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/process-stage-5.md b/bin/process-stage-5.md new file mode 100644 index 0000000..fe90c05 --- /dev/null +++ b/bin/process-stage-5.md @@ -0,0 +1,116 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please refactor the process-stage-5 script below to be in accordance with the above. + +This script will not need to check for the existence of Stage 5, or worry about creating it. This task is handled elsewhere now. Remove any function that checks for Stage 5, creates it or recreates it. + +Please create the process-stage-5 script in the same style as the process-stage-2 script below: + + +#!/usr/bin/env python +# Script Name: process-stage-2 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_2_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-2') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_2_DIR, "prepare-stage-2.py"), + os.path.join(BIN_STAGE_2_DIR, "remove-utm.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-review-count.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-phone-data.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-gbp-business-website.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-1]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-2' argument + subprocess.run(["python", scripts[-1], "stage-2"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 2...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +Here is the current version of the process-stage-5 script which needs refactoring: + + + + + +Item 1: The process-stage-5 script will need to call a script named "prepare-stage-5" script as the first script it calls. This script will need to be ran first by process-stage-5, and will be located in [Stage 5 Binaries] + +Item 2: The following scripts which process-stage-5 calls are located in [Stage 5 Binaries]: + + +standardize-location-data.py +retrieve-root-domain.py +strip-brackets-from-review-count.py + + +The scripts need to be ran in the following order: + + +standardize-location-data.py +retrieve-root-domain.py +strip-brackets-from-review-count.py + + diff --git a/bin/process-stage-5.py b/bin/process-stage-5.py new file mode 100755 index 0000000..4c9c47c --- /dev/null +++ b/bin/process-stage-5.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# Script Name: process-stage-5 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_5_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-5') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_5_DIR, "prepare-stage-5.py"), + os.path.join(BIN_STAGE_5_DIR, "standardize-location-data.py"), + os.path.join(BIN_STAGE_5_DIR, "retrieve-root-domain.py"), + os.path.join(BIN_STAGE_5_DIR, "strip-brackets-from-review-count.py"), + ] + + # Run scripts that do not require arguments + for script in scripts: + subprocess.run(["python", script], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-5".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 5...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/process-stage-6.md b/bin/process-stage-6.md new file mode 100644 index 0000000..55c259a --- /dev/null +++ b/bin/process-stage-6.md @@ -0,0 +1,1026 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please refactor the process-stage-6 script below to be in accordance with the above. + +This script will not need to check for the existence of Stage 6, or worry about creating it. This task is handled elsewhere now. Remove any function that checks for Stage 6, creates it or recreates it. + +Please create the process-stage-6 script in the same style as the process-stage-2 script below: + + +#!/usr/bin/env python +# Script Name: process-stage-2 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_2_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-2') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_2_DIR, "prepare-stage-2.py"), + os.path.join(BIN_STAGE_2_DIR, "remove-utm.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-review-count.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-phone-data.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-gbp-business-website.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-1]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-2' argument + subprocess.run(["python", scripts[-1], "stage-2"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 2...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +Here is the current version of the process-stage-6 script which needs refactoring: + + +#!/usr/bin/env python + +import os +import shutil +import csv +import subprocess +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage5_dir = os.path.join(project_root, '.data', 'stage-5') +stage6_dir = os.path.join(project_root, '.data', 'stage-6') + +def prepare_stage6_directory(): + # Check if the stage-6 directory exists + if os.path.exists(stage6_dir): + # Remove the existing stage-6 directory + shutil.rmtree(stage6_dir) + + # Create the stage-6 directory + os.makedirs(stage6_dir) + +def merge_csv_files(): + merged_csv_path = os.path.join(stage6_dir, '01-first-merger.csv') + headers_written = False + + with open(merged_csv_path, 'w', newline='') as merged_csv_file: + writer = csv.writer(merged_csv_file) + + for state_dir in os.listdir(stage5_dir): + state_path = os.path.join(stage5_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + + progress_bar = tqdm(csv_files, desc=f"Merging CSV files from: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + with open(file_path, 'r') as csv_file: + reader = csv.reader(csv_file) + headers = next(reader) + + if not headers_written: + writer.writerow(headers) + headers_written = True + + for row in reader: + writer.writerow(row) + + progress_bar.update(1) + +def run_scripts(): + # Run trim-invalid-rows-from-merger.py script + subprocess.run(["python", "trim-invalid-rows-from-merger.py"], check=True) + + # Run initial-deduplication.py script + subprocess.run(["python", "initial-deduplication.py"], check=True) + + # Run delete-bad-matching-services.py script + subprocess.run(["python", "delete-bad-matching-services.py"], check=True) + + # Run find-unknown-categories-and-services.py script + subprocess.run(["python", "find-unknown-categories-and-services.py"], check=True) + + # Run standardize-capital-letters.py script + subprocess.run(["python", "standardize-capital-letters.py"], check=True) + + # Run tally-initial-deduplication.py script + subprocess.run(["python", "tally-initial-deduplication.py"], check=True) + +if __name__ == '__main__': + # Prepare stage-6 directory + prepare_stage6_directory() + + # Merge CSV files from stage-5 + merge_csv_files() + + # Run the necessary scripts + run_scripts() + + +Item 1: The process-stage-6 script will need to call a script named "prepare-stage-6" script as the first script it calls. This script will need to be ran first by process-stage-6, and will be located in [Stage 6 Binaries] + +Item 2: The following scripts which process-stage-6 calls are located in [Stage 6 Binaries]: + + +trim-invalid-rows-from-merger.py +delete-bad-matching-services.py +standardize-capital-letters.py +find-unknown-categories-and-services.py +initial-deduplication.py + + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I wish to adjust the 'process-stage-6' script: + + + +Change 1: I it to now run the following scripts, and in the order I present them. These will all be located in [Stage 6 Binaries]. + + +prepare-stage-6.py +pre-sort.py +trim-invalid-rows-from-merger.py +delete-bad-matching-services.py +standardize-capital-letters.py + + + +# Prompt 3 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +Please extend the 'process-stage-6' script to also call the script 'business-name-strip-structures.py' inside of [Stage 6 Binaries]. This new script can be added to the end of the list of existing scripts, meaning run in it last in the current last. + + +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + subprocess.run(["python", script], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + + + +# Prompt 4 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +Please extend the 'process-stage-6' script to also call the script 'business-name-strip-structures.py' inside of [Stage 6 Binaries]. This new script can be added to the end of the list of existing scripts, meaning run in it last in the current last. + + +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + subprocess.run(["python", script], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + + + +I got the following error running the 'process-stage-6' script: + + +⠙ Stripping business name structuresTraceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/stage-6/business-name-strip-structures.py", line 71, in + main() + File "/home/ld/mgk-scrapes/bin/stage-6/business-name-strip-structures.py", line 64, in main + strip_structures_from_business_names() + File "/home/ld/mgk-scrapes/bin/stage-6/business-name-strip-structures.py", line 35, in strip_structures_from_business_names + with open(INPUT_CSV, 'r') as infile: + ^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: '/home/ld/mgk-scrapes/current-data/.data/stage-6/06-initial-deduplication.csv' +Traceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/./process-stage-6.py", line 41, in + main() + File "/home/ld/mgk-scrapes/bin/./process-stage-6.py", line 34, in main + run_scripts() + File "/home/ld/mgk-scrapes/bin/./process-stage-6.py", line 26, in run_scripts + subprocess.run(["python", script], check=True) + File "/usr/lib64/python3.12/subprocess.py", line 571, in run + raise CalledProcessError(retcode, process.args, +subprocess.CalledProcessError: Command '['python', '/home/ld/mgk-scrapes/bin/stage-6/business-name-strip-structures.py']' returned non-zero exit status 1. + + +Which looks like there was an error when running the script 'business-name-strip-structures.py' from the 'process-stage-6.py' script. + +It could be because the 'business-name-strip-structures.py' script does not have the paths to the assets it needs to work on set explicitly or correctly. + +I will provide to you two more scripts below so you can judge for yourself where the problem seems to be. I will provide the 'process-stage-6' script, as well as the 'business-name-strip-structures' script. + + +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py"), + os.path.join(BIN_STAGE_6_DIR, "business-name-strip-structures.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + subprocess.run(["python", script], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + + +#!/usr/bin/env python +# Script Name: business-name-strip-structures + +import os +import csv +import re +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '06-initial-deduplication.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '07-business-name-structures-stripped.csv') + +# List of terms to remove +terms_to_remove = [ + r'\bllc\b', r'\bllc\.\b', r'\bl\.l\.c\.\b', + r'\bco\b', r'\bco\.\b', r'\binc\b', r'\binc\.\b', + r'\bcorp\b', r'\bcorp\.\b' +] + +def clean_business_name(business_name): + """Remove specified terms from the business name and clean up punctuation.""" + for term in terms_to_remove: + # Remove term and handle preceding punctuation and spaces + business_name = re.sub(r'\s*[,.]*\s*' + term + r'[,.]*\s*', ' ', business_name, flags=re.IGNORECASE) + # Remove extra spaces and trailing punctuation + business_name = re.sub(r'\s+', ' ', business_name).strip() + business_name = re.sub(r'[,.]\s*$', '', business_name) + return business_name + +def strip_structures_from_business_names(): + """Strip specified structures from business names.""" + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + + if 'Business Name' not in headers: + print("Error: 'Business Name' column is missing.") + return + + rows = list(reader) + + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + row['Business Name'] = clean_business_name(row.get('Business Name', '').strip()) + writer.writerow(row) + +def main(): + figlet = Figlet(font='slant') + script_name = "business-name-strip-structures".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Stripping business name structures', spinner='dots') + spinner.start() + strip_structures_from_business_names() + spinner.succeed('Business name structures stripped.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + +Please do your best to source out the cause of this error, and then provide the what you judge to be a good solution. + +# Prompt 5 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +Please extend the 'process-stage-6' script to also call the script 'initial-deduplication.py' inside of [Stage 6 Binaries]. This new script can be made to run between the 'standardize-capital-letters' and 'business-name-strip-structures' scripts. initial-deduplication to be the second to last script to be ran in other words. + +Here is the current process-stage-6 script: + + +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py"), + os.path.join(BIN_STAGE_6_DIR, "business-name-strip-structures.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + try: + subprocess.run(["python", script], check=True) + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + break + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + + +# Prompt 6 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +Please extend the 'process-stage-6' to add some new scripts to the list of scripts already being called. + +Here is the current process-stage-6 script: + + +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py"), + os.path.join(BIN_STAGE_6_DIR, "initial-deduplication.py"), + os.path.join(BIN_STAGE_6_DIR, "business-name-strip-structures.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + try: + subprocess.run(["python", script], check=True) + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + break + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +After the 'business-name-strip-structures' script is called, I would like to run these additional scripts. Please have them ran in the order in which I provide them: + + +business-names-with-locations.py +uuid-business-1-by-gbp-business-phone.py +uuid-business-2-by-root-domain.py +uuid-business-3-by-business-name.py + + + +# Prompt 7 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +Please extend the 'process-stage-6' to add some new scripts to the list of scripts already being called. + +Here is the current process-stage-6 script: + + +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py"), + os.path.join(BIN_STAGE_6_DIR, "initial-deduplication.py"), + os.path.join(BIN_STAGE_6_DIR, "business-name-strip-structures.py"), + os.path.join(BIN_STAGE_6_DIR, "business-names-with-locations.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-1-by-gbp-business-phone.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-2-by-root-domain.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-3-by-business-name.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + try: + subprocess.run(["python", script], check=True) + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + break + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +After the 'uuid-business-3-by-business-name' script is called, I would like to run these additional scripts. Please have them ran in the order in which I provide them: + + +re-order-columns.py +fill-in-missing-data.py +uuid-business-4-single-location-entries.py +standardize-yib.py +uniqify-pass-1.py + + + +# Prompt 8 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +Please extend the 'process-stage-6' to add some new scripts to the list of scripts already being called. + +Here is the current process-stage-6 script: + + +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py"), + os.path.join(BIN_STAGE_6_DIR, "initial-deduplication.py"), + os.path.join(BIN_STAGE_6_DIR, "business-name-strip-structures.py"), + os.path.join(BIN_STAGE_6_DIR, "business-names-with-locations.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-1-by-gbp-business-phone.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-2-by-root-domain.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-3-by-business-name.py"), + os.path.join(BIN_STAGE_6_DIR, "re-order-columns.py"), + os.path.join(BIN_STAGE_6_DIR, "fill-in-missing-data.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-4-single-location-entries.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-yib.py"), + os.path.join(BIN_STAGE_6_DIR, "uniqify-pass-1.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + try: + subprocess.run(["python", script], check=True) + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + break + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +After the 'uniqify-pass-1' script is called, I would like to run these additional scripts. Please have them ran in the order in which I provide them: + + +uniqify-pass-2.py + diff --git a/bin/process-stage-6.py b/bin/process-stage-6.py new file mode 100755 index 0000000..47669e6 --- /dev/null +++ b/bin/process-stage-6.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# Script Name: process-stage-6 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_6_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-6') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_6_DIR, "prepare-stage-6.py"), + os.path.join(BIN_STAGE_6_DIR, "pre-sort.py"), + os.path.join(BIN_STAGE_6_DIR, "trim-invalid-rows-from-merger.py"), + os.path.join(BIN_STAGE_6_DIR, "delete-bad-matching-services.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-capital-letters.py"), + os.path.join(BIN_STAGE_6_DIR, "initial-deduplication.py"), + os.path.join(BIN_STAGE_6_DIR, "business-name-strip-structures.py"), + os.path.join(BIN_STAGE_6_DIR, "business-names-with-locations.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-1-by-gbp-business-phone.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-2-by-root-domain.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-3-by-business-name.py"), + os.path.join(BIN_STAGE_6_DIR, "re-order-columns.py"), + os.path.join(BIN_STAGE_6_DIR, "fill-in-missing-data.py"), + os.path.join(BIN_STAGE_6_DIR, "uuid-business-4-single-location-entries.py"), + os.path.join(BIN_STAGE_6_DIR, "standardize-yib.py"), + os.path.join(BIN_STAGE_6_DIR, "uniqify-pass-1.py"), + os.path.join(BIN_STAGE_6_DIR, "uniqify-pass-2.py"), + os.path.join(BIN_STAGE_6_DIR, "services-gbp-reputation-management.py"), + os.path.join(BIN_STAGE_6_DIR, "services-gbp-optimization.py"), + os.path.join(BIN_STAGE_6_DIR, "services-needs-website.py") + ] + + # Run scripts that do not require arguments + for script in scripts: + spinner = Halo(text=f'Running {os.path.basename(script)}', spinner='dots') + spinner.start() + try: + subprocess.run(["python", script], check=True) + spinner.succeed(f'Successfully ran {os.path.basename(script)}') + except subprocess.CalledProcessError as e: + spinner.fail(f"Error: {e}") + break + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 6...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + diff --git a/bin/remove-empty-columns.md b/bin/remove-empty-columns.md new file mode 100644 index 0000000..ddbd883 --- /dev/null +++ b/bin/remove-empty-columns.md @@ -0,0 +1,477 @@ +# Prompt 1 + + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I have the following script which you made earlier named "remove-empty-columns". + +Item 1: I would like you to refactor it according to the above. + +Item 2: This script should be able to be ran from anywhere. It will need to be able to be called upon different Stage's using a command line option like "remove-empty-columns.py stage-1". + +Item 3: I would also like this script to use a spinner to report working being done rather than a progress bar. It should report to the user state by state, and not county by county as it does not. + +item 4: This script should use pyfiglet to repor the script name as it runs as the first output. + +Here is the script: + + + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I have the following script which you made earlier named "remove-empty-columns". + +Item 1: I would like you to refactor it according to the above. + +Item 2: This script should be able to be ran from anywhere. It will need to be able to be called upon different Stage's using a command line option like "remove-empty-columns.py stage-1". + +Item 3: I would also like this script to use a spinner to report working being done rather than a progress bar. It should report to the user state by state, and not county by county as it does not. + +item 4: This script should use pyfiglet to repor the script name as it runs as the first output. + +Here is the script: + + + +I would like to modify this script: + +Item 1: It should tally up the deleted columns for each state, and report on that tally. + +When reporting as: + + +Finsihed processing . + + +I want it instead to say: + + +Removed columns from . Columns removed: + + +Where is the tally of columns removed from that state. + +Item 2: The first line of output besides the pyfiglet header is a line that reads: + + +Correcting column counts in /home/ld/mgk-scrapes/current-data/.data/stage-1... + + +This should report as: + + +Removing empty columns in + + +Where is the stage being operated on. + +Item 3: I wish to modify this reporting line too: + + +Correcting CSV files in + + +I would like the above line to be modified to the following instead: + + +Searching for empty columns to delete in + + + +# Prompt 3 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "remove-empty-columns" script: + + + +I would like this scripts pyfiglet function to be modified. + +Before displaying the pyfiglet banner as the first output, I would like the string that is output to be modified first. + +The hyphens in the script name should be replaced with spaces. Then the words that are left behind in the script name should be capitalized. Then we can print that version of the script name with pyfiglet. diff --git a/bin/remove-empty-columns.py b/bin/remove-empty-columns.py new file mode 100755 index 0000000..c0558ef --- /dev/null +++ b/bin/remove-empty-columns.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# Script Name: remove-empty-columns + +import os +import csv +import sys +from pyfiglet import Figlet +import halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") + +def print_help_message(): + """Prints the help message for script usage.""" + print("Usage: remove-empty-columns.py ") + print("Example: remove-empty-columns.py stage-1") + print("The provided stage directory must exist within the .data directory of the project root.") + +def delete_empty_columns(file_path): + """Deletes empty columns from a CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + + # Ensure each row has the same length as the headers + for row in data_rows: + while len(row) < len(headers): + row.append('') + + # Identify non-empty columns + non_empty_columns = [i for i in range(len(headers)) if any(row[i].strip() for row in data_rows)] + empty_column_count = len(headers) - len(non_empty_columns) + + # Create a new list to store the modified rows without empty columns + modified_rows = [[headers[i] for i in non_empty_columns]] + for row in data_rows: + modified_rows.append([row[i] for i in non_empty_columns]) + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + + return empty_column_count + +def remove_empty_columns(stage_directory): + """Remove empty columns from all CSV files in the given stage directory.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + print(f"Searching for empty columns to delete in {state_dir}...") + spinner = halo.Halo(spinner='dots', color='green') + spinner.start() + + total_removed_columns = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + total_removed_columns += delete_empty_columns(file_path) + + spinner.succeed(f"Removed columns from {state_dir}. Columns removed: {total_removed_columns}") + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-empty-columns".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if len(sys.argv) != 2: + print("Error: Incorrect number of arguments.") + print_help_message() + sys.exit(1) + + stage_arg = sys.argv[1] + if not stage_arg.startswith('stage-'): + print("Error: Invalid stage directory format.") + print_help_message() + sys.exit(1) + + stage_dir = os.path.join(DATA_DIRECTORY, stage_arg) + if not os.path.exists(stage_dir): + print(f"Error: The directory '{stage_dir}' does not exist.") + sys.exit(1) + + print(f"Removing empty columns in {stage_arg}...") + remove_empty_columns(stage_dir) + + final_spinner = halo.Halo(spinner='dots', color='green') + final_spinner.succeed("Finished removing empty columns.") + +if __name__ == "__main__": + main() + diff --git a/bin/report-unknowns.md b/bin/report-unknowns.md new file mode 100644 index 0000000..2e74805 --- /dev/null +++ b/bin/report-unknowns.md @@ -0,0 +1,224 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'report-unknowns' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +import sys +from tqdm import tqdm +from pyfiglet import Figlet + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + headers = next(reader) + + # Count the number of unknown columns + unknown_columns = [header for header in headers if header.startswith('unknown-')] + return len(unknown_columns) + +def report_unknowns(stage_dir): + # Get the project root directory + project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) + stage_dir_path = os.path.join(project_root, '.data', stage_dir) + unknown_columns_report_file = os.path.join(stage_dir_path, 'unknown-columns.csv') + + # Print the report title using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('Unknown Columns Report')) + + # Clear the report file before writing + with open(unknown_columns_report_file, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(['csv', 'unknowns']) + + total_unknown_columns = 0 + + # Process each CSV file in the specified stage directory + for state_dir in os.listdir(stage_dir_path): + state_path = os.path.join(stage_dir_path, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Reporting unknown columns in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + unknown_count = process_csv_file(file_path) + if unknown_count > 0: + total_unknown_columns += unknown_count + with open(unknown_columns_report_file, 'a', newline='') as report_file: + writer = csv.writer(report_file) + writer.writerow([file_path, unknown_count]) + progress_bar.update(1) + + # Print the total unknown columns found + print(f"Total Unknown Columns Found: {total_unknown_columns:,}") + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Usage: ./report-unknown.py ") + sys.exit(1) + + stage_directory = sys.argv[1] + report_unknowns(stage_directory) + + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 +I wish to modify the "report-unknowns" script further. + +I do not want it output progress bars for every county. + +Just the output we find in the remove-utm script is fine, but reworded to be accurate for this script. + + +# Prompt 3 + +The Total Tally is not correct. + +If you add up the tally of each state the number is much larger than the number reported for the Total Unknown Columns. + +Please examine the output below to see what I mean: + + +✔ Finished processing Alabama. Unknown columns found: 0 +✔ Finished processing Connecticut. Unknown columns found: 0 +✔ Finished processing Delaware. Unknown columns found: 0 +✔ Finished processing Florida. Unknown columns found: 2 +✔ Finished processing Georgia. Unknown columns found: 6 +✔ Finished processing Maine. Unknown columns found: 7 +✔ Finished processing Maryland. Unknown columns found: 7 +✔ Finished processing New Hampshire. Unknown columns found: 7 +✔ Finished processing North Carolina. Unknown columns found: 8 +✔ Finished processing Pennsylvania. Unknown columns found: 8 +✔ Finished processing Rhode Island. Unknown columns found: 8 +✔ Finished processing South Carolina. Unknown columns found: 9 +✔ Finished processing Vermont. Unknown columns found: 9 +✔ Finished processing Virginia. Unknown columns found: 11 +✔ Finished processing West Virginia. Unknown columns found: 11 +✔ Total Unknown Columns Found: 11 + diff --git a/bin/report-unknowns.py b/bin/report-unknowns.py new file mode 100755 index 0000000..12b6767 --- /dev/null +++ b/bin/report-unknowns.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# Script Name: report-unknowns + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") + +def process_csv_file(file_path): + """Read the CSV file and count the number of unknown columns.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + headers = next(reader) + + # Count the number of unknown columns + unknown_columns = [header for header in headers if header.startswith('unknown-')] + return len(unknown_columns) + +def report_unknowns(stage_dir): + """Generate a report of unknown columns in the specified stage directory.""" + stage_dir_path = os.path.join(DATA_DIRECTORY, stage_dir) + unknown_columns_report_file = os.path.join(stage_dir_path, 'unknown-columns.csv') + + # Print the report title using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('Unknown Columns Report')) + + # Clear the report file before writing + with open(unknown_columns_report_file, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(['csv', 'unknowns']) + + total_unknown_columns = 0 + + # Process each CSV file in the specified stage directory + for state_dir in os.listdir(stage_dir_path): + state_path = os.path.join(stage_dir_path, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_unknown_columns = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + unknown_count = process_csv_file(file_path) + if unknown_count > 0: + state_unknown_columns += unknown_count + with open(unknown_columns_report_file, 'a', newline='') as report_file: + writer = csv.writer(report_file) + writer.writerow([file_path, unknown_count]) + total_unknown_columns += state_unknown_columns + spinner.succeed(f'Finished processing {state_dir}. Unknown columns found: {state_unknown_columns}') + + # Print the total unknown columns found + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed(f"Total Unknown Columns Found: {total_unknown_columns:,}") + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Usage: ./report-unknowns.py ") + sys.exit(1) + + stage_directory = sys.argv[1] + report_unknowns(stage_directory) + diff --git a/bin/reshift-terms.py b/bin/reshift-terms.py new file mode 100755 index 0000000..76a6e12 --- /dev/null +++ b/bin/reshift-terms.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# reshift-terms + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage4_dir = os.path.join(project_root, '.data', 'stage-4') +data_dir = os.path.join(project_root, '.data') + +# Read the list of terms for GBP Business Category and GBP Matching Service from the text files +gbp_business_categories_file = os.path.join(data_dir, 'gbp-business-categories.txt') +gbp_matching_services_file = os.path.join(data_dir, 'gbp-matching-services.txt') + +# Helper function to read terms from a file +def read_terms(file_path): + if not os.path.isfile(file_path) or os.path.getsize(file_path) == 0: + print(f"Error: The file {file_path} is missing or empty.") + exit(1) + with open(file_path, 'r') as file: + terms = [line.strip().lower() for line in file if line.strip()] + if not terms: + print(f"Error: The file {file_path} does not contain any valid terms.") + exit(1) + return terms + +gbp_business_categories = read_terms(gbp_business_categories_file) +gbp_matching_services = read_terms(gbp_matching_services_file) + +def process_csv_file(file_path): + # Regular expressions for different data types + location_pattern = re.compile(r'^[\w\s]+, [A-Z]{2}(, United States)?$', re.IGNORECASE) + yib_pattern = re.compile(r'^\d+\+ years in business$', re.IGNORECASE) + review_rating_pattern = re.compile(r'^[1-5]\.\d$', re.IGNORECASE) + review_count_pattern = re.compile(r'^\(\d+\)$', re.IGNORECASE) + + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.DictReader(file) + rows = list(reader) + + if not rows: + return + + headers = reader.fieldnames + + # Iterate over each row + for row in rows: + for header in headers: + if header.startswith('unknown-'): + cell = row[header].strip().lower() + + if location_pattern.match(cell): + if not row['GBP Location']: + row['GBP Location'] = row[header] + row[header] = '' + elif yib_pattern.match(cell): + if not row['YiB']: + row['YiB'] = row[header] + row[header] = '' + elif review_rating_pattern.match(cell): + if not row['GBP Review Rating']: + row['GBP Review Rating'] = row[header] + row[header] = '' + elif review_count_pattern.match(cell): + if not row['GBP Review Count']: + row['GBP Review Count'] = row[header] + row[header] = '' + elif cell in gbp_business_categories: + if not row['GBP Business Category']: + row['GBP Business Category'] = row[header] + row[header] = '' + elif cell in gbp_matching_services: + if not row['GBP Matching Service']: + row['GBP Matching Service'] = row[header] + row[header] = '' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=headers) + writer.writeheader() + writer.writerows(rows) + +def reshift_terms(): + # Print the task name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('Reshift Terms')) + + # Process each CSV file in the stage 4 directory + for state_dir in os.listdir(stage4_dir): + state_path = os.path.join(stage4_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Shifting Terms in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + reshift_terms() + diff --git a/bin/scrape-census.py b/bin/scrape-census.py new file mode 100755 index 0000000..8f04f1c --- /dev/null +++ b/bin/scrape-census.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +import requests +import csv +import io + +# Send a GET request to the US Census Bureau dataset URL +url = "https://www2.census.gov/programs-surveys/popest/datasets/2010-2019/cities/totals/sub-est2019_all.csv" +response = requests.get(url) + +# Check if the request was successful +if response.status_code == 200: + # Read the CSV content from the response + csv_content = io.StringIO(response.text) + + # Create a CSV reader object + reader = csv.DictReader(csv_content) + + # Open the output CSV file in write mode + output_file = "cities_data.csv" + with open(output_file, "w", newline="", encoding="utf-8") as output: + fieldnames = ["city", "state", "population"] + writer = csv.DictWriter(output, fieldnames=fieldnames) + + # Write the header + writer.writeheader() + + # Iterate over each row in the input CSV + for row in reader: + # Extract the required fields + city = row["NAME"] + state = row["STNAME"] + population = row["POPESTIMATE2019"] + + # Write the data to the output CSV file + writer.writerow({"city": city, "state": state, "population": population}) + + print("Data extraction completed. Output saved as", output_file) +else: + print("Failed to fetch the dataset.") diff --git a/bin/scrape-simplemaps.py b/bin/scrape-simplemaps.py new file mode 100755 index 0000000..f7abc47 --- /dev/null +++ b/bin/scrape-simplemaps.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +import requests +import csv + +# Send a GET request to the SimpleMaps US Cities Database URL +url = "https://simplemaps.com/static/data/us-cities/1.73/basic/simplemaps_uscities_basicv1.73.zip" +response = requests.get(url) + +# Check if the request was successful +if response.status_code == 200: + # Save the ZIP file + with open("simplemaps_uscities_basic.zip", "wb") as file: + file.write(response.content) + print("ZIP file downloaded successfully.") +else: + print("Failed to download the ZIP file.") + +# Extract the CSV file from the ZIP (assuming you have already downloaded and saved the ZIP file) +import zipfile + +with zipfile.ZipFile("simplemaps_uscities_basic.zip", "r") as zip_ref: + zip_ref.extractall(".") + +# Read the extracted CSV file +csv_file = "uscities.csv" +output_file = "cities_data.csv" + +with open(csv_file, "r", encoding="utf-8") as file: + reader = csv.DictReader(file) + + # Open the output CSV file in write mode + with open(output_file, "w", newline="", encoding="utf-8") as output: + fieldnames = ["city", "state_id", "county_fips", "population"] + writer = csv.DictWriter(output, fieldnames=fieldnames) + + # Write the header + writer.writeheader() + + # Iterate over each row in the input CSV + for row in reader: + # Extract the required fields + city = row["city"] + state_id = row["state_id"] + county_fips = row["county_fips"] + population = row["population"] + + # Write the data to the output CSV file + writer.writerow({"city": city, "state_id": state_id, "county_fips": county_fips, "population": population}) + +print("Data extraction completed. Output saved as", output_file) diff --git a/bin/stage-1/delete-malformed-csvs.md b/bin/stage-1/delete-malformed-csvs.md new file mode 100644 index 0000000..ef567b0 --- /dev/null +++ b/bin/stage-1/delete-malformed-csvs.md @@ -0,0 +1,427 @@ +# Prompt 1 +Please explain the following script: + + + +# Prompt 2 +Please recreate the script for me. But refactor it according to the project details below: + + +- Description: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- Project Root: "/home/ld/mgk-scrapes" +- Current Dataset: "/current-data/" +- Data Directory: "/.data/" +- Parent Data: "/"(All directories that do not begin with a period ".") +- Stage Directories: "/stage-#" (Where # is a number.) +- Stage 1: "/stage-1/" +- Stage 2: "/stage-2/" +- Stage 3: "/stage-3/" +- Stage 4: "/stage-4/" +- Stage 5: "/stage-5/" +- Stage 6: "/stage-6/" +- Binaries: "/bin/" +- Stage 1 Binaries: "/stage-1/" +- Stage 2 Binaries: "/stage-2/" +- Stage 3 Binaries: "/stage-3/" +- Stage 4 Binaries: "/stage-4/" +- Stage 5 Binaries: "/stage-5/" +- Stage 6 Binaries: "/stage-6/" +- Rule 1: All scripts need to be able to be ran from any directory. +- Rule 2: All scripts need output what they are doing, as they are doing it. +- Rule 3: All tasks should use halo to report successes and failures. +- Rule 4: The python module "tqdm" can be used to report progress when appropriate. +- Rule 5: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- Rule 6: Every script should begin with a hashbang "#!/usr/bin/env python". +- Rule 7: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- Rule 8: Scripts should generally have robust error checking. + + +This script is named "delete-malformed-csvs". It will reside in . + + +# Prompt 3 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I have the following script which you made earlier named "delete-malformed-csvs". + + + +This script is exiting with an error when I run it. The error is: + + +Traceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/stage-1/./delete-malformed-csvs.py", line 79, in + main() + File "/home/ld/mgk-scrapes/bin/stage-1/./delete-malformed-csvs.py", line 48, in main + if len(sys.argv) != 2: + ^^^ +NameError: name 'sys' is not defined. Did you forget to import 'sys'? + + +Can you please fix it? + + +# Prompt 4 +I wish to make a modification to this script. + +Change 1: This script is asking for a stage directory to operate on. This script does not need that functionality. It can simply operate on Stage 1. + + +# Prompt 5 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "delete-malformed-csvs" script: + + + +I would like this scripts pyfiglet function to be modified. + +Before displaying the pyfiglet banner as the first output, I would like the string that is output to be modified first. + +The hyphens in the script name should be replaced with spaces. Then the words that are left behind in the script name should be capitalized. Then we can print that version of the script name with pyfiglet. + + +# Prompt 6 + +Please summarize the following script to me: + + + + +# Prompt 6 +Ok the logic behind what makes a CSV file is reversed. A CSV file is malformed if it has more than 12 columns, not fewer than. Please adjust the script accordingly. diff --git a/bin/stage-1/delete-malformed-csvs.py b/bin/stage-1/delete-malformed-csvs.py new file mode 100755 index 0000000..fcabe8f --- /dev/null +++ b/bin/stage-1/delete-malformed-csvs.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# Script Name: delete-malformed-csvs + +import os +import csv +import shutil +import sys +import halo +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") + +def find_malformed_csvs(stage_directory): + """Find all malformed CSV files in the given stage directory.""" + malformed_csvs = [] + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + with open(file_path, 'r') as f: + reader = csv.reader(f) + try: + rows = list(reader) + if len(rows) == 0 or len(rows[0]) > 12: + malformed_csvs.append(file_path) + except Exception as e: + malformed_csvs.append(file_path) + return malformed_csvs + +def delete_files(files): + """Delete the specified files.""" + for file in files: + os.remove(file) + +def main(): + figlet = Figlet(font='slant') + script_name = "delete-malformed-csvs".replace("-", " ").title() + print(figlet.renderText(script_name)) + + stage_directory = STAGE_1_DIRECTORY + + if not os.path.exists(stage_directory): + print(f"Error: The directory {stage_directory} does not exist.") + sys.exit(1) + + print(f"Scanning for malformed CSV files in {stage_directory}...") + spinner = halo.Halo(text='Scanning', spinner='dots') + spinner.start() + malformed_csvs = find_malformed_csvs(stage_directory) + spinner.succeed("Scan complete.") + + if malformed_csvs: + print("Malformed CSV files found:") + print(", ".join(malformed_csvs)) + + print("Deleting malformed CSV files...") + spinner.start() + delete_files(malformed_csvs) + spinner.succeed("Deletion complete.") + + print(f"Total files deleted: {len(malformed_csvs)}") + else: + print("No malformed CSV files found.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-1/md5-the-scrapes.py b/bin/stage-1/md5-the-scrapes.py new file mode 100755 index 0000000..31565b4 --- /dev/null +++ b/bin/stage-1/md5-the-scrapes.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# Script Name: md5-the-scrapes + +import os +import hashlib +import pandas as pd +import pyfiglet +from halo import Halo +from tqdm import tqdm +from multiprocessing import Pool, cpu_count + +# Function to calculate MD5 hash +def calculate_md5(row, exclude_columns): + md5_hash = hashlib.md5() + for column in row.index: + if column not in exclude_columns: + md5_hash.update(str(row[column]).encode('utf-8')) + return md5_hash.hexdigest() + +# Function to determine if a column should be excluded based on patterns +def should_exclude(column_data): + # Ensure the column data is treated as strings + column_data = column_data.astype(str) + + # Exclude column if it contains 5-star ratings (e.g., "4.1", "5.0") + if column_data.str.contains(r'^[1-5]\.\d$').any(): + return True + # Exclude column if it contains review counts (e.g., "(10)", "(1,234)", "-10") + if column_data.str.contains(r'^\(\d{1,3}(?:,\d{3})*\)$').any() or column_data.str.contains(r'^-\d+$').any(): + return True + # Exclude column if it contains years in business (e.g., "5+ years in business") + if column_data.str.contains(r'^\d+\+ years in business$').any(): + return True + return False + +# Function to check if a column contains terms from a given list +def contains_terms(column_data, terms): + column_data = column_data.astype(str) + match_count = column_data.apply(lambda x: any(term in x for term in terms)).sum() + return match_count / len(column_data) > 0.5 # Consider column matching if more than 50% cells match terms + +# Function to process a single state directory +def process_state(state_dir_info): + state_dir, stage_1_directory, gbp_matching_services, gbp_business_categories = state_dir_info + state_path = os.path.join(stage_1_directory, state_dir) + spinner = Halo(text=f'Processing state: {state_dir}', spinner='dots') + spinner.start() + + try: + for root, dirs, files in os.walk(state_path): + for file in files: + if file.endswith(".csv"): + file_path = os.path.join(root, file) + df = pd.read_csv(file_path) + + # Identify columns to exclude + exclude_columns = [] + + # Identify GBP Business Categories column + gbp_business_col = None + for column in df.columns: + if contains_terms(df[column], gbp_business_categories): + gbp_business_col = column + break + + # Identify GBP Matching Services column + for column in df.columns: + if column != gbp_business_col and contains_terms(df[column], gbp_matching_services): + exclude_columns.append(column) + break + + # Identify other columns to exclude based on patterns + for column in df.columns: + if should_exclude(df[column]): + exclude_columns.append(column) + + # Calculate MD5 for each row and add to new column "MD5 for Scrape" + df['MD5 for Scrape'] = df.apply(lambda row: calculate_md5(row, exclude_columns), axis=1) + + # Save the modified CSV + df.to_csv(file_path, index=False) + + spinner.succeed(f'Processing of state {state_dir} completed successfully.') + except Exception as e: + spinner.fail(f'Error processing state {state_dir}: {e}') + +def main(): + # Print script name using pyfiglet + script_name = "md5-the-scrapes".replace("-", " ").title() + print(pyfiglet.figlet_format(script_name)) + + # Define the directory structure + project_root = "/home/ld/mgk-scrapes" + stage_1_directory = os.path.join(project_root, "current-data", ".data", "stage-1") + gbp_matching_services_path = os.path.join(project_root, "current-data", ".data", "gbp-matching-services.txt") + gbp_business_categories_path = os.path.join(project_root, "current-data", ".data", "gbp-business-categories.txt") + + # Load GBP Matching Services and Business Categories lists + with open(gbp_matching_services_path, 'r') as file: + gbp_matching_services = file.read().splitlines() + with open(gbp_business_categories_path, 'r') as file: + gbp_business_categories = file.read().splitlines() + + # Get the list of state directories + state_dirs = [d for d in os.listdir(stage_1_directory) if os.path.isdir(os.path.join(stage_1_directory, d))] + + # Prepare arguments for parallel processing + state_dir_info_list = [(state_dir, stage_1_directory, gbp_matching_services, gbp_business_categories) for state_dir in state_dirs] + + # Process each state directory in parallel using multiprocessing + with Pool(cpu_count()) as pool: + pool.map(process_state, state_dir_info_list) + +if __name__ == "__main__": + main() + diff --git a/bin/stage-1/prepare-stage-1.md b/bin/stage-1/prepare-stage-1.md new file mode 100644 index 0000000..1c0b52f --- /dev/null +++ b/bin/stage-1/prepare-stage-1.md @@ -0,0 +1,325 @@ +# First Prompt + +- Description: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- Project Root: "/home/ld/mgk-scrapes" +- Current Dataset: "/current-data/" +- Data Directory: "/.data/" +- Parent Data: "/"(All directories that do not begin with a period ".") +- Stage Directories: "/stage-#" (Where # is a number.) +- Stage 1: "/stage-1/" +- Stage 2: "/stage-2/" +- Stage 3: "/stage-3/" +- Stage 4: "/stage-4/" +- Stage 5: "/stage-5/" +- Stage 6: "/stage-6/" +- Binaries: "/bin/" +- Stage 1 Binaries: "/stage-1/" +- Stage 2 Binaries: "/stage-2/" +- Stage 3 Binaries: "/stage-3/" +- Stage 4 Binaries: "/stage-4/" +- Stage 5 Binaries: "/stage-5/" +- Stage 6 Binaries: "/stage-6/" +- Rule 1: All scripts need to be able to be ran from any directory. +- Rule 2: All scripts need output what they are doing, as they are doing it. +- Rule 3: All tasks should use halo to report successes and failures. +- Rule 4: The python module "tqdm" can be used to report progress when appropriate. +- Rule 5: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- Rule 6: Every script should begin with a hashbang "#!/usr/bin/env python". +- Rule 7: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- Rule 8: Scripts should generally have robust error checking. + + +I need help creating a new script. See above for metadata and information regarding our project setup, configuration, purpose, and general rules to follow. + +This script should be called "prepare-stage-1". + +It should begin by telling us it is preparing for us. + +Then it should check for the existence of Stage 1. + +If Stage 1 is found to already exist, then the script should display a message that an existing Stage 1 was found, and is going to be delete and replaced. + +Then the script should delete Stage 1, and replace it. + +If Stage 1 was not found to already exist, the script can go straigh to creating Stage 1 for us. + +Stage 1 is created by creating the "stage-1" directory in the . + +Once the directory for Stage 1 has been created, the script needs to check for the . + +This part of the script should report: "Found data for the following states:". +Then on the line after that message, it should provide a comma seperated list of all the directories found in the . (Those directories, we call the ). + +After this, the script should then provide a message telling the user that the data is being copied, and present a spinner to the user as the data is being copied. + +After the data has been copied, report to the user that data was copied. + +Then the script should compare the data in with the to make sure all the CSV files and sub directories were copied over correctly. The data in should match the . + +As this check is being performed, the script should report "Verifying Stage 1 data." + +Once complete, the script should report whether or not the data was valid. + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "prepare-stage-1" script. + + +#!/usr/bin/env python +# Script Name: prepare-stage-1 + +import os +import shutil +import halo +from tqdm import tqdm + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +PARENT_DATA_DIRECTORY = CURRENT_DATASET + +def create_stage_1_directory(): + """Create the stage-1 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_1_DIRECTORY): + print("Existing Stage 1 found, deleting and replacing...") + shutil.rmtree(STAGE_1_DIRECTORY) + os.makedirs(STAGE_1_DIRECTORY) + print("Stage 1 directory created.") + +def find_parent_data(): + """Find all parent data directories.""" + parent_data_dirs = [ + d for d in os.listdir(PARENT_DATA_DIRECTORY) + if os.path.isdir(os.path.join(PARENT_DATA_DIRECTORY, d)) and not d.startswith('.') + ] + return parent_data_dirs + +def copy_data_to_stage_1(parent_data_dirs): + """Copy data from parent data directories to stage-1.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + shutil.copytree(src_dir, dest_dir) + +def verify_stage_1_data(parent_data_dirs): + """Verify that the data in stage-1 matches the parent data.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + + for root, dirs, files in os.walk(src_dir): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(PARENT_DATA_DIRECTORY, STAGE_1_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + print("Preparing Stage 1...") + + create_stage_1_directory() + + parent_data_dirs = find_parent_data() + + if parent_data_dirs: + print("Found data for the following states:") + print(", ".join(parent_data_dirs)) + + print("Copying data to Stage 1...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_1(parent_data_dirs) + spinner.succeed("Data copied.") + + print("Verifying Stage 1 data...") + is_valid = verify_stage_1_data(parent_data_dirs) + + if is_valid: + print("Data verification successful. Stage 1 data is valid.") + else: + print("Data verification failed. Stage 1 data is not valid.") + else: + print("No parent data found.") + +if __name__ == "__main__": + main() + + +This script should use pyfiglet to output the name of the script as it runs. + + +# Prompt 3 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "prepare-stage-1" script: + + + +I would like this scripts pyfiglet function to be modified. + +Before displaying the pyfiglet banner as the first output, I would like the string that is output to be modified first. + +The hyphens in the script name should be replaced with spaces. Then the words that are left behind in the script name should be capitalized. Then we can print that version of the script name with pyfiglet. diff --git a/bin/stage-1/prepare-stage-1.py b/bin/stage-1/prepare-stage-1.py new file mode 100755 index 0000000..0eec291 --- /dev/null +++ b/bin/stage-1/prepare-stage-1.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# Script Name: prepare-stage-1 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +PARENT_DATA_DIRECTORY = CURRENT_DATASET + +def create_stage_1_directory(): + """Create the stage-1 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_1_DIRECTORY): + print("Existing Stage 1 found, deleting and replacing...") + shutil.rmtree(STAGE_1_DIRECTORY) + os.makedirs(STAGE_1_DIRECTORY) + print("Stage 1 directory created.") + +def find_parent_data(): + """Find all parent data directories.""" + parent_data_dirs = [ + d for d in os.listdir(PARENT_DATA_DIRECTORY) + if os.path.isdir(os.path.join(PARENT_DATA_DIRECTORY, d)) and not d.startswith('.') + ] + return parent_data_dirs + +def copy_data_to_stage_1(parent_data_dirs): + """Copy data from parent data directories to stage-1.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + shutil.copytree(src_dir, dest_dir) + +def verify_stage_1_data(parent_data_dirs): + """Verify that the data in stage-1 matches the parent data.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + + for root, dirs, files in os.walk(src_dir): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(PARENT_DATA_DIRECTORY, STAGE_1_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-1".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 1...") + + create_stage_1_directory() + + parent_data_dirs = find_parent_data() + + if parent_data_dirs: + print("Found data for the following states:") + print(", ".join(parent_data_dirs)) + + print("Copying data to Stage 1...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_1(parent_data_dirs) + spinner.succeed("Data copied.") + + print("Verifying Stage 1 data...") + is_valid = verify_stage_1_data(parent_data_dirs) + + if is_valid: + print("Data verification successful. Stage 1 data is valid.") + else: + print("Data verification failed. Stage 1 data is not valid.") + else: + print("No parent data found.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-1/remove-obviously-bad-columns.md b/bin/stage-1/remove-obviously-bad-columns.md new file mode 100644 index 0000000..20ef574 --- /dev/null +++ b/bin/stage-1/remove-obviously-bad-columns.md @@ -0,0 +1,573 @@ +# First Prompt +Please explain the following script: + + + + +# Second Prompt + +- Description: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- Project Root: "/home/ld/mgk-scrapes" +- Current Dataset: "/current-data/" +- Data Directory: "/.data/" +- Parent Data: "/"(All directories that do not begin with a period ".") +- Stage Directories: "/stage-#" (Where # is a number.) +- Stage 1: "/stage-1/" +- Stage 2: "/stage-2/" +- Stage 3: "/stage-3/" +- Stage 4: "/stage-4/" +- Stage 5: "/stage-5/" +- Stage 6: "/stage-6/" +- Binaries: "/bin/" +- Stage 1 Binaries: "/stage-1/" +- Stage 2 Binaries: "/stage-2/" +- Stage 3 Binaries: "/stage-3/" +- Stage 4 Binaries: "/stage-4/" +- Stage 5 Binaries: "/stage-5/" +- Stage 6 Binaries: "/stage-6/" +- Rule 1: All scripts need to be able to be ran from any directory. +- Rule 2: All scripts need output what they are doing, as they are doing it. +- Rule 3: All tasks should use halo to report successes and failures. +- Rule 4: The python module "tqdm" can be used to report progress when appropriate. +- Rule 5: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- Rule 6: Every script should begin with a hashbang "#!/usr/bin/env python". +- Rule 7: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- Rule 8: Scripts should generally have robust error checking. + + +Ok please refactor that script to match the above. + +This script is named "prepare-csv-files", and will reside the directory. + + +# Prompt 3 +That script should not ask for a stage directory. It should simply operate on . + + +# Prompt 4 +I wish to make some changes to this script: + +Change 1: I do not want it to report for every county is looping through, instead just display which state is being processed, and use a spinner beside it instead of a progress bar. + +Change 2: The new name of this script is "remove-obviously-bad-columns". + +Change 3: The Reporting line should not say "Processing CSV files in , ". It should say "Removing obviously bad columns in ." and then have the progress spinner there. + +Change 4: We do not need to use pyfiglet to display the state name when processing a state. + + +# Prompt 5 +I wish to make some changes to this script: + +Change 1: Instead of starting it with "Preparing CSV files in ..." I want it to say "Removing obviously bad columns in ." + +Change 2: The second line the script displays is: + + +⠙ ProcessingRemoving obviously bad columns in Alabama...". + + +That looks ugly. The ProcessingRemoving. + +Just get rid of the "⠙ Processing" portion of the reporting. + +Change 3: At the end of the script, instead of saying "CSV file preparation complete." say "Obviously bad columns removed.". + +Change 4: Use pyfiglet to display the name of the script as the first output from the script. + + +# Prompt 6 +I wish to make some changes to this script: + +Change 1: Remove the spinner before the first "Removing obviously bad columns in . Also remove the spinner before the halo checkmark before "Finished processing " after processing the first state. + +Here is the output I want to fix with this change: + + +Removing obviously bad columns in /home/ld/mgk-scrapes/current-data/.data/stage-1... +⠙ Removing obviously bad columns in Alabama... +⠦ ✔ Finished processing Alabama. + + +To look correct, this output should resemble: + + +Removing obviously bad columns in /home/ld/mgk-scrapes/current-data/.data/stage-1... +Removing obviously bad columns in Alabama... +✔ Finished processing Alabama. + + + +# Prompt 7 +There are some changes I wish to make to this script: + +Change 1: The halo checkmark denoting success of a completed task is no longer green. I would like the halo icons to be colored like they were before. + + +# Prompt 8 +There are some changes I wish to make to this script: + +Change 1: There are 2 checkmarks appearing before every "Finished processing ." line. There should only be 1. One of the checkmarks is green as desired, and the other checkmark is uncolored, as not desired. + +Here is an example of the output I wish to fix: + + +Removing obviously bad columns in /home/ld/mgk-scrapes/current-data/.data/stage-1... +Removing obviously bad columns in Alabama... +✔ ✔ Finished processing Alabama. +Removing obviously bad columns in Connecticut... +✔ ✔ Finished processing Connecticut. +Removing obviously bad columns in Delaware... +✔ ✔ Finished processing Delaware. +Removing obviously bad columns in Florida... +✔ ✔ Finished processing Florida. +Removing obviously bad columns in Georgia... +✔ ✔ Finished processing Georgia. +Removing obviously bad columns in Maine... +✔ ✔ Finished processing Maine. + + + +# Prompt 9 +There are some changes I wish to make to this script: + +Change 1: The final line of the script "Obviously bad columns removed." has an uncolored checkmark from halo. It should be colored green like the other checkmarks. + + +# Prompt 10 +There are some changes I wish to make to this script: + +Change 1: The final line of the script "Obviously bad columns removed." has two checkmarks in it. I only want one halo checkmark. One of the checkmarks is colored green, which is desired. The other checkmark is uncolored, which is undesired. Please remove the undesired checkmark. + +Here is the undesired output: + + +✔ Finished processing West Virginia. +✔ ✔ Obviously bad columns removed. + + +Here is what it should look like instead: + + +✔ Finished processing West Virginia. +✔ Obviously bad columns removed. + + + +# Prompt 11 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "remove-obviously-bad-columns" script: + + + +I would like this scripts pyfiglet function to be modified. + +Before displaying the pyfiglet banner as the first output, I would like the string that is output to be modified first. + +The hyphens in the script name should be replaced with spaces. Then the words that are left behind in the script name should be capitalized. Then we can print that version of the script name with pyfiglet. + + +# Prompt 12 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to modify the "remove-obviously-bad-columns" script: + + + +Change 1: I would like this script to add a tallying function to it. For each line that reports such as: + + +Finished processing [State]. + + +I want it to instead report as such: + + +Removed bad columns in [State]. Bad columns found: [Tally] + + +Where [Tally] is the total tally of the number of bad columns which were removed from all of the CSV files in that state. + +At the very end of the script, it should provide a total tally: + + +Total number of obviously bad columns deleted: [Total Tally] + + +Where [Total Tally] is the total number of all bad columns which were deleted. diff --git a/bin/stage-1/remove-obviously-bad-columns.py b/bin/stage-1/remove-obviously-bad-columns.py new file mode 100755 index 0000000..abd83fe --- /dev/null +++ b/bin/stage-1/remove-obviously-bad-columns.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# Script Name: remove-obviously-bad-columns + +import os +import csv +import sys +from pyfiglet import Figlet +import halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") + +def process_csv_file(file_path): + """ + Process a CSV file to remove columns containing specific unwanted patterns. + + Parameters: + file_path (str): The path to the CSV file to be processed. + + Returns: + int: The number of columns removed from the CSV file. + """ + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + num_columns = len(rows[0]) + columns_to_delete = [False] * num_columns + + for col in range(num_columns - 1, -1, -1): + for row in rows: + cell_value = row[col] + if ( + "googleusercontent" in cell_value or + "googleapis" in cell_value or + "www.google.com/maps/vt/data" in cell_value or + " Opens " in cell_value or + "geocode" in cell_value or + "gstatic.com" in cell_value or + cell_value.startswith("tel:+") or + cell_value.startswith("⋅ ") or + cell_value in ["Open", "Quote", "Share", "Closed", "Closes soon", "Open 24 hours", + "Directions", "Booking", "Website", "https://www.google.com/#", "Call", + "Provides:", "Online estimates", "Onsite services", "No ratings or reviews"] or + cell_value.startswith('"') + ): + columns_to_delete[col] = True + break + + modified_rows = [] + for row in rows: + modified_row = [cell for idx, cell in enumerate(row) if not columns_to_delete[idx]] + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + + return sum(columns_to_delete) + +def process_csv_files(stage_directory): + """ + Process all CSV files in the given stage directory to remove unwanted columns. + + Parameters: + stage_directory (str): The path to the stage directory containing the CSV files to be processed. + """ + total_bad_columns = 0 + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + print(f"Removing obviously bad columns in {state_dir}...") + spinner = halo.Halo(spinner='dots', color='green') + spinner.start() + + state_bad_columns = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_bad_columns += process_csv_file(file_path) + + total_bad_columns += state_bad_columns + spinner.succeed(f"Removed bad columns in {state_dir}. Bad columns found: {state_bad_columns}") + + return total_bad_columns + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-obviously-bad-columns".replace("-", " ").title() + print(figlet.renderText(script_name)) + + stage_directory = STAGE_1_DIRECTORY + + if not os.path.exists(stage_directory): + print(f"Error: The directory {stage_directory} does not exist.") + sys.exit(1) + + print(f"Removing obviously bad columns in {stage_directory}...") + total_bad_columns = process_csv_files(stage_directory) + + # Final success message with a single colored checkmark + final_spinner = halo.Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed(f"Total number of obviously bad columns deleted: {total_bad_columns}") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/prepare-stage-2.md b/bin/stage-2/prepare-stage-2.md new file mode 100644 index 0000000..a14b894 --- /dev/null +++ b/bin/stage-2/prepare-stage-2.md @@ -0,0 +1,160 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +Now please create the prepare-stage-2 script. + +Base the 'prepare-stage-2' script on the 'prepare-stage-1' script below: + + + +#!/usr/bin/env python +# Script Name: prepare-stage-1 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +PARENT_DATA_DIRECTORY = CURRENT_DATASET + +def create_stage_1_directory(): + """Create the stage-1 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_1_DIRECTORY): + print("Existing Stage 1 found, deleting and replacing...") + shutil.rmtree(STAGE_1_DIRECTORY) + os.makedirs(STAGE_1_DIRECTORY) + print("Stage 1 directory created.") + +def find_parent_data(): + """Find all parent data directories.""" + parent_data_dirs = [ + d for d in os.listdir(PARENT_DATA_DIRECTORY) + if os.path.isdir(os.path.join(PARENT_DATA_DIRECTORY, d)) and not d.startswith('.') + ] + return parent_data_dirs + +def copy_data_to_stage_1(parent_data_dirs): + """Copy data from parent data directories to stage-1.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + shutil.copytree(src_dir, dest_dir) + +def verify_stage_1_data(parent_data_dirs): + """Verify that the data in stage-1 matches the parent data.""" + for directory in parent_data_dirs: + src_dir = os.path.join(PARENT_DATA_DIRECTORY, directory) + dest_dir = os.path.join(STAGE_1_DIRECTORY, directory) + + for root, dirs, files in os.walk(src_dir): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(PARENT_DATA_DIRECTORY, STAGE_1_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-1".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 1...") + + create_stage_1_directory() + + parent_data_dirs = find_parent_data() + + if parent_data_dirs: + print("Found data for the following states:") + print(", ".join(parent_data_dirs)) + + print("Copying data to Stage 1...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_1(parent_data_dirs) + spinner.succeed("Data copied.") + + print("Verifying Stage 1 data...") + is_valid = verify_stage_1_data(parent_data_dirs) + + if is_valid: + print("Data verification successful. Stage 1 data is valid.") + else: + print("Data verification failed. Stage 1 data is not valid.") + else: + print("No parent data found.") + +if __name__ == "__main__": + main() + + + + +# Prompt 2 +The script errored out. + +Here is the output: + + +Preparing Stage 2... +Stage 2 directory created. +Copying data to Stage 2... +⠙ Copying dataTraceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/stage-2/./prepare-stage-2.py", line 63, in + main() + File "/home/ld/mgk-scrapes/bin/stage-2/./prepare-stage-2.py", line 51, in main + copy_data_to_stage_2() + File "/home/ld/mgk-scrapes/bin/stage-2/./prepare-stage-2.py", line 27, in copy_data_to_stage_2 + shutil.copytree(STAGE_1_DIRECTORY, STAGE_2_DIRECTORY) + File "/usr/lib64/python3.12/shutil.py", line 600, in copytree + return _copytree(entries=entries, src=src, dst=dst, symlinks=symlinks, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/lib64/python3.12/shutil.py", line 498, in _copytree + os.makedirs(dst, exist_ok=dirs_exist_ok) + File "", line 225, in makedirs +FileExistsError: [Errno 17] File exists: '/home/ld/mgk-scrapes/current-data/.data/stage-2' + diff --git a/bin/stage-2/prepare-stage-2.py b/bin/stage-2/prepare-stage-2.py new file mode 100755 index 0000000..05bec2c --- /dev/null +++ b/bin/stage-2/prepare-stage-2.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Script Name: prepare-stage-2 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def create_stage_2_directory(): + """Create the stage-2 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_2_DIRECTORY): + print("Existing Stage 2 found, deleting and replacing...") + shutil.rmtree(STAGE_2_DIRECTORY) + os.makedirs(STAGE_2_DIRECTORY) + print("Stage 2 directory created.") + +def copy_data_to_stage_2(): + """Copy data from stage-1 to stage-2.""" + for item in os.listdir(STAGE_1_DIRECTORY): + s = os.path.join(STAGE_1_DIRECTORY, item) + d = os.path.join(STAGE_2_DIRECTORY, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + +def verify_stage_2_data(): + """Verify that the data in stage-2 matches the data in stage-1.""" + for root, dirs, files in os.walk(STAGE_1_DIRECTORY): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(STAGE_1_DIRECTORY, STAGE_2_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 2...") + + create_stage_2_directory() + + print("Copying data to Stage 2...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_2() + spinner.succeed("Data copied.") + + print("Verifying Stage 2 data...") + is_valid = verify_stage_2_data() + + if is_valid: + print("Data verification successful. Stage 2 data is valid.") + else: + print("Data verification failed. Stage 2 data is not valid.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/remove-utm.md b/bin/stage-2/remove-utm.md new file mode 100644 index 0000000..26d1be8 --- /dev/null +++ b/bin/stage-2/remove-utm.md @@ -0,0 +1,418 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I have a script 'remove-utm' which needs refactoring. + +Please refactor it to be in accordance with the above. + +This script currently reports on a per county basis, and I would prefer it to report on a per state basis. + +The only pyfiglet line in this script should be from [Rule 9] in . Get rid of the other pyfiglets. + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The utm codes were not correctly removed. + +Please make sure you are removing the utm codes in the same way the old 'remove-utm' script was. + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage2_dir = os.path.join(project_root, '.data', 'stage-2') + +def process_csv_file_remove_utm(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + # Create a new list to store the modified rows + modified_rows = [] + + # Iterate over each row + for row in rows: + # Create a new list to store the modified cells + modified_row = [] + + # Iterate over each cell in the row + for cell in row: + # Check if the cell contains the "?utm" parameter + if "?utm" in cell: + # Remove the "?utm" parameter and everything after it + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + + modified_row.append(modified_cell) + + modified_rows.append(modified_row) + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def remove_utm_parameters(): + # Process for removing "?utm" parameters + for state_dir in os.listdir(stage2_dir): + state_path = os.path.join(stage2_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + + progress_bar = tqdm(csv_files, desc=f"Trimming UTM tracking from: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + remove_utm_parameters() + + +Here is some sample CSV data how where we can see utm data remaining: + + +Alford Foundation and Crawl Space Repair,4.7,(272),Concrete contractor,+1 304-948-5855,Concrete Sealant,https://www.goalford.com/?utm_source=google&utm_medium=organic&utm_campaign=gmb-website,60+ years in business,"St Albans, WV, United States" +Basement Authority of West Virginia,4.8,(167),Waterproofing service,+1 304-898-1872,Concrete Leveling,https://www.basementauthorityofwv.com/service-areas/charleston-wv/?utm_source=google&utm_medium=organic&utm_campaign=gmb,20+ years in business,"Scott Depot, WV, United States" +Seal-Tite Basement Waterproofing Co,4.5,(51),Waterproofing service,+1 540-546-4724,Driveway Drains,https://www.sealtitebasement.com/?utm_source=gmb&utm_medium=organic&utm_campaign=gmb-troutville&utm_content=website,50+ years in business,"Troutville, VA, United States" + + + +# Prompt 3 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I need to enhance the script 'remove-utm': + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +It does a fine job right now of what it is supposed to do. We do not want to modify that aspect of the code. It works fine. However, I need to add a new string that it looks for and removes. + +Currently it is designed to search for cells that have an URL in them, and that URL contains a string of "?utm" in it. If that string is found in an URL in a cell, that "?utm" and everything after it gets deleted and stripped. This helps remove tracking mechanisms from the URLs in our database essentially. + +Another string I found which we need to remove is "?cid". I would like this script to handle "?cid" in the same way it handles the "?utm" string. + +Please adjust the remove-utm script accordingly for me. + + +# Prompt 4 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I would like to extend the script 'remove-utm': + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_tracking_params(file_path): + """Remove UTM and CID parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + elif "?cid" in cell: + modified_cell = cell.split("?cid")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM and CID parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_tracking_params(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing tracking parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("Tracking parameters removed.") + +if __name__ == "__main__": + main() + + +In addition to the existing strings being stripped, "?uid", and "?cid", I would like to add: "?cm_mmc". diff --git a/bin/stage-2/remove-utm.py b/bin/stage-2/remove-utm.py new file mode 100755 index 0000000..e5b0bb9 --- /dev/null +++ b/bin/stage-2/remove-utm.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_tracking_params(file_path): + """Remove UTM, CID, and CMMMC parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + elif "?cid" in cell: + modified_cell = cell.split("?cid")[0] + elif "?cm_mmc" in cell: + modified_cell = cell.split("?cm_mmc")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM, CID, and CMMMC parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_tracking_params(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing tracking parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("Tracking parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/sanitize-gbp-business-website.md b/bin/stage-2/sanitize-gbp-business-website.md new file mode 100644 index 0000000..77e22d0 --- /dev/null +++ b/bin/stage-2/sanitize-gbp-business-website.md @@ -0,0 +1,196 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'sanitize-gbp-business-website' now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage2_dir = os.path.join(project_root, '.data', 'stage-2') + +def sanitize_website_data(file_path): + new_column_name = 'GBP Business Website' + + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + # Get the headers and add the new column name if it does not exist + headers = rows[0] + if new_column_name not in headers: + headers.append(new_column_name) + new_column_index = headers.index(new_column_name) + + data_rows = rows[1:] + + # Create a new list to store the modified rows + modified_rows = [headers] + + # Iterate over each row + for row in data_rows: + new_row = row[:] + # Extend the row if the new column index is out of range + while len(new_row) <= new_column_index: + new_row.append('') + for i, cell in enumerate(row): + if cell.startswith('http://') or cell.startswith('https://'): + new_row[new_column_index] = cell # Assign the website to the new column + new_row[i] = '' + break + modified_rows.append(new_row) + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def sanitize_websites(): + # Process for sanitizing website data + for state_dir in os.listdir(stage2_dir): + state_path = os.path.join(stage2_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + + progress_bar = tqdm(csv_files, desc=f"Sanitizing Website data in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + sanitize_website_data(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + sanitize_websites() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/sanitize-gbp-business-website.py b/bin/stage-2/sanitize-gbp-business-website.py new file mode 100755 index 0000000..ea00610 --- /dev/null +++ b/bin/stage-2/sanitize-gbp-business-website.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# Script Name: sanitize-gbp-business-website + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def sanitize_website_data(file_path): + """Sanitize website data in the given CSV file.""" + new_column_name = 'GBP Business Website' + + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + if new_column_name not in headers: + headers.append(new_column_name) + new_column_index = headers.index(new_column_name) + + data_rows = rows[1:] + modified_rows = [headers] + + for row in data_rows: + new_row = row[:] + while len(new_row) <= new_column_index: + new_row.append('') + for i, cell in enumerate(row): + if cell.startswith('http://') or cell.startswith('https://'): + new_row[new_column_index] = cell + new_row[i] = '' + break + modified_rows.append(new_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to sanitize website data.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + sanitize_website_data(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "sanitize-gbp-business-website".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Sanitizing website data in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("Website data sanitized.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/sanitize-phone-data.md b/bin/stage-2/sanitize-phone-data.md new file mode 100644 index 0000000..a0fcc62 --- /dev/null +++ b/bin/stage-2/sanitize-phone-data.md @@ -0,0 +1,207 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'sanitize-phone-data' now: + + +#!/usr/bin/env python + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage2_dir = os.path.join(project_root, '.data', 'stage-2') + +def sanitize_phone_data(file_path): + new_column_name = 'GBP Business Phone' + + # Read the CSV file + with open(file_path, 'r', newline='') as file: + reader = csv.DictReader(file) + rows = list(reader) + + if not rows: + return + + # Get the headers and add the new column name if it does not exist + headers = reader.fieldnames + if new_column_name not in headers: + headers.append(new_column_name) + + # Regular expressions to match phone numbers + phone_patterns = [ + re.compile(r'^\(\d{3}\) \d{3}-\d{4}$'), # (555) 555-5555 + re.compile(r'^\+1 \d{3}-\d{3}-\d{4}$'), # +1 555-555-5555 + re.compile(r'^\(\d{3}\) \d{3}-\d{4} ext\. \d+$') # (555) 555-5555 ext. 5 + ] + + # Create a new list to store the modified rows + modified_rows = [] + + # Iterate over each row + for row in rows: + if new_column_name not in row: + row[new_column_name] = '' + for key, cell in row.items(): + if any(pattern.match(cell) for pattern in phone_patterns): + row[new_column_name] = cell # Move the phone number to the new column + row[key] = '' + break + modified_rows.append(row) + + # Format phone numbers in the new column + for row in modified_rows: + if re.match(r'^\(\d{3}\) \d{3}-\d{4}$', row[new_column_name]): + row[new_column_name] = '+1 ' + row[new_column_name][1:4] + '-' + row[new_column_name][6:9] + '-' + row[new_column_name][10:] + elif re.match(r'^\(\d{3}\) \d{3}-\d{4} ext\. \d+$', row[new_column_name]): + row[new_column_name] = '+1 ' + row[new_column_name][1:4] + '-' + row[new_column_name][6:9] + '-' + row[new_column_name][10:14] + ' ext. ' + row[new_column_name][19:] + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=headers) + writer.writeheader() + writer.writerows(modified_rows) + +def sanitize_phones(): + # Process for sanitizing phone data + for state_dir in os.listdir(stage2_dir): + state_path = os.path.join(stage2_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + + progress_bar = tqdm(csv_files, desc=f"Sanitizing Phone data in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + sanitize_phone_data(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + sanitize_phones() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/sanitize-phone-data.py b/bin/stage-2/sanitize-phone-data.py new file mode 100755 index 0000000..c31cefa --- /dev/null +++ b/bin/stage-2/sanitize-phone-data.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# Script Name: sanitize-phone-data + +import os +import csv +import re +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def sanitize_phone_data(file_path): + """Sanitize phone data in the given CSV file.""" + new_column_name = 'GBP Business Phone' + + with open(file_path, 'r', newline='') as file: + reader = csv.DictReader(file) + rows = list(reader) + + if not rows: + return + + headers = reader.fieldnames + if new_column_name not in headers: + headers.append(new_column_name) + + phone_patterns = [ + re.compile(r'^\(\d{3}\) \d{3}-\d{4}$'), # (555) 555-5555 + re.compile(r'^\+1 \d{3}-\d{3}-\d{4}$'), # +1 555-555-5555 + re.compile(r'^\(\d{3}\) \d{3}-\d{4} ext\. \d+$') # (555) 555-5555 ext. 5 + ] + + modified_rows = [] + + for row in rows: + if new_column_name not in row: + row[new_column_name] = '' + for key, cell in row.items(): + if any(pattern.match(cell) for pattern in phone_patterns): + row[new_column_name] = cell + row[key] = '' + break + modified_rows.append(row) + + for row in modified_rows: + if re.match(r'^\(\d{3}\) \d{3}-\d{4}$', row[new_column_name]): + row[new_column_name] = '+1 ' + row[new_column_name][1:4] + '-' + row[new_column_name][6:9] + '-' + row[new_column_name][10:] + elif re.match(r'^\(\d{3}\) \d{3}-\d{4} ext\. \d+$', row[new_column_name]): + row[new_column_name] = '+1 ' + row[new_column_name][1:4] + '-' + row[new_column_name][6:9] + '-' + row[new_column_name][10:14] + ' ext. ' + row[new_column_name][19:] + + with open(file_path, 'w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=headers) + writer.writeheader() + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to sanitize phone data.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + sanitize_phone_data(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "sanitize-phone-data".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Sanitizing phone data in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("Phone data sanitized.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/sanitize-review-count.md b/bin/stage-2/sanitize-review-count.md new file mode 100644 index 0000000..1036b26 --- /dev/null +++ b/bin/stage-2/sanitize-review-count.md @@ -0,0 +1,197 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'sanitize-review-count' now: + + +#!/usr/bin/env python + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage2_dir = os.path.join(project_root, '.data', 'stage-2') + +def sanitize_review_count_data(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + # Get the headers + headers = rows[0] + data_rows = rows[1:] + + # Regular expression to match the review count pattern + review_count_pattern = re.compile(r'^-\d{1,3}(?:,\d{3})*$') + + # Identify the "GBP Review Count" column + review_count_index = None + for i, header in enumerate(headers): + if any(review_count_pattern.match(row[i]) for row in data_rows): + review_count_index = i + break + + if review_count_index is None: + return # No matching column found + + # Create a new list to store the modified rows + modified_rows = [] + + # Iterate over each row (skip header row) + for row in rows: + if review_count_pattern.match(row[review_count_index]): + row[review_count_index] = f"({row[review_count_index][1:]})" + modified_rows.append(row) + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def sanitize_review_counts(): + # Process for sanitizing review count data + for state_dir in os.listdir(stage2_dir): + state_path = os.path.join(stage2_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + + progress_bar = tqdm(csv_files, desc=f"Sanitizing GBP Review Count data in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + sanitize_review_count_data(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + sanitize_review_counts() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-2/sanitize-review-count.py b/bin/stage-2/sanitize-review-count.py new file mode 100755 index 0000000..9fb7d27 --- /dev/null +++ b/bin/stage-2/sanitize-review-count.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# Script Name: sanitize-review-count + +import os +import csv +import re +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def sanitize_review_count_data(file_path): + """Sanitize review count data in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + review_count_pattern = re.compile(r'^-\d{1,3}(?:,\d{3})*$') + + review_count_index = None + for i, header in enumerate(headers): + if any(review_count_pattern.match(row[i]) for row in data_rows): + review_count_index = i + break + + if review_count_index is None: + return + + modified_rows = [] + for row in rows: + if review_count_pattern.match(row[review_count_index]): + row[review_count_index] = f"({row[review_count_index][1:]})" + modified_rows.append(row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to sanitize review count data.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + sanitize_review_count_data(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "sanitize-review-count".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Sanitizing review counts in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("Review counts sanitized.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_business-name.md b/bin/stage-3/column-search_business-name.md new file mode 100644 index 0000000..f3d2434 --- /dev/null +++ b/bin/stage-3/column-search_business-name.md @@ -0,0 +1,178 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_business-name' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'Business Name' in headers: + return + + # Check if the first column is not already renamed + if headers[0] not in ['GBP Business Category', 'GBP Matching Service', 'GBP Review Rating', 'GBP Review Count', 'Business Phone', 'GBP Business Website', 'YiB', 'GBP Location']: + headers[0] = 'Business Name' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_business_name(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('Business Name')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Searching for 'Business Name' in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_business_name() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_business-name.py b/bin/stage-3/column-search_business-name.py new file mode 100755 index 0000000..62c7800 --- /dev/null +++ b/bin/stage-3/column-search_business-name.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Script Name: column-search-business-name + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +def process_csv_file(file_path): + """Identify and rename the 'Business Name' column in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'Business Name' in headers: + return + + # Check if the first column is not already renamed + if headers[0] not in ['GBP Business Category', 'GBP Matching Service', 'GBP Review Rating', 'GBP Review Count', 'Business Phone', 'GBP Business Website', 'YiB', 'GBP Location', 'MD5 for Scrape']: + headers[0] = 'Business Name' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to rename the 'Business Name' column.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-business-name".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Renaming 'Business Name' columns in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("Business Name renaming completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_gbp-business-category.md b/bin/stage-3/column-search_gbp-business-category.md new file mode 100644 index 0000000..b50dc0d --- /dev/null +++ b/bin/stage-3/column-search_gbp-business-category.md @@ -0,0 +1,215 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_gbp-business-category' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') +data_dir = os.path.join(project_root, '.data') + +# Read the list of terms for GBP Business Category matching from the text file +gbp_business_categories_file = os.path.join(data_dir, 'gbp-business-categories.txt') + +# Check if the gbp-business-categories.txt file exists and is non-empty +if not os.path.isfile(gbp_business_categories_file) or os.path.getsize(gbp_business_categories_file) == 0: + print("Error: The gbp-business-categories.txt file is missing or empty.") + exit(1) + +with open(gbp_business_categories_file, 'r') as file: + gbp_business_categories = [line.strip().lower() for line in file if line.strip()] + +# Check if the gbp_business_categories list is empty +if not gbp_business_categories: + print("Error: The gbp-business-categories.txt file does not contain any valid categories.") + exit(1) + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Business Category' in headers: + return + + # Function to count matches + def count_matches(column_data): + matches = sum(1 for cell in column_data if cell.strip().lower() in gbp_business_categories) + return matches + + # Find the index of the column to rename + column_index = None + highest_matches = 0 + + for i, header in enumerate(headers): + if header not in ['GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'GBP Location']: + column_data = [row[i] for row in data_rows if row[i].strip()] + matches = count_matches(column_data) + if matches > highest_matches: + highest_matches = matches + column_index = i + + # Rename the column with the highest number of matches + if column_index is not None and highest_matches > 0: + headers[column_index] = 'GBP Business Category' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_gbp_business_category(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('GBP Business Category')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Searching for 'GBP Business Category' in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_gbp_business_category() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_gbp-business-category.py b/bin/stage-3/column-search_gbp-business-category.py new file mode 100755 index 0000000..ad9dc75 --- /dev/null +++ b/bin/stage-3/column-search_gbp-business-category.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# Script Name: column-search-gbp-business-category + +import os +import csv +import sys +from tqdm import tqdm +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +# Read the list of terms for GBP Business Category matching from the text file +GBP_BUSINESS_CATEGORIES_FILE = os.path.join(DATA_DIRECTORY, 'gbp-business-categories.txt') + +# Check if the gbp-business-categories.txt file exists and is non-empty +if not os.path.isfile(GBP_BUSINESS_CATEGORIES_FILE) or os.path.getsize(GBP_BUSINESS_CATEGORIES_FILE) == 0: + print("Error: The gbp-business-categories.txt file is missing or empty.") + sys.exit(1) + +with open(GBP_BUSINESS_CATEGORIES_FILE, 'r') as file: + gbp_business_categories = [line.strip().lower() for line in file if line.strip()] + +# Check if the gbp_business_categories list is empty +if not gbp_business_categories: + print("Error: The gbp-business-categories.txt file does not contain any valid categories.") + sys.exit(1) + +def process_csv_file(file_path): + """Identify and rename the 'GBP Business Category' column in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Business Category' in headers: + return + + def count_matches(column_data): + matches = sum(1 for cell in column_data if cell.strip().lower() in gbp_business_categories) + return matches + + # Find the index of the column to rename + column_index = None + highest_matches = 0 + + for i, header in enumerate(headers): + if header not in ['GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'GBP Location', 'MD5 for Scrape']: + column_data = [row[i] for row in data_rows if row[i].strip()] + matches = count_matches(column_data) + if matches > highest_matches: + highest_matches = matches + column_index = i + + # Rename the column with the highest number of matches + if column_index is not None and highest_matches > 0: + headers[column_index] = 'GBP Business Category' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to search for 'GBP Business Category'.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-gbp-business-category".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Searching for 'GBP Business Category' in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("GBP Business Category search completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_gbp-location.md b/bin/stage-3/column-search_gbp-location.md new file mode 100644 index 0000000..b5c39e7 --- /dev/null +++ b/bin/stage-3/column-search_gbp-location.md @@ -0,0 +1,195 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_gbp-location' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Location' in headers: + return + + # Function to count matches + def count_matches(column_data): + matches = sum(1 for cell in column_data if re.match(r'^[\w\s]+, [A-Z]{2}$', cell) or re.match(r'^[\w\s]+, [A-Z]{2}, United States$', cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Review Count', 'Business Phone', 'GBP Business Website', 'YiB']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'GBP Location' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_gbp_location(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('GBP Location')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + num_csv_files = len(csv_files) + progress_bar = tqdm(csv_files, desc=f"Searching for 'GBP Location' in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_gbp_location() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_gbp-location.py b/bin/stage-3/column-search_gbp-location.py new file mode 100755 index 0000000..95a2d9d --- /dev/null +++ b/bin/stage-3/column-search_gbp-location.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Script Name: column-search-gbp-location + +import os +import csv +import re +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +def process_csv_file(file_path): + """Identify and rename the 'GBP Location' column in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Location' in headers: + return + + def count_matches(column_data): + matches = sum(1 for cell in column_data if re.match(r'^[\w\s]+, [A-Z]{2}$', cell) or re.match(r'^[\w\s]+, [A-Z]{2}, United States$', cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'MD5 for Scrape']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'GBP Location' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to search for 'GBP Location'.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-gbp-location".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Searching for 'GBP Location' in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("GBP Location search completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_gbp-matching-services.md b/bin/stage-3/column-search_gbp-matching-services.md new file mode 100644 index 0000000..3277d53 --- /dev/null +++ b/bin/stage-3/column-search_gbp-matching-services.md @@ -0,0 +1,221 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_gbp-matching-services' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') +data_dir = os.path.join(project_root, '.data') + +# Read the list of terms for GBP Matching Service from the text file +gbp_matching_services_file = os.path.join(data_dir, 'gbp-matching-services.txt') +with open(gbp_matching_services_file, 'r') as file: + gbp_matching_services = {line.strip().lower() for line in file} + +# List of headers to skip +headers_to_skip = [ + 'GBP Business Category', 'GBP Matching Service', 'Business Name', + 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', + 'GBP Business Website', 'YiB', 'GBP Location' +] + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Matching Service' in headers: + return + + # Function to count matches and non-matches + def count_matches(column_data): + matches = sum(1 for cell in column_data if cell.strip().lower() in gbp_matching_services) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in headers_to_skip: + column_data = [row[i] for row in data_rows if row[i].strip()] + matches = count_matches(column_data) + if matches > 0: + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by number of matches + best_column_index = best_column[0] + + # Insert the 'GBP Matching Service' column if it doesn't exist + if 'GBP Matching Service' not in headers: + headers.insert(best_column_index + 1, 'GBP Matching Service') + for row in data_rows: + row.insert(best_column_index + 1, '') + + # Move the data from the best column to the 'GBP Matching Service' column + gbp_matching_service_index = headers.index('GBP Matching Service') + for row in data_rows: + row[gbp_matching_service_index] = row[best_column_index] + row[best_column_index] = '' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_gbp_matching_service(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('GBP Matching Service')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Searching for 'GBP Matching Service' in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_gbp_matching_service() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_gbp-matching-services.py b/bin/stage-3/column-search_gbp-matching-services.py new file mode 100755 index 0000000..7518515 --- /dev/null +++ b/bin/stage-3/column-search_gbp-matching-services.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# Script Name: column-search-gbp-matching-services + +import os +import csv +import sys +from tqdm import tqdm +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +# Read the list of terms for GBP Matching Service from the text file +GBP_MATCHING_SERVICES_FILE = os.path.join(DATA_DIRECTORY, 'gbp-matching-services.txt') + +# Check if the gbp-matching-services.txt file exists and is non-empty +if not os.path.isfile(GBP_MATCHING_SERVICES_FILE) or os.path.getsize(GBP_MATCHING_SERVICES_FILE) == 0: + print("Error: The gbp-matching-services.txt file is missing or empty.") + sys.exit(1) + +with open(GBP_MATCHING_SERVICES_FILE, 'r') as file: + gbp_matching_services = {line.strip().lower() for line in file} + +# List of headers to skip +HEADERS_TO_SKIP = [ + 'GBP Business Category', 'GBP Matching Service', 'Business Name', + 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', + 'GBP Business Website', 'YiB', 'GBP Location', 'MD5 for Scrape' +] + +def process_csv_file(file_path): + """Identify and rename the 'GBP Matching Service' column in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Matching Service' in headers: + return + + def count_matches(column_data): + matches = sum(1 for cell in column_data if cell.strip().lower() in gbp_matching_services) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in HEADERS_TO_SKIP: + column_data = [row[i] for row in data_rows if row[i].strip()] + matches = count_matches(column_data) + if matches > 0: + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by number of matches + best_column_index = best_column[0] + + # Insert the 'GBP Matching Service' column if it doesn't exist + if 'GBP Matching Service' not in headers: + headers.insert(best_column_index + 1, 'GBP Matching Service') + for row in data_rows: + row.insert(best_column_index + 1, '') + + # Move the data from the best column to the 'GBP Matching Service' column + gbp_matching_service_index = headers.index('GBP Matching Service') + for row in data_rows: + row[gbp_matching_service_index] = row[best_column_index] + row[best_column_index] = '' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to search for 'GBP Matching Service'.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-gbp-matching-services".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Searching for 'GBP Matching Service' in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("GBP Matching Service search completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_review-count.md b/bin/stage-3/column-search_review-count.md new file mode 100644 index 0000000..36b871f --- /dev/null +++ b/bin/stage-3/column-search_review-count.md @@ -0,0 +1,197 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_review-count' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Review Count' in headers: + return + + # Function to count matches and non-matches + def count_matches(column_data): + pattern = re.compile(r'^\(\d+\)$') + phone_pattern = re.compile(r'^\(\d{3}\) \d{3}-\d{4}$') + matches = sum(1 for cell in column_data if pattern.match(cell) and not phone_pattern.match(cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'GBP Location']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'GBP Review Count' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_review_count(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('GBP Review Count')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + num_csv_files = len(csv_files) + progress_bar = tqdm(csv_files, desc=f"Searching for 'GBP Review Count' in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_review_count() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_review-count.py b/bin/stage-3/column-search_review-count.py new file mode 100755 index 0000000..b97d0e6 --- /dev/null +++ b/bin/stage-3/column-search_review-count.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# Script Name: column-search-review-count + +import os +import csv +import re +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +def process_csv_file(file_path): + """Identify and rename the 'GBP Review Count' column in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Review Count' in headers: + return + + def count_matches(column_data): + pattern = re.compile(r'^\(\d+\)$') + phone_pattern = re.compile(r'^\(\d{3}\) \d{3}-\d{4}$') + matches = sum(1 for cell in column_data if pattern.match(cell) and not phone_pattern.match(cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'GBP Location', 'MD5 for Scrape']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'GBP Review Count' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to search for 'GBP Review Count'.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-review-count".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Searching for 'GBP Review Count' in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("'GBP Review Count' search completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_review-rating.md b/bin/stage-3/column-search_review-rating.md new file mode 100644 index 0000000..29dcd28 --- /dev/null +++ b/bin/stage-3/column-search_review-rating.md @@ -0,0 +1,196 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_review-rating' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Review Rating' in headers: + return + + # Function to count matches and non-matches + def count_matches(column_data): + pattern = re.compile(r'^\d\.\d$') + matches = sum(1 for cell in column_data if pattern.match(cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'GBP Location']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'GBP Review Rating' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_review_rating(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('GBP Review Rating')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + num_csv_files = len(csv_files) + progress_bar = tqdm(csv_files, desc=f"Searching for 'GBP Review Rating' in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_review_rating() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_review-rating.py b/bin/stage-3/column-search_review-rating.py new file mode 100755 index 0000000..2d5cbe8 --- /dev/null +++ b/bin/stage-3/column-search_review-rating.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# Script Name: column-search-review-rating + +import os +import csv +import re +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +def process_csv_file(file_path): + """Identify and rename the 'GBP Review Rating' column in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'GBP Review Rating' in headers: + return + + def count_matches(column_data): + pattern = re.compile(r'^\d\.\d$') + matches = sum(1 for cell in column_data if pattern.match(cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'GBP Location', 'MD5 for Scrape']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'GBP Review Rating' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to search for 'GBP Review Rating'.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-review-rating".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Searching for 'GBP Review Rating' in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("'GBP Review Rating' search completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_unknowns.md b/bin/stage-3/column-search_unknowns.md new file mode 100644 index 0000000..83fbc6b --- /dev/null +++ b/bin/stage-3/column-search_unknowns.md @@ -0,0 +1,182 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_unknowns' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + # Check for columns with data that don't have a header matching any of the target column headers + target_headers = ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'YiB', 'GBP Location'] + unknown_count = 1 + for i, header in enumerate(headers): + if header not in target_headers: + # Check if the column has data + has_data = any(row[i] for row in data_rows) + if has_data: + headers[i] = f'unknown-{unknown_count}' + unknown_count += 1 + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_unknowns(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('Unknown Columns')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + num_csv_files = len(csv_files) + progress_bar = tqdm(csv_files, desc=f"Searching for unknown columns in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_unknowns() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_unknowns.py b/bin/stage-3/column-search_unknowns.py new file mode 100755 index 0000000..8cffe9d --- /dev/null +++ b/bin/stage-3/column-search_unknowns.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Script Name: column-search-unknowns + +import os +import csv +import sys +from tqdm import tqdm +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +def process_csv_file(file_path): + """Identify and rename columns with unknown data in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check for columns with data that don't have a header matching any of the target column headers + target_headers = [ + 'GBP Business Category', 'GBP Matching Service', 'Business Name', + 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', + 'GBP Business Website', 'YiB', 'GBP Location', 'MD5 for Scrape' + ] + unknown_count = 1 + for i, header in enumerate(headers): + if header not in target_headers: + # Check if the column has data + has_data = any(row[i] for row in data_rows) + if has_data: + headers[i] = f'unknown-{unknown_count}' + unknown_count += 1 + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to identify unknown columns.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-unknowns".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Identifying unknown columns in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("Unknown columns identification completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/column-search_yib.md b/bin/stage-3/column-search_yib.md new file mode 100644 index 0000000..60c3831 --- /dev/null +++ b/bin/stage-3/column-search_yib.md @@ -0,0 +1,251 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'column-search_yib' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage3_dir = os.path.join(project_root, '.data', 'stage-3') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'YiB' in headers: + return + + # Function to count matches + def count_matches(column_data): + pattern = re.compile(r'^\d+\+ years in business$') + matches = sum(1 for cell in column_data if pattern.match(cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'GBP Location']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'YiB' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def search_yib(): + # Print the column name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('YiB')) + + # Process each CSV file in the stage 3 directory + for state_dir in os.listdir(stage3_dir): + state_path = os.path.join(stage3_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + num_csv_files = len(csv_files) + progress_bar = tqdm(csv_files, desc=f"Searching for 'YiB' in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + search_yib() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The script was not correctly refactored according to the + +The script errored out. It would not have errored out if the script was referring to the correct locations for data as described in the + +Here is the error: + + +Traceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/stage-3/./column-search_yib.py", line 74, in + search_yib() + File "/home/ld/mgk-scrapes/bin/stage-3/./column-search_yib.py", line 58, in search_yib + for state_dir in os.listdir(stage3_dir): + ^^^^^^^^^^^^^^^^^^^^^^ +FileNotFoundError: [Errno 2] No such file or directory: '/home/ld/mgk-scrapes/bin/Concrete Sealing Company/.data/stage-3' + diff --git a/bin/stage-3/column-search_yib.py b/bin/stage-3/column-search_yib.py new file mode 100755 index 0000000..6f67853 --- /dev/null +++ b/bin/stage-3/column-search_yib.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# Script Name: column-search-yib + +import os +import csv +import re +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +def process_csv_file(file_path): + """Identify and rename the 'YiB' column in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + headers = rows[0] + data_rows = rows[1:] + + # Check if the column has already been processed by other scripts + if 'YiB' in headers: + return + + def count_matches(column_data): + pattern = re.compile(r'^\d+\+ years in business$') + matches = sum(1 for cell in column_data if pattern.match(cell)) + return matches + + # Evaluate all columns + column_scores = [] + for i, header in enumerate(headers): + if header not in ['GBP Business Category', 'GBP Matching Service', 'Business Name', 'GBP Review Rating', 'GBP Review Count', 'GBP Business Phone', 'GBP Business Website', 'GBP Location', 'MD5 for Scrape']: + column_data = [row[i] for row in data_rows if row[i]] + matches = count_matches(column_data) + column_scores.append((i, matches)) + + # Find the column with the highest number of matches + if column_scores: + best_column = max(column_scores, key=lambda x: x[1]) # max by absolute matches + if best_column[1] > 0: # at least one match + headers[best_column[0]] = 'YiB' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to search for 'YiB'.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "column-search-yib".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_3_DIRECTORY): + print(f"Error: The directory {STAGE_3_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Searching for 'YiB' in {STAGE_3_DIRECTORY}...") + process_csv_files(STAGE_3_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("'YiB' search completed.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-3/prepare-stage-3.md b/bin/stage-3/prepare-stage-3.md new file mode 100644 index 0000000..6e26712 --- /dev/null +++ b/bin/stage-3/prepare-stage-3.md @@ -0,0 +1,116 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +Now please create a prepare-stage-3 version of the script. + +Base the 'prepare-stage-3' script on the 'prepare-stage-2' script below: + + +#!/usr/bin/env python +# Script Name: prepare-stage-2 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def create_stage_2_directory(): + """Create the stage-2 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_2_DIRECTORY): + print("Existing Stage 2 found, deleting and replacing...") + shutil.rmtree(STAGE_2_DIRECTORY) + os.makedirs(STAGE_2_DIRECTORY) + print("Stage 2 directory created.") + +def copy_data_to_stage_2(): + """Copy data from stage-1 to stage-2.""" + for item in os.listdir(STAGE_1_DIRECTORY): + s = os.path.join(STAGE_1_DIRECTORY, item) + d = os.path.join(STAGE_2_DIRECTORY, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + +def verify_stage_2_data(): + """Verify that the data in stage-2 matches the data in stage-1.""" + for root, dirs, files in os.walk(STAGE_1_DIRECTORY): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(STAGE_1_DIRECTORY, STAGE_2_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 2...") + + create_stage_2_directory() + + print("Copying data to Stage 2...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_2() + spinner.succeed("Data copied.") + + print("Verifying Stage 2 data...") + is_valid = verify_stage_2_data() + + if is_valid: + print("Data verification successful. Stage 2 data is valid.") + else: + print("Data verification failed. Stage 2 data is not valid.") + +if __name__ == "__main__": + main() + + diff --git a/bin/stage-3/prepare-stage-3.py b/bin/stage-3/prepare-stage-3.py new file mode 100755 index 0000000..1d8ab43 --- /dev/null +++ b/bin/stage-3/prepare-stage-3.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Script Name: prepare-stage-3 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") + +def create_stage_3_directory(): + """Create the stage-3 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_3_DIRECTORY): + print("Existing Stage 3 found, deleting and replacing...") + shutil.rmtree(STAGE_3_DIRECTORY) + os.makedirs(STAGE_3_DIRECTORY) + print("Stage 3 directory created.") + +def copy_data_to_stage_3(): + """Copy data from stage-2 to stage-3.""" + for item in os.listdir(STAGE_2_DIRECTORY): + s = os.path.join(STAGE_2_DIRECTORY, item) + d = os.path.join(STAGE_3_DIRECTORY, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + +def verify_stage_3_data(): + """Verify that the data in stage-3 matches the data in stage-2.""" + for root, dirs, files in os.walk(STAGE_2_DIRECTORY): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(STAGE_2_DIRECTORY, STAGE_3_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-3".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 3...") + + create_stage_3_directory() + + print("Copying data to Stage 3...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_3() + spinner.succeed("Data copied.") + + print("Verifying Stage 3 data...") + is_valid = verify_stage_3_data() + + if is_valid: + print("Data verification successful. Stage 3 data is valid.") + else: + print("Data verification failed. Stage 3 data is not valid.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-4/delete-sponsored-listings.md b/bin/stage-4/delete-sponsored-listings.md new file mode 100644 index 0000000..3571fd4 --- /dev/null +++ b/bin/stage-4/delete-sponsored-listings.md @@ -0,0 +1,181 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'delete-sponsored-listings' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage4_dir = os.path.join(project_root, '.data', 'stage-4') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + # Get the headers and data rows + headers = rows[0] + data_rows = rows[1:] + + # Check each "unknown-#" column for the criteria + for i, header in enumerate(headers): + if header.startswith("unknown-"): + non_empty_cells = [row[i] for row in data_rows if row[i]] + if all(cell == "Sponsored" for cell in non_empty_cells): + os.remove(file_path) + print(f"Deleted file: {file_path}") + return True # File was deleted + + return False # No file was deleted + +def strip_negative_patterns(): + # Process each CSV file in the stage-4 directory + for state_dir in os.listdir(stage4_dir): + state_path = os.path.join(stage4_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Stripping sponsored listings in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + if process_csv_file(file_path): + progress_bar.update(1) + # Check if the directory is now empty + if not os.listdir(county_path): + os.rmdir(county_path) + print(f"Deleted directory: {county_path}") + +if __name__ == '__main__': + strip_negative_patterns() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + diff --git a/bin/stage-4/delete-sponsored-listings.py b/bin/stage-4/delete-sponsored-listings.py new file mode 100755 index 0000000..083a0a1 --- /dev/null +++ b/bin/stage-4/delete-sponsored-listings.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# Script Name: delete-sponsored-listings + +import os +import csv +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4") + +def process_csv_file(file_path): + """Process a CSV file to check and delete sponsored listings.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return False + + headers = rows[0] + data_rows = rows[1:] + + for i, header in enumerate(headers): + if header.startswith("unknown-"): + non_empty_cells = [row[i] for row in data_rows if row[i]] + if all(cell == "Sponsored" for cell in non_empty_cells): + os.remove(file_path) + return True # File was deleted + + return False # No file was deleted + +def delete_sponsored_listings(): + """Delete sponsored listings in all CSV files in the stage 4 directory.""" + figlet = Figlet(font='slant') + script_name = "delete-sponsored-listings".replace("-", " ").title() + print(figlet.renderText(script_name)) + + total_files_deleted = 0 + + for state_dir in os.listdir(STAGE_4_DIRECTORY): + state_path = os.path.join(STAGE_4_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_files_deleted = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + if process_csv_file(file_path): + state_files_deleted += 1 + # Check if the directory is now empty + if not os.listdir(county_path): + os.rmdir(county_path) + total_files_deleted += state_files_deleted + spinner.succeed(f'Finished processing {state_dir}. Files deleted: {state_files_deleted}') + + print(f"Total Files Deleted: {total_files_deleted}") + +if __name__ == "__main__": + delete_sponsored_listings() + diff --git a/bin/stage-4/prepare-stage-4.md b/bin/stage-4/prepare-stage-4.md new file mode 100644 index 0000000..e78a9cb --- /dev/null +++ b/bin/stage-4/prepare-stage-4.md @@ -0,0 +1,116 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +Now please create a prepare-stage-4 version of the script. + +Base the 'prepare-stage-4' script on the 'prepare-stage-2' script below: + + +#!/usr/bin/env python +# Script Name: prepare-stage-2 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def create_stage_2_directory(): + """Create the stage-2 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_2_DIRECTORY): + print("Existing Stage 2 found, deleting and replacing...") + shutil.rmtree(STAGE_2_DIRECTORY) + os.makedirs(STAGE_2_DIRECTORY) + print("Stage 2 directory created.") + +def copy_data_to_stage_2(): + """Copy data from stage-1 to stage-2.""" + for item in os.listdir(STAGE_1_DIRECTORY): + s = os.path.join(STAGE_1_DIRECTORY, item) + d = os.path.join(STAGE_2_DIRECTORY, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + +def verify_stage_2_data(): + """Verify that the data in stage-2 matches the data in stage-1.""" + for root, dirs, files in os.walk(STAGE_1_DIRECTORY): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(STAGE_1_DIRECTORY, STAGE_2_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 2...") + + create_stage_2_directory() + + print("Copying data to Stage 2...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_2() + spinner.succeed("Data copied.") + + print("Verifying Stage 2 data...") + is_valid = verify_stage_2_data() + + if is_valid: + print("Data verification successful. Stage 2 data is valid.") + else: + print("Data verification failed. Stage 2 data is not valid.") + +if __name__ == "__main__": + main() + + diff --git a/bin/stage-4/prepare-stage-4.py b/bin/stage-4/prepare-stage-4.py new file mode 100755 index 0000000..46c9f21 --- /dev/null +++ b/bin/stage-4/prepare-stage-4.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Script Name: prepare-stage-4 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_3_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-3") +STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4") + +def create_stage_4_directory(): + """Create the stage-4 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_4_DIRECTORY): + print("Existing Stage 4 found, deleting and replacing...") + shutil.rmtree(STAGE_4_DIRECTORY) + os.makedirs(STAGE_4_DIRECTORY) + print("Stage 4 directory created.") + +def copy_data_to_stage_4(): + """Copy data from stage-3 to stage-4.""" + for item in os.listdir(STAGE_3_DIRECTORY): + s = os.path.join(STAGE_3_DIRECTORY, item) + d = os.path.join(STAGE_4_DIRECTORY, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + +def verify_stage_4_data(): + """Verify that the data in stage-4 matches the data in stage-3.""" + for root, dirs, files in os.walk(STAGE_3_DIRECTORY): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(STAGE_3_DIRECTORY, STAGE_4_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-4".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 4...") + + create_stage_4_directory() + + print("Copying data to Stage 4...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_4() + spinner.succeed("Data copied.") + + print("Verifying Stage 4 data...") + is_valid = verify_stage_4_data() + + if is_valid: + print("Data verification successful. Stage 4 data is valid.") + else: + print("Data verification failed. Stage 4 data is not valid.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-4/reshift-terms.md b/bin/stage-4/reshift-terms.md new file mode 100644 index 0000000..e2d288e --- /dev/null +++ b/bin/stage-4/reshift-terms.md @@ -0,0 +1,245 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'reshift-terms' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python +# reshift-terms + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage4_dir = os.path.join(project_root, '.data', 'stage-4') +data_dir = os.path.join(project_root, '.data') + +# Read the list of terms for GBP Business Category and GBP Matching Service from the text files +gbp_business_categories_file = os.path.join(data_dir, 'gbp-business-categories.txt') +gbp_matching_services_file = os.path.join(data_dir, 'gbp-matching-services.txt') + +# Helper function to read terms from a file +def read_terms(file_path): + if not os.path.isfile(file_path) or os.path.getsize(file_path) == 0: + print(f"Error: The file {file_path} is missing or empty.") + exit(1) + with open(file_path, 'r') as file: + terms = [line.strip().lower() for line in file if line.strip()] + if not terms: + print(f"Error: The file {file_path} does not contain any valid terms.") + exit(1) + return terms + +gbp_business_categories = read_terms(gbp_business_categories_file) +gbp_matching_services = read_terms(gbp_matching_services_file) + +def process_csv_file(file_path): + # Regular expressions for different data types + location_pattern = re.compile(r'^[\w\s]+, [A-Z]{2}(, United States)?$', re.IGNORECASE) + yib_pattern = re.compile(r'^\d+\+ years in business$', re.IGNORECASE) + review_rating_pattern = re.compile(r'^[1-5]\.\d$', re.IGNORECASE) + review_count_pattern = re.compile(r'^\(\d+\)$', re.IGNORECASE) + + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.DictReader(file) + rows = list(reader) + + if not rows: + return + + headers = reader.fieldnames + + # Iterate over each row + for row in rows: + for header in headers: + if header.startswith('unknown-'): + cell = row[header].strip().lower() + + if location_pattern.match(cell): + if not row['GBP Location']: + row['GBP Location'] = row[header] + row[header] = '' + elif yib_pattern.match(cell): + if not row['YiB']: + row['YiB'] = row[header] + row[header] = '' + elif review_rating_pattern.match(cell): + if not row['GBP Review Rating']: + row['GBP Review Rating'] = row[header] + row[header] = '' + elif review_count_pattern.match(cell): + if not row['GBP Review Count']: + row['GBP Review Count'] = row[header] + row[header] = '' + elif cell in gbp_business_categories: + if not row['GBP Business Category']: + row['GBP Business Category'] = row[header] + row[header] = '' + elif cell in gbp_matching_services: + if not row['GBP Matching Service']: + row['GBP Matching Service'] = row[header] + row[header] = '' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=headers) + writer.writeheader() + writer.writerows(rows) + +def reshift_terms(): + # Print the task name using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText('Reshift Terms')) + + # Process each CSV file in the stage 4 directory + for state_dir in os.listdir(stage4_dir): + state_path = os.path.join(stage4_dir, state_dir) + if os.path.isdir(state_path): + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Shifting Terms in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + reshift_terms() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 +I would like it if this script could include a tally of the terms that were shifted in each state. And then a total tally when complete. + + +# Prompt 3 +It reported that zero terms were moved at all. That seems unlikely to me. To add a little assurance for me to know that nothing indeed was modified at all, can you do some sort of verification between the data before the script ran compared to the data after it was ran? Perhaps creating some kind of checksum before script has been ran, and then after the script is done and comparing the checksum from the before and after snapshot to see if anything was changed. + +A final verification step. + + +# Prompt 4 +Ok that was useful. But it was also spammy. The validation at the end I mean. I would like to only report when a change has been detected. We can be silent about files with no changes detected. diff --git a/bin/stage-4/reshift-terms.py b/bin/stage-4/reshift-terms.py new file mode 100755 index 0000000..deaa0fc --- /dev/null +++ b/bin/stage-4/reshift-terms.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# Script Name: reshift-terms + +import os +import csv +import re +import hashlib +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4") + +# Helper function to read terms from a file +def read_terms(file_path): + if not os.path.isfile(file_path) or os.path.getsize(file_path) == 0: + print(f"Error: The file {file_path} is missing or empty.") + exit(1) + with open(file_path, 'r') as file: + terms = [line.strip().lower() for line in file if line.strip()] + if not terms: + print(f"Error: The file {file_path} does not contain any valid terms.") + exit(1) + return terms + +# Load GBP Business Categories and GBP Matching Services terms +GBP_BUSINESS_CATEGORIES = read_terms(os.path.join(DATA_DIRECTORY, 'gbp-business-categories.txt')) +GBP_MATCHING_SERVICES = read_terms(os.path.join(DATA_DIRECTORY, 'gbp-matching-services.txt')) + +# Patterns for different data types +LOCATION_PATTERN = re.compile(r'^[\w\s]+, [A-Z]{2}(, United States)?$', re.IGNORECASE) +YIB_PATTERN = re.compile(r'^\d+\+ years in business$', re.IGNORECASE) +REVIEW_RATING_PATTERN = re.compile(r'^[1-5]\.\d$', re.IGNORECASE) +REVIEW_COUNT_PATTERN = re.compile(r'^\(\d+\)$', re.IGNORECASE) + +def calculate_checksum(file_path): + """Calculate the MD5 checksum of a file.""" + hash_md5 = hashlib.md5() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + +def process_csv_file(file_path): + """Process a CSV file to reshift terms and return the number of terms shifted.""" + with open(file_path, 'r') as file: + reader = csv.DictReader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = reader.fieldnames + terms_shifted = 0 + + for row in rows: + for header in headers: + if header.startswith('unknown-'): + cell = row[header].strip().lower() + + if LOCATION_PATTERN.match(cell): + if not row['GBP Location']: + row['GBP Location'] = row[header] + row[header] = '' + terms_shifted += 1 + elif YIB_PATTERN.match(cell): + if not row['YiB']: + row['YiB'] = row[header] + row[header] = '' + terms_shifted += 1 + elif REVIEW_RATING_PATTERN.match(cell): + if not row['GBP Review Rating']: + row['GBP Review Rating'] = row[header] + row[header] = '' + terms_shifted += 1 + elif REVIEW_COUNT_PATTERN.match(cell): + if not row['GBP Review Count']: + row['GBP Review Count'] = row[header] + row[header] = '' + terms_shifted += 1 + elif cell in GBP_BUSINESS_CATEGORIES: + if not row['GBP Business Category']: + row['GBP Business Category'] = row[header] + row[header] = '' + terms_shifted += 1 + elif cell in GBP_MATCHING_SERVICES: + if not row['GBP Matching Service']: + row['GBP Matching Service'] = row[header] + row[header] = '' + terms_shifted += 1 + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=headers) + writer.writeheader() + writer.writerows(rows) + + return terms_shifted + +def reshift_terms(): + """Reshift terms in all CSV files in the stage 4 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Reshift Terms')) + + total_terms_shifted = 0 + before_checksums = {} + after_checksums = {} + + for state_dir in os.listdir(STAGE_4_DIRECTORY): + state_path = os.path.join(STAGE_4_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_terms_shifted = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + before_checksums[file_path] = calculate_checksum(file_path) + state_terms_shifted += process_csv_file(file_path) + after_checksums[file_path] = calculate_checksum(file_path) + total_terms_shifted += state_terms_shifted + spinner.succeed(f'Finished processing {state_dir}. Terms shifted: {state_terms_shifted}') + + # Verification step + for file_path in before_checksums: + if before_checksums[file_path] != after_checksums[file_path]: + print(f"Changes detected in: {file_path}") + + print(f"Total Terms Shifted: {total_terms_shifted}") + +if __name__ == "__main__": + reshift_terms() + diff --git a/bin/stage-4/strip-neg-patterns-from-unknown-cols.md b/bin/stage-4/strip-neg-patterns-from-unknown-cols.md new file mode 100644 index 0000000..5eb4915 --- /dev/null +++ b/bin/stage-4/strip-neg-patterns-from-unknown-cols.md @@ -0,0 +1,185 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'strip-neg-patterns-from-unknown-cols' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +import re +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage4_dir = os.path.join(project_root, '.data', 'stage-4') + +# Define the pattern for foreign phone numbers +foreign_phone_pattern = re.compile(r'^\+\d{1,3} \d{3,4} \d{3} \d{3,4}$') + +def process_csv_file(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + # Get the headers and data rows + headers = rows[0] + data_rows = rows[1:] + + # Create a new list to store the modified rows + modified_rows = [headers] + + # Iterate through the data rows and remove foreign phone numbers from "unknown-#" columns + for row in data_rows: + for i, header in enumerate(headers): + if header.startswith("unknown-") and foreign_phone_pattern.match(row[i]): + row[i] = '' # Clear the cell if it contains a foreign phone number + modified_rows.append(row) + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def strip_negative_patterns(): + # Process each CSV file in the stage-4 directory + for state_dir in os.listdir(stage4_dir): + state_path = os.path.join(stage4_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Stripping negative patterns from unknowns in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + strip_negative_patterns() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + diff --git a/bin/stage-4/strip-neg-patterns-from-unknown-cols.py b/bin/stage-4/strip-neg-patterns-from-unknown-cols.py new file mode 100755 index 0000000..4ece38c --- /dev/null +++ b/bin/stage-4/strip-neg-patterns-from-unknown-cols.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Script Name: strip-neg-patterns-from-unknown-cols + +import os +import csv +import re +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4") + +# Define the pattern for foreign phone numbers +foreign_phone_pattern = re.compile(r'^\+\d{1,3} \d{3,4} \d{3} \d{3,4}$') + +def process_csv_file(file_path): + """Process a CSV file to remove negative patterns.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + + modified_rows = [headers] + terms_removed = 0 + + for row in data_rows: + for i, header in enumerate(headers): + if header.startswith("unknown-") and foreign_phone_pattern.match(row[i]): + row[i] = '' # Clear the cell if it contains a foreign phone number + terms_removed += 1 + modified_rows.append(row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + + return terms_removed + +def strip_negative_patterns(): + """Strip negative patterns from all CSV files in the stage 4 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Strip Negative Patterns')) + + total_terms_removed = 0 + + for state_dir in os.listdir(STAGE_4_DIRECTORY): + state_path = os.path.join(STAGE_4_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_terms_removed = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_terms_removed += process_csv_file(file_path) + total_terms_removed += state_terms_removed + spinner.succeed(f'Finished processing {state_dir}. Terms removed: {state_terms_removed}') + + print(f"Total Terms Removed: {total_terms_removed}") + +if __name__ == "__main__": + strip_negative_patterns() + diff --git a/bin/stage-4/strip-neg-terms-from-unknown-cols.md b/bin/stage-4/strip-neg-terms-from-unknown-cols.md new file mode 100644 index 0000000..55fa088 --- /dev/null +++ b/bin/stage-4/strip-neg-terms-from-unknown-cols.md @@ -0,0 +1,318 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'strip-neg-terms-from-unknown-cols' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage4_dir = os.path.join(project_root, '.data', 'stage-4') +data_dir = os.path.join(project_root, '.data') +blacklist_file = os.path.join(data_dir, 'unknown-blacklist.txt') + +def process_csv_file(file_path, blacklist): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + # Get the headers and data rows + headers = rows[0] + data_rows = rows[1:] + + # Create a new list to store the modified rows + modified_rows = [headers] + + # Iterate through the data rows and remove blacklist terms from "unknown-#" columns + for row in data_rows: + for i, header in enumerate(headers): + if header.startswith("unknown-") and row[i] in blacklist: + row[i] = '' # Clear the cell if it contains a blacklist term + modified_rows.append(row) + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def strip_negative_terms(): + # Read the blacklist terms + if not os.path.isfile(blacklist_file): + print("Error: The unknown-blacklist.txt file is missing.") + return + + with open(blacklist_file, 'r') as file: + blacklist = [line.strip() for line in file if line.strip()] + + # Process each CSV file in the stage-4 directory + for state_dir in os.listdir(stage4_dir): + state_path = os.path.join(stage4_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + progress_bar = tqdm(csv_files, desc=f"Stripping neg terms from unknown columns in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file(file_path, blacklist) + progress_bar.update(1) + +if __name__ == '__main__': + strip_negative_terms() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I wish to adjust the 'strip-neg-terms-from-unknown-cols' script: + + + +This script needs to have the matching it performs be done in a case insensitive manner. Capitalization needs to be ignored. diff --git a/bin/stage-4/strip-neg-terms-from-unknown-cols.py b/bin/stage-4/strip-neg-terms-from-unknown-cols.py new file mode 100755 index 0000000..9818638 --- /dev/null +++ b/bin/stage-4/strip-neg-terms-from-unknown-cols.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Script Name: strip-neg-terms-from-unknown-cols + +import os +import csv +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4") +BLACKLIST_FILE = os.path.join(DATA_DIRECTORY, 'unknown-blacklist.txt') + +def read_blacklist(file_path): + """Read the blacklist terms from a file.""" + if not os.path.isfile(file_path): + print("Error: The unknown-blacklist.txt file is missing.") + return [] + + with open(file_path, 'r') as file: + blacklist = [line.strip().lower() for line in file if line.strip()] + return blacklist + +def process_csv_file(file_path, blacklist): + """Process a CSV file to remove blacklist terms from unknown columns.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + modified_rows = [headers] + terms_removed = 0 + + for row in data_rows: + for i, header in enumerate(headers): + if header.startswith("unknown-") and row[i].strip().lower() in blacklist: + row[i] = '' # Clear the cell if it contains a blacklist term + terms_removed += 1 + modified_rows.append(row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + + return terms_removed + +def strip_negative_terms(): + """Strip negative terms from all CSV files in the stage 4 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Strip Neg Terms')) + + blacklist = read_blacklist(BLACKLIST_FILE) + if not blacklist: + return + + total_terms_removed = 0 + + for state_dir in os.listdir(STAGE_4_DIRECTORY): + state_path = os.path.join(STAGE_4_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_terms_removed = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_terms_removed += process_csv_file(file_path, blacklist) + total_terms_removed += state_terms_removed + spinner.succeed(f'Finished processing {state_dir}. Terms removed: {state_terms_removed}') + + print(f"Total Terms Removed: {total_terms_removed}") + +if __name__ == "__main__": + strip_negative_terms() + diff --git a/bin/stage-5/prepare-stage-5.md b/bin/stage-5/prepare-stage-5.md new file mode 100644 index 0000000..961acde --- /dev/null +++ b/bin/stage-5/prepare-stage-5.md @@ -0,0 +1,115 @@ + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +Now please create a prepare-stage-5 version of the script. + +Base the 'prepare-stage-5' script on the 'prepare-stage-2' script below: + + +#!/usr/bin/env python +# Script Name: prepare-stage-2 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_1_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-1") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def create_stage_2_directory(): + """Create the stage-2 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_2_DIRECTORY): + print("Existing Stage 2 found, deleting and replacing...") + shutil.rmtree(STAGE_2_DIRECTORY) + os.makedirs(STAGE_2_DIRECTORY) + print("Stage 2 directory created.") + +def copy_data_to_stage_2(): + """Copy data from stage-1 to stage-2.""" + for item in os.listdir(STAGE_1_DIRECTORY): + s = os.path.join(STAGE_1_DIRECTORY, item) + d = os.path.join(STAGE_2_DIRECTORY, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + +def verify_stage_2_data(): + """Verify that the data in stage-2 matches the data in stage-1.""" + for root, dirs, files in os.walk(STAGE_1_DIRECTORY): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(STAGE_1_DIRECTORY, STAGE_2_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 2...") + + create_stage_2_directory() + + print("Copying data to Stage 2...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_2() + spinner.succeed("Data copied.") + + print("Verifying Stage 2 data...") + is_valid = verify_stage_2_data() + + if is_valid: + print("Data verification successful. Stage 2 data is valid.") + else: + print("Data verification failed. Stage 2 data is not valid.") + +if __name__ == "__main__": + main() + + diff --git a/bin/stage-5/prepare-stage-5.py b/bin/stage-5/prepare-stage-5.py new file mode 100755 index 0000000..edfb2fc --- /dev/null +++ b/bin/stage-5/prepare-stage-5.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Script Name: prepare-stage-5 + +import os +import shutil +import halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_4_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-4") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") + +def create_stage_5_directory(): + """Create the stage-5 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_5_DIRECTORY): + print("Existing Stage 5 found, deleting and replacing...") + shutil.rmtree(STAGE_5_DIRECTORY) + os.makedirs(STAGE_5_DIRECTORY) + print("Stage 5 directory created.") + +def copy_data_to_stage_5(): + """Copy data from stage-4 to stage-5.""" + for item in os.listdir(STAGE_4_DIRECTORY): + s = os.path.join(STAGE_4_DIRECTORY, item) + d = os.path.join(STAGE_5_DIRECTORY, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + +def verify_stage_5_data(): + """Verify that the data in stage-5 matches the data in stage-4.""" + for root, dirs, files in os.walk(STAGE_4_DIRECTORY): + for file in files: + src_file = os.path.join(root, file) + dest_file = src_file.replace(STAGE_4_DIRECTORY, STAGE_5_DIRECTORY) + if not os.path.exists(dest_file): + return False + return True + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-5".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 5...") + + create_stage_5_directory() + + print("Copying data to Stage 5...") + spinner = halo.Halo(text='Copying data', spinner='dots') + spinner.start() + copy_data_to_stage_5() + spinner.succeed("Data copied.") + + print("Verifying Stage 5 data...") + is_valid = verify_stage_5_data() + + if is_valid: + print("Data verification successful. Stage 5 data is valid.") + else: + print("Data verification failed. Stage 5 data is not valid.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-5/retrieve-root-domain.md b/bin/stage-5/retrieve-root-domain.md new file mode 100644 index 0000000..31d552d --- /dev/null +++ b/bin/stage-5/retrieve-root-domain.md @@ -0,0 +1,276 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'retrieve-root-domain' to be in accordance with the above, and to adjust the reporting style now: + + + + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I need to modify the 'retrieve-root-domain' script: + + +#!/usr/bin/env python +# Script Name: retrieve-root-domain + +import os +import csv +import re +from urllib.parse import urlparse +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") + +def extract_root_domain(url): + """Extract the root domain from a URL.""" + parsed_url = urlparse(url) + domain = parsed_url.netloc + if domain.startswith("www."): + domain = domain[4:] + return domain + +def process_csv_file(file_path): + """Process a CSV file to extract root domains.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + + # Ensure the necessary columns exist + if 'GBP Business Website' not in headers: + return 0 + + # Create new column for root domains if it doesn't exist + if 'Root Domain' not in headers: + headers.append('Root Domain') + + website_index = headers.index('GBP Business Website') + root_domain_index = headers.index('Root Domain') + + changes = 0 + for row in data_rows: + while len(row) < len(headers): + row.append('') + + website = row[website_index] + + # Extract root domain + if website: + root_domain = extract_root_domain(website) + if root_domain != row[root_domain_index]: + row[root_domain_index] = root_domain + changes += 1 + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + + return changes + +def retrieve_root_domain(): + """Retrieve root domains from all CSV files in the stage 5 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Retrieve Root Domain')) + + total_changes = 0 + + for state_dir in os.listdir(STAGE_5_DIRECTORY): + state_path = os.path.join(STAGE_5_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_changes = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_changes += process_csv_file(file_path) + total_changes += state_changes + spinner.succeed(f'Finished processing {state_dir}. Changes made: {state_changes}') + + print(f"Total Changes Made: {total_changes}") + +if __name__ == "__main__": + retrieve_root_domain() + + +I need the script to blacklist some domains from being added to the "Root Domain" column: + + +facebook.com +instagram.com +twitter.com +x.com +linkedin.com + + + diff --git a/bin/stage-5/retrieve-root-domain.py b/bin/stage-5/retrieve-root-domain.py new file mode 100755 index 0000000..e498f76 --- /dev/null +++ b/bin/stage-5/retrieve-root-domain.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# Script Name: retrieve-root-domain + +import os +import csv +from urllib.parse import urlparse +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") + +# List of blacklisted root domains +BLACKLISTED_DOMAINS = { + "facebook.com", + "instagram.com", + "twitter.com", + "x.com", + "linkedin.com" +} + +def extract_root_domain(url): + """Extract the root domain from a URL.""" + parsed_url = urlparse(url) + domain = parsed_url.netloc + if domain.startswith("www."): + domain = domain[4:] + return domain + +def is_blacklisted(domain): + """Check if the domain or its subdomain is blacklisted.""" + for blacklisted in BLACKLISTED_DOMAINS: + if domain == blacklisted or domain.endswith("." + blacklisted): + return True + return False + +def process_csv_file(file_path): + """Process a CSV file to extract root domains.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + + # Ensure the necessary columns exist + if 'GBP Business Website' not in headers: + return 0 + + # Create new column for root domains if it doesn't exist + if 'Root Domain' not in headers: + headers.append('Root Domain') + + website_index = headers.index('GBP Business Website') + root_domain_index = headers.index('Root Domain') + + changes = 0 + for row in data_rows: + while len(row) < len(headers): + row.append('') + + website = row[website_index] + + # Extract root domain + if website: + root_domain = extract_root_domain(website) + if not is_blacklisted(root_domain) and root_domain != row[root_domain_index]: + row[root_domain_index] = root_domain + changes += 1 + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + + return changes + +def retrieve_root_domain(): + """Retrieve root domains from all CSV files in the stage 5 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Retrieve Root Domain')) + + total_changes = 0 + + for state_dir in os.listdir(STAGE_5_DIRECTORY): + state_path = os.path.join(STAGE_5_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_changes = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_changes += process_csv_file(file_path) + total_changes += state_changes + spinner.succeed(f'Finished processing {state_dir}. Changes made: {state_changes}') + + print(f"Total Changes Made: {total_changes}") + +if __name__ == "__main__": + retrieve_root_domain() + diff --git a/bin/stage-5/standardize-location-data.md b/bin/stage-5/standardize-location-data.md new file mode 100644 index 0000000..e6ca5ab --- /dev/null +++ b/bin/stage-5/standardize-location-data.md @@ -0,0 +1,221 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'standardize-location-data' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage5_dir = os.path.join(project_root, '.data', 'stage-5') + +def process_csv_file_standardize_location(file_path): + # Read the CSV file + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + # Ensure the necessary columns exist + if 'GBP Location' not in headers: + return + + # Create new columns for municipality and state + if 'GBP Location Municipality' not in headers: + headers.append('GBP Location Municipality') + if 'GBP Location State' not in headers: + headers.append('GBP Location State') + + municipality_index = headers.index('GBP Location Municipality') + state_index = headers.index('GBP Location State') + location_index = headers.index('GBP Location') + + for row in data_rows: + while len(row) < len(headers): + row.append('') + + location = row[location_index] + + # Standardize location data by removing ", United States" + if ', United States' in location: + location = location.replace(', United States', '').strip() + + # Split location into municipality and state + if ', ' in location: + parts = location.split(', ') + if len(parts) == 2: + municipality, state = parts + else: + municipality = ', '.join(parts[:-1]) + state = parts[-1] + row[municipality_index] = municipality + row[state_index] = state + row[location_index] = '' + else: + row[municipality_index] = '' + row[state_index] = '' + + # Check if the GBP Location column is empty and act accordingly + if all(row[location_index] == '' for row in data_rows): + headers.pop(location_index) + for row in data_rows: + row.pop(location_index) + else: + # Rename the GBP Location column to unknown-# + unknown_count = sum(1 for header in headers if header.startswith('unknown-')) + headers[location_index] = f'unknown-{unknown_count + 1}' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + +def standardize_location_data(): + # Process for standardizing location data + for state_dir in os.listdir(stage5_dir): + state_path = os.path.join(stage5_dir, state_dir) + if os.path.isdir(state_path): + # Print the state name in big print using Figlet + figlet = Figlet(font='slant') + print(figlet.renderText(state_dir)) + + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + + progress_bar = tqdm(csv_files, desc=f"Standardizing Location data in: {state_dir}, {county_dir}", + unit="file", bar_format="{l_bar}{bar:10}| {n_fmt}/{total_fmt}") + for file in progress_bar: + file_path = os.path.join(county_path, file) + process_csv_file_standardize_location(file_path) + progress_bar.update(1) + +if __name__ == '__main__': + standardize_location_data() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + diff --git a/bin/stage-5/standardize-location-data.py b/bin/stage-5/standardize-location-data.py new file mode 100755 index 0000000..385b69e --- /dev/null +++ b/bin/stage-5/standardize-location-data.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# Script Name: standardize-location-data + +import os +import csv +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") + +def process_csv_file_standardize_location(file_path): + """Process a CSV file to standardize location data.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + + # Ensure the necessary columns exist + if 'GBP Location' not in headers: + return 0 + + # Create new columns for municipality and state if they don't exist + if 'GBP Location Municipality' not in headers: + headers.append('GBP Location Municipality') + if 'GBP Location State' not in headers: + headers.append('GBP Location State') + + municipality_index = headers.index('GBP Location Municipality') + state_index = headers.index('GBP Location State') + location_index = headers.index('GBP Location') + + changes = 0 + for row in data_rows: + while len(row) < len(headers): + row.append('') + + location = row[location_index] + + # Standardize location data by removing ", United States" + if ', United States' in location: + location = location.replace(', United States', '').strip() + + # Split location into municipality and state + if ', ' in location: + parts = location.split(', ') + if len(parts) == 2: + municipality, state = parts + else: + municipality = ', '.join(parts[:-1]) + state = parts[-1] + row[municipality_index] = municipality + row[state_index] = state + row[location_index] = '' + changes += 1 + else: + row[municipality_index] = '' + row[state_index] = '' + + # Write the modified rows back to the CSV file + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + + return changes + +def standardize_location_data(): + """Standardize location data in all CSV files in the stage 5 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Standardize Location Data')) + + total_changes = 0 + + for state_dir in os.listdir(STAGE_5_DIRECTORY): + state_path = os.path.join(STAGE_5_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_changes = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_changes += process_csv_file_standardize_location(file_path) + total_changes += state_changes + spinner.succeed(f'Finished processing {state_dir}. Changes made: {state_changes}') + + print(f"Total Changes Made: {total_changes}") + +if __name__ == "__main__": + standardize_location_data() + diff --git a/bin/stage-5/strip-brackets-from-review-count.md b/bin/stage-5/strip-brackets-from-review-count.md new file mode 100644 index 0000000..212f681 --- /dev/null +++ b/bin/stage-5/strip-brackets-from-review-count.md @@ -0,0 +1,290 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'strip-brackets-from-review-count' to be in accordance with the above, and to adjust the reporting style now: + + + + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I ran into a problem with the 'strip-brackets-from-reviews-count' script: + + +#!/usr/bin/env python +# Script Name: strip-brackets-from-review-count + +import os +import csv +import re +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") + +def process_csv_file_strip_brackets(file_path): + """Strip brackets from review count in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + review_count_index = None + + if 'GBP Review Count' in headers: + review_count_index = headers.index('GBP Review Count') + else: + return 0 + + pattern = re.compile(r'^\((\d+)\)$') + changes_made = 0 + + for row in data_rows: + if review_count_index is not None: + match = pattern.match(row[review_count_index]) + if match: + row[review_count_index] = match.group(1) + changes_made += 1 + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + + return changes_made + +def strip_brackets_from_review_count(): + """Strip brackets from review count in all CSV files in the stage 5 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Strip Brackets')) + + total_changes_made = 0 + + for state_dir in os.listdir(STAGE_5_DIRECTORY): + state_path = os.path.join(STAGE_5_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_changes_made = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_changes_made += process_csv_file_strip_brackets(file_path) + total_changes_made += state_changes_made + spinner.succeed(f'Finished processing {state_dir}. Changes made: {state_changes_made}') + + print(f"Total Changes Made: {total_changes_made}") + +if __name__ == "__main__": + strip_brackets_from_review_count() + + +I discovered that not every row is having the brackets properly stripped from the numbers in the "GBP Review Count" column as we intended. + +Most entries did have the brackets removed, but some entries kept them. + +This may have something to do with the number inside these entries which got missed all seem to have a comma in them as a thousands seperator. + +Here is some sample data which contains entries in the "GBP Review Count" column that retain their brackets, which this script is supposed to remove. + + +Business Name,Business UUID,Location UUID,GBP Review Rating,GBP Review Count,YiB,GBP Business Category,GBP Matching Service,GBP Location Municipality,GBP Location State,Location from GBP Business Name,GBP Business Phone,GBP Business Website,Root Domain +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,5620e68e-249c-4913-a4cc-16fe4b06e8cc,4.3,"(1,821)",,Home Improvement Store,Repair,Cary,NC,,+1 919-380-3210,https://www.lowes.com/store/NC-Cary/1835?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-1835-_-na-_-0-_-0&y_source=1_MTE4NDQxMy03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,1c71993f-3f20-44da-966d-12b31fc67f72,4.2,"(1,310)",,Home Improvement Store,Services,Matthews,NC,,+1 704-321-7170,https://www.lowes.com/store/NC-Matthews/1124?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-1124-_-na-_-0-_-0&y_source=1_MTE4MzgyMy03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,711c10bd-a4eb-45f9-ac28-eb1a64b1469c,4.2,"(2,480)",,Home Improvement Store,Repair,High Point,NC,,+1 336-889-8600,https://www.lowes.com/store/NC-High-Point/0459?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-459-_-na-_-0-_-0&y_source=1_MTE4MzE4MC03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,246fec7f-c54b-43e7-85f3-bca0c94fd399,4.1,"(2,044)",,Home Improvement Store,Repair,Winterville,NC,,+1 252-355-5211,https://www.lowes.com/store/NC-Winterville/0598?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-598-_-na-_-0-_-0&y_source=1_MTE4MjkyOC03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,a548b3e7-8135-4e1c-9b88-b8766b814d98,4.2,"(2,333)",,Home Improvement Store,Services,Gastonia,NC,,+1 704-865-6767,https://www.lowes.com/store/NC-Gastonia/0457?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-457-_-na-_-0-_-0&y_source=1_MTE4Mzg0MC03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,2605004b-89db-4227-a63c-fa5ba8e6946d,4.0,"(1,626)",,Home Improvement Store,Repair,Elizabeth City,NC,,+1 252-331-6160,https://www.lowes.com/store/NC-Elizabeth-City/1713?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-1713-_-na-_-0-_-0&y_source=1_MTE4MjkyNy03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,f3bc4f4e-d03e-421b-8bdf-745b7681ce89,4.2,"(1,766)",,Home Improvement Store,Repair,Greensboro,NC,,+1 336-541-1890,https://www.lowes.com/store/NC-Greensboro/2771?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-2771-_-na-_-0-_-0&y_source=1_MTE4MzE3Mi03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,60191dc1-b471-4137-b3ad-8fb5120af969,4.0,"(2,240)",,Home Improvement Store,Repair,Raleigh,NC,,+1 919-850-9300,https://www.lowes.com/store/NC-Raleigh/0444?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-444-_-na-_-0-_-0&y_source=1_MTE4NDQyMi03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,acfa3d36-094a-4eaa-a090-298709aea066,4.1,"(1,887)",,Home Improvement Store,Repair,Lincolnton,NC,,+1 704-748-9335,https://www.lowes.com/store/NC-Lincolnton/0700?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-700-_-na-_-0-_-0&y_source=1_MTE4MzgzNy03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,4c8bfc16-b7f4-45fb-ab50-d6970589599a,4.2,"(1,017)",,Home Improvement Store,Repair,Waxhaw,NC,,+1 704-843-8520,https://www.lowes.com/store/NC-Waxhaw/2638?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-2638-_-na-_-0-_-0&y_source=1_MTE4MzgzOS03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,e5eb1412-0dea-4f71-acec-b94ec8812ad3,4.0,"(1,483)",,Home Improvement Store,Repair,Henderson,NC,,+1 252-436-0050,https://www.lowes.com/store/NC-Henderson/0738?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-738-_-na-_-0-_-0&y_source=1_MTE4MjkyOS03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,93b7282b-a6ed-45fd-8716-753f51824c60,3.8,"(1,897)",,Home Improvement Store,Repair,Hendersonville,NC,,+1 828-696-4900,https://www.lowes.com/store/NC-Hendersonville/0031?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-31-_-na-_-0-_-0&y_source=1_MTE4NDE3Ni03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,6cf64b13-1986-401e-bdcc-7df4529aa2dd,3.8,"(1,119)",,Home Improvement Store,Repair,Sylva,NC,,+1 828-586-1170,https://www.lowes.com/store/NC-Sylva/2257?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-2257-_-na-_-0-_-0&y_source=1_MTE4NDE3My03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,d3909855-229c-4b0f-be0e-1a7cf82c73d4,4.1,"(1,959)",,Home Improvement Store,Repair,Charlotte,NC,,+1 704-597-2000,https://www.lowes.com/store/NC-Charlotte/0408?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-408-_-na-_-0-_-0&y_source=1_MTE4MzgzMS03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,a3d406a0-bba7-4772-81e4-675d65e33dee,4.1,"(2,185)",,Home Improvement Store,Repair,Charlotte,NC,,+1 704-501-4420,https://www.lowes.com/store/NC-Charlotte/2352?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-2352-_-na-_-0-_-0&y_source=1_MTE4MzgyNy03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,88097b9c-15cc-4a7f-a740-01200ac4312b,4.1,"(1,956)",,Home Improvement Store,Repair,Charlotte,NC,,+1 704-335-5021,https://www.lowes.com/store/NC-Charlotte/2348?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-2348-_-na-_-0-_-0&y_source=1_MTE4MzgyNC03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,fbd16ea4-6a69-482a-a08d-2e3682bcb1e0,4.2,"(1,140)",,Home Improvement Store,Companies,Elkin,NC,,+1 336-526-6550,https://www.lowes.com/store/NC-Elkin/1653?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-1653-_-na-_-0-_-0&y_source=1_MTE4MzE3MS03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,43c1ea1e-e582-4b10-a90a-dfe298cc10df,4.0,"(1,367)",,Home Improvement Store,Repair,Wilkesboro,NC,,+1 336-838-1500,https://www.lowes.com/store/NC-Wilkesboro/0701?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-701-_-na-_-0-_-0&y_source=1_MTE4MzE3OC03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,1c62fedf-41ea-4641-800b-81b09ae4aeb2,4.0,"(1,923)",,Home Improvement Store,Concrete,Munhall,PA,,+1 412-461-8002,https://www.lowes.com/store/PA-Munhall/0780?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-780-_-na-_-0-_-0&y_source=1_MTE4MzI5NS03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,c163adb0-bf5a-4dc8-af63-a3d69bd75017,4.2,"(2,145)",,Home Improvement Store,Construction,Reading,PA,,+1 610-939-0100,https://www.lowes.com/store/PA-Reading/0279?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-279-_-na-_-0-_-0&y_source=1_MTE4MzY4OS03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,606eb01a-4ad4-4125-bce1-229e3fca7d09,4.3,"(1,031)",,Home Improvement Store,Repair,Central,SC,,+1 864-722-6001,https://www.lowes.com/store/SC-Central/3071?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-3071-_-na-_-0-_-0&y_source=1_MTE4NDI3Ny03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,6fbe9ee0-f628-4b83-ab0f-e1d93d6fc14e,4.1,"(1,638)",,Home Improvement Store,Repair,Columbia,SC,,+1 803-476-1320,https://www.lowes.com/store/SC-Columbia/3026?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-3026-_-na-_-0-_-0&y_source=1_MTE4NDA2Mi03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,97c81d30-528e-4d32-8cc8-d8cd2146b9c8,4.2,"(1,732)",,Home Improvement Store,Repair,Harrisonburg,VA,,+1 540-433-7660,https://www.lowes.com/store/VA-Harrisonburg/0509?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-509-_-na-_-0-_-0&y_source=1_MTE4MzUzOS03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,c44ff9eb-7746-484c-b63f-a89f4acf05e9,4.2,"(2,266)",,Home Improvement Store,Companies,Jacksonville,FL,,+1 904-855-8088,https://www.lowes.com/store/FL-Jacksonville/0503?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-503-_-na-_-0-_-0&y_source=1_MTE4NDMyMC03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com +Lowe's Home Improvement,f0cc368c-0126-46d1-a64b-3ca90b33f528,4180f074-7218-44d3-8b39-a4c847007261,4.0,"(3,050)",,Home Improvement Store,Repair,Norfolk,VA,,+1 757-455-5205,https://www.lowes.com/store/VA-Norfolk/1065?cm_mmc=lod-_-c-_-lcl-_-awr-_-yxt-_-go-_-1065-_-na-_-0-_-0&y_source=1_MTE4Mzk2OC03MTUtbG9jYXRpb24ud2Vic2l0ZQ%3D%3D,lowes.com + + +I would like this script to be adjusted slightly to also remove the thousands separator from entries in the "GBP Review Count" column as well as the brackets. + +It may be be simpler for the script if it processes for the thousands separator first. And then after that remove the brackets. + +I would also like this script to as a final loop, run a check on the "GBP Review Count" column looking for any variety of ) or ( style brackets. I want the script to double check that they are gone. + +If any are found on the verification check, then I'd like the script to attempt to remove them again, but using a different method. Then doing one more verification check. And if it fails that time, exit out of the script with an error. diff --git a/bin/stage-5/strip-brackets-from-review-count.py b/bin/stage-5/strip-brackets-from-review-count.py new file mode 100755 index 0000000..574cebd --- /dev/null +++ b/bin/stage-5/strip-brackets-from-review-count.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# Script Name: strip-brackets-from-review-count + +import os +import csv +import re +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") + +def process_csv_file_strip_brackets(file_path): + """Strip brackets and thousands separator from review count in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return 0 + + headers = rows[0] + data_rows = rows[1:] + review_count_index = None + + if 'GBP Review Count' in headers: + review_count_index = headers.index('GBP Review Count') + else: + return 0 + + pattern_brackets = re.compile(r'^\((\d{1,3}(?:,\d{3})*)\)$') + changes_made = 0 + + # First, remove the thousands separators + for row in data_rows: + if review_count_index is not None: + match = pattern_brackets.match(row[review_count_index]) + if match: + # Remove thousands separators + review_count_no_commas = match.group(1).replace(',', '') + row[review_count_index] = review_count_no_commas + changes_made += 1 + + # Second, remove any remaining brackets + for row in data_rows: + if review_count_index is not None: + row[review_count_index] = re.sub(r'[\(\)]', '', row[review_count_index]) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(data_rows) + + return changes_made + +def verify_no_brackets(file_path): + """Verify that there are no brackets remaining in the GBP Review Count column.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return True + + headers = rows[0] + data_rows = rows[1:] + review_count_index = None + + if 'GBP Review Count' in headers: + review_count_index = headers.index('GBP Review Count') + else: + return True + + for row in data_rows: + if review_count_index is not None and ( '(' in row[review_count_index] or ')' in row[review_count_index] ): + return False + + return True + +def strip_brackets_from_review_count(): + """Strip brackets from review count in all CSV files in the stage 5 directory and tally the results.""" + figlet = Figlet(font='slant') + print(figlet.renderText('Strip Brackets')) + + total_changes_made = 0 + + for state_dir in os.listdir(STAGE_5_DIRECTORY): + state_path = os.path.join(STAGE_5_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + state_changes_made = 0 + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + state_changes_made += process_csv_file_strip_brackets(file_path) + + if not verify_no_brackets(file_path): + process_csv_file_strip_brackets(file_path) + if not verify_no_brackets(file_path): + spinner.fail(f"Failed to remove brackets from {file_path}") + return + + total_changes_made += state_changes_made + spinner.succeed(f'Finished processing {state_dir}. Changes made: {state_changes_made}') + + print(f"Total Changes Made: {total_changes_made}") + +if __name__ == "__main__": + strip_brackets_from_review_count() + diff --git a/bin/stage-6/business-name-strip-structures.md b/bin/stage-6/business-name-strip-structures.md new file mode 100644 index 0000000..1d3c007 --- /dev/null +++ b/bin/stage-6/business-name-strip-structures.md @@ -0,0 +1,107 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I wish to make a new script which will have as input the csv file: "06-initial-deduplication.csv". And as output, the csv file: "07-business-name-structures-stripped.csv". + +This script should be called 'business-name-strip-structures'. + +It will reside in [Stage 6 Binaries] + +The file needs to strip various acronyms from Business Names in the "Business Name" column. + +The script needs to parse the Business Names for Business Names which contain various acronyms, such as "LLC, "Inc", and "Co". + +When a matching string is found, that string needs to be removed. + +If the matched string is the end of the entire string found in the cell, then be certain to remove trailing spaces, and commas which may have preceeded the matched string as well as the matched string we are removing. + +If the matched string has more unmatched strings occurring after it in the cell, then the script needs to be sure there is a single space between the portion of the cell string which came before the matched string, and the portion of the cell string which came after the matched string. Essentially if we are removing the center part of a longer string, we do not want to mash two words together by removing this matched string. We want those two words to maintain readable spacing. (1 space apart) + +All these strings we are searching for should only match if they are found on their own as a complete word. They must not match if they are found inside of another word, or mashed up against another word. + +Some of these terms may be the last component in the entire cell string, meaning there will be no spaces after the matching string sometimes. + +They will never be the start of the entire cell string. They will always be in the middle or at the end. + +Most of the time they will have a space before the string, however sometimes the string we are searching for may have punctuation before it instead of a space. Such as a comma or period. + +Also, we are not looking for case sensitive matching. We can ignore case when searching for matches. + +If a string in the list we are searching to match against has puntuation in it we need to be careful to match against that punctuation as well. Periods after "Co." count for example as something we may want to match against. + +Here are the strings we searching for to remove from the Business Name entries: + + +llc +llc. +l.l.c. +co +co. +inc +inc. +corp +corp. + + + +# Prompt 2 +The script also needs to check and make sure 07-business-name-structures-stripped.csv doesn't already exist before it begins. If it does exist, it needs to be deleted before the script continues. + + +# Prompt 3 +This is close. It is almost working. + +However, some punctuation is being left behind. + +Look at the following example outputs: + + +Original: ​B & S Paving and Construction, Inc., Cleaned: ​B & S Paving and Construction, . +Original: Zorro Concrete LLC., Cleaned: Zorro Concrete . +Original: Young's Concrete Pumping, Inc., Cleaned: Young's Concrete Pumping, . + + +In some instances commas and periods are being left behind. + +For example "B & S Paving and Construction, Inc." became: "B & S Paving and Construction, ." but should have become: "B & S Paving and Construction" + +And "Zorro Concrete LLC." became: "Zorro Concrete ." but should have became: "Zorro Concrete" + +And "Young's Concrete Pumping, Inc." became: "Young's Concrete Pumping, ." but should have became: "Young's Concrete Pumping" + +Please adjust the script to check for clean punctuation in the area the matched string was removed from. We want to clean up left over punctuation as a part of this process. diff --git a/bin/stage-6/business-name-strip-structures.py b/bin/stage-6/business-name-strip-structures.py new file mode 100755 index 0000000..b93fdea --- /dev/null +++ b/bin/stage-6/business-name-strip-structures.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Script Name: business-name-strip-structures + +import os +import csv +import re +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '06-initial-deduplication.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '07-business-name-structures-stripped.csv') + +# List of terms to remove +terms_to_remove = [ + r'\bllc\b', r'\bllc\.\b', r'\bl\.l\.c\.\b', + r'\bco\b', r'\bco\.\b', r'\binc\b', r'\binc\.\b', + r'\bcorp\b', r'\bcorp\.\b' +] + +def clean_business_name(business_name): + """Remove specified terms from the business name and clean up punctuation.""" + for term in terms_to_remove: + business_name = re.sub(r'\s*[,.]*\s*' + term + r'[,.]*\s*', ' ', business_name, flags=re.IGNORECASE) + business_name = re.sub(r'\s+', ' ', business_name).strip() + business_name = re.sub(r'[,.]\s*$', '', business_name) + return business_name + +def strip_structures_from_business_names(): + """Strip specified structures from business names.""" + if not os.path.exists(INPUT_CSV): + print(f"Error: The input file {INPUT_CSV} does not exist.") + return + + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + + if 'Business Name' not in headers: + print("Error: 'Business Name' column is missing.") + return + + rows = list(reader) + + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + row['Business Name'] = clean_business_name(row.get('Business Name', '').strip()) + writer.writerow(row) + +def main(): + figlet = Figlet(font='slant') + script_name = "business-name-strip-structures".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Stripping business name structures', spinner='dots') + spinner.start() + strip_structures_from_business_names() + spinner.succeed('Business name structures stripped.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/business-names-with-locations.md b/bin/stage-6/business-names-with-locations.md new file mode 100644 index 0000000..2cea3d2 --- /dev/null +++ b/bin/stage-6/business-names-with-locations.md @@ -0,0 +1,99 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I wish to create a new script named 'business-names-with-locations', which will have as input the csv file "07-business-name-structures-stripped.csv", and output will have the csv file "08-business-names-with-locations.csv". These csv files will be located in [Stage 6]. + +Script Overview: The overview for this script is that we are attempting to process the "Business Name" field of the input CSV looking for Business Names which have locations appended to the end of them. Such as the examples in the following example CSV data: + + +Business Name,GBP Review Rating,GBP Review Count,GBP Business Category,YiB,GBP Matching Service,GBP Business Phone,GBP Business Website,GBP Location Municipality,GBP Location State,Root Domain +ultimate concrete resurfacing of scotch valley,3.8,12,Concrete Contractor,5+ years in business,,+1 814-931-7100,http://www.concretedesigns4you.com/,Hollidaysburg,PA,concretedesigns4you.com +Yellow Dawg Asphalt of The Triad,5,12,Asphalt Contractor,7+ years in business,Driveway Sealcoating,+1 336-272-5757,https://www.yellowdawgasphalt.com/,Greensboro,NC,yellowdawgasphalt.com +Window Ninjas of Raleigh,5.0,129,Window Cleaning Service,7+ years in business,"Concrete, Brick, And Stone Washing",+1 919-867-6276,https://windowninjas.com/north-carolina/raleigh-durham/,Garner,NC,windowninjas.com +Window Ninjas of Greenville/Spartanburg,4.9,201,Window Cleaning Service,5+ years in business,"Concrete, Brick, And Stone Washing",+1 864-558-7758,https://windowninjas.com/south-carolina/greenville-spartanburg/,Greenville,SC,windowninjas.com + + +In the above example data we have the following example Business Names: + + +ultimate concrete resurfacing of scotch valley +Yellow Dawg Asphalt of The Triad +Window Ninjas of Raleigh +Window Ninjas of Greenville/Spartanburg + + +This is an example of a Business Name which follows a formula of "BUSINESS-NAME of LOCATION". + +I want to focus on the " of " portion of the string here. + +I want the first pass that this script takes when parsing the data to be for the string " of ". When we see that string, including spaces, we can assume that what comes before it is the Business Name, and what comes after it is the Location. + +Handling these three string components when a " of " match occurs is what I want to deal with right now in this script. + +First, the script should create a new column in the destination CSV file named "Location from GBP Business Name". + +That "Location from GBP Business Name" column is where we want to move the Location to when we find a " of " match. + +After the "Location from GBP Business Name" column is created we can begin processing the "Business Name" field row by row looking for matches against " of ". + +When a match with " of " happens, the script should move whatever is to the right of the " of " into the "Location from GBP Business Name" column for that row. This move process should not allow for leading spaces to be added to the "Location from GBP Business Name" column during this move though. We do not want strings that begin with spaces added to this column. + +Then the script should remove the " of " portion of the string from the matching cell to clean it up a bit. + +Finally, the script should check for punctuation or symbols at the end of the matching cell, such as commas, hyphens, periods, etc, and delete them if found. + +If you understand and have no questions you need to ask first, begin writing the python script for me now. + + +# Prompt 2 +I got the following error when running the script: + + +⠙ Processing business names with locationsTraceback (most recent call last): + File "/home/ld/mgk-scrapes/bin/stage-6/./business-names-with-locations.py", line 68, in + main() + File "/home/ld/mgk-scrapes/bin/stage-6/./business-names-with-locations.py", line 61, in main + process_business_names() + File "/home/ld/mgk-scrapes/bin/stage-6/./business-names-with-locations.py", line 44, in process_business_names + cleaned_name, location = extract_location(business_name) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ld/mgk-scrapes/bin/stage-6/./business-names-with-locations.py", line 20, in extract_location + location = parts[1].strip() + ~~~~~^^^ +IndexError: list index out of range + diff --git a/bin/stage-6/business-names-with-locations.py b/bin/stage-6/business-names-with-locations.py new file mode 100755 index 0000000..ec77907 --- /dev/null +++ b/bin/stage-6/business-names-with-locations.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Script Name: business-names-with-locations + +import os +import csv +import re +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '07-business-name-structures-stripped.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '08-business-names-with-locations.csv') + +def extract_location(business_name): + """Extract location from business name if it contains ' of '.""" + if ' of ' in business_name.lower(): + parts = business_name.rsplit(' of ', 1) + if len(parts) == 2: # Ensure the split produced two parts + location = parts[1].strip() + business_name_clean = parts[0].strip() + # Remove trailing punctuation from the cleaned business name + business_name_clean = re.sub(r'[,.!?;:\-]+$', '', business_name_clean).strip() + return business_name_clean, location + return business_name, '' + +def process_business_names(): + """Process the business names to extract locations and save the results.""" + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business Name' not in headers: + print("Error: 'Business Name' column is missing.") + return + headers.append('Location from GBP Business Name') + + rows = list(reader) + + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + business_name = row.get('Business Name', '').strip() + cleaned_name, location = extract_location(business_name) + row['Business Name'] = cleaned_name + row['Location from GBP Business Name'] = location + writer.writerow(row) + +def main(): + figlet = Figlet(font='slant') + script_name = "business-names-with-locations".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Processing business names with locations', spinner='dots') + spinner.start() + process_business_names() + spinner.succeed('Business names with locations processed.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/delete-bad-matching-services.md b/bin/stage-6/delete-bad-matching-services.md new file mode 100644 index 0000000..0105a18 --- /dev/null +++ b/bin/stage-6/delete-bad-matching-services.md @@ -0,0 +1,169 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'delete-bad-matching-services' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage6_dir = os.path.join(project_root, '.data', 'stage-6') +deduplicated_csv_path = os.path.join(stage6_dir, '02-initial-deduplication.csv') +data_dir = os.path.join(project_root, '.data') +bad_services_file_path = os.path.join(data_dir, 'bad-matching-services.txt') + +def load_bad_services(file_path): + with open(file_path, 'r') as file: + bad_services = [line.strip().lower() for line in file if line.strip()] + return bad_services + +def delete_bad_matching_services(): + bad_services = load_bad_services(bad_services_file_path) + bad_services_set = set(bad_services) + cells_cleared = 0 + + with open(deduplicated_csv_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + headers = reader.fieldnames + rows = list(reader) + + with open(deduplicated_csv_path, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + + for row in tqdm(rows, desc="Processing rows", unit="row"): + matching_service = row.get('GBP Matching Service', '').strip().lower() + if matching_service in bad_services_set: + row['GBP Matching Service'] = '' + cells_cleared += 1 + writer.writerow(row) + + print(f'Total cells cleared: {cells_cleared}') + +if __name__ == '__main__': + delete_bad_matching_services() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 +I forgot to mention that 'delete-bad-matching-services' should use as input, the csv file: "03-trimmed-invalid-rows.csv". And as output, the csv file: "04-deleted-bad-matching-services.csv". diff --git a/bin/stage-6/delete-bad-matching-services.py b/bin/stage-6/delete-bad-matching-services.py new file mode 100755 index 0000000..ded4abe --- /dev/null +++ b/bin/stage-6/delete-bad-matching-services.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Script Name: delete-bad-matching-services + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") +INPUT_CSV_PATH = os.path.join(STAGE_6_DIRECTORY, '03-trimmed-invalid-rows.csv') +OUTPUT_CSV_PATH = os.path.join(STAGE_6_DIRECTORY, '04-deleted-bad-matching-services.csv') +BAD_SERVICES_FILE_PATH = os.path.join(DATA_DIRECTORY, 'bad-matching-services.txt') + +def load_bad_services(file_path): + """Load bad services from the specified file.""" + if not os.path.exists(file_path): + print(f"Error: The file {file_path} does not exist.") + sys.exit(1) + + with open(file_path, 'r') as file: + bad_services = [line.strip().lower() for line in file if line.strip()] + return bad_services + +def delete_bad_matching_services(): + """Delete bad matching services from the input CSV and write to the output CSV.""" + bad_services = load_bad_services(BAD_SERVICES_FILE_PATH) + bad_services_set = set(bad_services) + cells_cleared = 0 + + with open(INPUT_CSV_PATH, 'r') as csv_file: + reader = csv.DictReader(csv_file) + headers = reader.fieldnames + rows = list(reader) + + with open(OUTPUT_CSV_PATH, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + + for row in rows: + matching_service = row.get('GBP Matching Service', '').strip().lower() + if matching_service in bad_services_set: + row['GBP Matching Service'] = '' + cells_cleared += 1 + writer.writerow(row) + + return cells_cleared + +def main(): + figlet = Figlet(font='slant') + script_name = "delete-bad-matching-services".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_CSV_PATH): + print(f"Error: The file {INPUT_CSV_PATH} does not exist.") + sys.exit(1) + + print(f"Processing {INPUT_CSV_PATH}...") + spinner = Halo(text='Processing data', spinner='dots') + spinner.start() + cells_cleared = delete_bad_matching_services() + spinner.succeed("Processing complete.") + + print(f"Total cells cleared: {cells_cleared}") + +if __name__ == "__main__": + main() diff --git a/bin/stage-6/fill-in-missing-data.md b/bin/stage-6/fill-in-missing-data.md new file mode 100644 index 0000000..519f96e --- /dev/null +++ b/bin/stage-6/fill-in-missing-data.md @@ -0,0 +1,58 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 'fill-in-missing-data' which will use the csv file "12-re-ordered-columns.csv" as input, and will output to the csv file "13-filled-in-missing-data.csv" inside of [Stage 6]. + +I would like the script to merge in missing data for each row of match group, by fetching the missing data from other rows in the match group. + +The match groups I want to focus on are rows that have the same "Business UUID" entry. + +Rows with the same "Business UUID" should have certain bits of information in column. + +I want the script to examine each row in each match group for cells that have data missing from certain rows. If data is found to missing in those certain rows, I would like the script to try and find that data in the cells of other match member rows. If that data can be found, it should be copied over to the row with the cell that is missing that data. + +The columns I wish to account for are: + + +YiB +GBP Business Category +GBP Matching Service + diff --git a/bin/stage-6/fill-in-missing-data.py b/bin/stage-6/fill-in-missing-data.py new file mode 100755 index 0000000..e3510d9 --- /dev/null +++ b/bin/stage-6/fill-in-missing-data.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# Script Name: fill-in-missing-data + +import os +import csv +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '12-re-ordered-columns.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '13-filled-in-missing-data.csv') + +def fill_in_missing_data(): + """Fill in missing data for each row of match group by fetching data from other rows in the group.""" + columns_to_merge = ['YiB', 'GBP Business Category', 'GBP Matching Service'] + uuid_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Business UUID + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business UUID' not in headers: + print("Error: 'Business UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + business_uuid = row['Business UUID'].strip() + if business_uuid: # Ignore empty UUIDs + uuid_to_rows[business_uuid].append(row) + + print(f"Total groups by Business UUID: {len(uuid_to_rows)}") + + # Fill in missing data within each group + for uuid, grouped_rows in uuid_to_rows.items(): + # Collect data from all rows in the group + merged_data = {column: '' for column in columns_to_merge} + for row in grouped_rows: + for column in columns_to_merge: + if row[column].strip(): + merged_data[column] = row[column] + + # Fill missing data in each row + for row in grouped_rows: + for column in columns_to_merge: + if not row[column].strip(): + row[column] = merged_data[column] + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "fill-in-missing-data".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Filling in missing data by Business UUID', spinner='dots') + spinner.start() + fill_in_missing_data() + spinner.succeed('Missing data filled successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/find-unknown-categories-and-services.md b/bin/stage-6/find-unknown-categories-and-services.md new file mode 100644 index 0000000..73fc662 --- /dev/null +++ b/bin/stage-6/find-unknown-categories-and-services.md @@ -0,0 +1,126 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'find-unknown-categories-and-services' to be in accordance with the above, and to adjust the reporting style now: + + + + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +The 'find-unknown-categories-and-services' script should use as input, the csv file: "05-standard-ized-capital-letters.csv". diff --git a/bin/stage-6/find-unknown-categories-and-services.py b/bin/stage-6/find-unknown-categories-and-services.py new file mode 100755 index 0000000..fcf694d --- /dev/null +++ b/bin/stage-6/find-unknown-categories-and-services.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Script Name: find-unknown-categories-and-services + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") +INPUT_CSV_PATH = os.path.join(STAGE_6_DIRECTORY, '05-standardized-capital-letters.csv') +GBP_CATEGORIES_PATH = os.path.join(DATA_DIRECTORY, 'gbp-business-categories.txt') +GBP_SERVICES_PATH = os.path.join(DATA_DIRECTORY, 'gbp-matching-services.txt') +UNKNOWN_CATEGORIES_PATH = os.path.join(STAGE_6_DIRECTORY, 'unknown-categories.txt') +UNKNOWN_SERVICES_PATH = os.path.join(STAGE_6_DIRECTORY, 'unknown-services.txt') + +def load_known_values(file_path): + """Load known values from a text file into a set.""" + with open(file_path, 'r') as file: + return set(line.strip().lower() for line in file if line.strip()) + +def find_unknowns(input_csv_path, known_categories, known_services): + """Find unknown categories and services in the input CSV file.""" + unknown_categories = set() + unknown_services = set() + + with open(input_csv_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + for row in reader: + category = row.get('GBP Business Category', '').strip().lower() + service = row.get('GBP Matching Service', '').strip().lower() + + if category and category not in known_categories: + unknown_categories.add(category) + if service and service not in known_services: + unknown_services.add(service) + + return unknown_categories, unknown_services + +def save_unknowns(unknowns, file_path): + """Save unknown values to a text file.""" + with open(file_path, 'w') as file: + for item in sorted(unknowns): + file.write(f"{item}\n") + +def main(): + figlet = Figlet(font='slant') + script_name = "find-unknown-categories-and-services".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_CSV_PATH): + print(f"Error: The file {INPUT_CSV_PATH} does not exist.") + sys.exit(1) + + print(f"Processing {INPUT_CSV_PATH}...") + spinner = Halo(text='Finding unknown categories and services', spinner='dots') + spinner.start() + + known_categories = load_known_values(GBP_CATEGORIES_PATH) + known_services = load_known_values(GBP_SERVICES_PATH) + unknown_categories, unknown_services = find_unknowns(INPUT_CSV_PATH, known_categories, known_services) + + save_unknowns(unknown_categories, UNKNOWN_CATEGORIES_PATH) + save_unknowns(unknown_services, UNKNOWN_SERVICES_PATH) + + spinner.succeed("Finding unknown categories and services complete.") + print(f"Total unknown categories found: {len(unknown_categories)}") + print(f"Total unknown services found: {len(unknown_services)}") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/initial-deduplication.md b/bin/stage-6/initial-deduplication.md new file mode 100644 index 0000000..b0d8c9b --- /dev/null +++ b/bin/stage-6/initial-deduplication.md @@ -0,0 +1,50 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I wish to make a new script which will have as input the csv file: "05-standardized-capital-letters.csv". And as output, the csv file: "06-initial-deduplication.csv". + +This script should perform a very basic deduplication on the 05 file. Removing any duplicate lines from the csv file. + +For this script, we are consider duplicates to be any line that has another line which is exactly the same accross all fields. + +When the script completes, it should report a before count for rows of data, and an after count for rows of data. It should also list how many duplicates were removed. + +This script should be called 'initial-deduplication'. + +It will reside in [Stage 6 Binaries] + diff --git a/bin/stage-6/initial-deduplication.py b/bin/stage-6/initial-deduplication.py new file mode 100755 index 0000000..cf88b16 --- /dev/null +++ b/bin/stage-6/initial-deduplication.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# Script Name: initial-deduplication + +import os +import csv +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '05-standardized-capital-letters.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '06-initial-deduplication.csv') +MD5_CSV = os.path.join(STAGE_6_DIRECTORY, 'aa-existing-md5-for-scrapes.csv') + +def extract_unique_md5(): + """Extract unique MD5 entries and write them to a new CSV file.""" + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + md5_set = set() + + for row in reader: + md5_value = row.get("MD5 for Scrape", "").strip() + if md5_value: + md5_set.add(md5_value) + + with open(MD5_CSV, 'w', newline='') as outfile: + writer = csv.writer(outfile) + writer.writerow(["MD5 for Scrape"]) + for md5_value in md5_set: + writer.writerow([md5_value]) + +def remove_duplicates(): + """Remove duplicate rows from the CSV file.""" + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + rows = list(reader) + total_rows_before = len(rows) + + # Use a set to track unique rows + unique_rows = [] + seen = set() + for row in rows: + row_tuple = tuple(row.items()) + if row_tuple not in seen: + seen.add(row_tuple) + unique_rows.append(row) + + total_rows_after = len(unique_rows) + duplicates_removed = total_rows_before - total_rows_after + + # Write the unique rows back to a new CSV file + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + writer.writerows(unique_rows) + + return total_rows_before, total_rows_after, duplicates_removed + +def main(): + figlet = Figlet(font='slant') + script_name = "initial-deduplication".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Extract unique MD5 entries + md5_spinner = Halo(text='Extracting unique MD5 entries', spinner='dots') + md5_spinner.start() + extract_unique_md5() + md5_spinner.succeed('Unique MD5 entries extracted.') + + # Remove duplicates + spinner = Halo(text='Removing duplicates', spinner='dots') + spinner.start() + total_rows_before, total_rows_after, duplicates_removed = remove_duplicates() + spinner.succeed('Duplicates removed.') + + print(f'Total rows before: {total_rows_before}') + print(f'Total rows after: {total_rows_after}') + print(f'Duplicates removed: {duplicates_removed}') + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/old-initial-dedup.py b/bin/stage-6/old-initial-dedup.py new file mode 100755 index 0000000..15d6168 --- /dev/null +++ b/bin/stage-6/old-initial-dedup.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python + +import os +import csv +import shutil +from tqdm import tqdm +from collections import defaultdict + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage6_dir = os.path.join(project_root, '.data', 'stage-6') +merged_csv_path = os.path.join(stage6_dir, '01-first-merger.csv') +deduplicated_csv_path = os.path.join(stage6_dir, '02-initial-deduplication.csv') +duplicates_csv_path = os.path.join(stage6_dir, 'AA-deduplicated-duplicates.csv') + +def copy_merged_to_deduplicated(): + if os.path.exists(deduplicated_csv_path): + os.remove(deduplicated_csv_path) + shutil.copy(merged_csv_path, deduplicated_csv_path) + +def merge_rows(rows): + merged_row = rows[0].copy() + for row in rows[1:]: + for key in merged_row: + if not merged_row[key] and row[key]: + merged_row[key] = row[key] + return merged_row + +def deduplicate_and_merge(): + business_dict = defaultdict(list) + unique_rows = [] + duplicate_rows = [] + total_duplicates_removed = 0 + + # Read the CSV file + with open(deduplicated_csv_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + headers = reader.fieldnames + rows = list(reader) + + # Use tqdm to display a progress bar + for row in tqdm(rows, desc="Grouping rows by Business Name", unit="row"): + business_name = row.get('Business Name', '').strip() + business_dict[business_name].append(row) + + # Process each group of duplicates + for business_name, business_rows in tqdm(business_dict.items(), desc="Processing business groups", unit="group"): + if len(business_rows) == 1: + unique_rows.append(business_rows[0]) + continue + + # Group by GBP Review Rating and GBP Review Count (location identifier) + location_dict = defaultdict(list) + for row in business_rows: + location_id = (row.get('GBP Review Rating', '').strip(), row.get('GBP Review Count', '').strip()) + location_dict[location_id].append(row) + + # Process each location group + for location_id, location_rows in location_dict.items(): + if len(location_rows) == 1: + unique_rows.append(location_rows[0]) + else: + # Merge rows and keep the most complete one + merged_row = merge_rows(location_rows) + unique_rows.append(merged_row) + duplicate_rows.extend(location_rows) + total_duplicates_removed += (len(location_rows) - 1) + + # Second pass for deduplication based on phone number within the same municipality + final_unique_rows = [] + municipality_dict = defaultdict(list) + for row in unique_rows: + municipality_key = (row['Business Name'], row['GBP Location Municipality']) + municipality_dict[municipality_key].append(row) + + for municipality_key, municipality_rows in municipality_dict.items(): + phone_dict = defaultdict(list) + for row in municipality_rows: + phone_dict[row['GBP Business Phone']].append(row) + + for phone, phone_rows in phone_dict.items(): + if len(phone_rows) == 1: + final_unique_rows.append(phone_rows[0]) + else: + merged_row = merge_rows(phone_rows) + final_unique_rows.append(merged_row) + duplicate_rows.extend(phone_rows) + total_duplicates_removed += (len(phone_rows) - 1) + + # Third pass for merging based on business name and phone number + final_deduplicated_rows = [] + business_dict = defaultdict(list) + for row in final_unique_rows: + business_name = row.get('Business Name', '').strip() + business_dict[business_name].append(row) + + for business_name, business_rows in tqdm(business_dict.items(), desc="Processing final business groups", unit="group"): + if len(business_rows) == 1: + final_deduplicated_rows.append(business_rows[0]) + continue + + location_dict = defaultdict(list) + for row in business_rows: + location_key = (row.get('GBP Location Municipality', '').strip(), row.get('GBP Location State', '').strip()) + location_dict[location_key].append(row) + + for location_key, location_rows in location_dict.items(): + phone_dict = defaultdict(list) + for row in location_rows: + phone_dict[row['GBP Business Phone']].append(row) + + for phone, phone_rows in phone_dict.items(): + if len(phone_rows) == 1: + final_deduplicated_rows.append(phone_rows[0]) + else: + merged_row = merge_rows(phone_rows) + final_deduplicated_rows.append(merged_row) + duplicate_rows.extend(phone_rows) + total_duplicates_removed += (len(phone_rows) - 1) + + # Additional pass to merge based on business name and phone number + final_pass_rows = [] + final_business_dict = defaultdict(list) + for row in final_deduplicated_rows: + business_name = row.get('Business Name', '').strip() + final_business_dict[business_name].append(row) + + for business_name, business_rows in tqdm(final_business_dict.items(), desc="Final processing for business groups", unit="group"): + if len(business_rows) == 1: + final_pass_rows.append(business_rows[0]) + continue + + phone_dict = defaultdict(list) + for row in business_rows: + phone_dict[row['GBP Business Phone']].append(row) + + for phone, phone_rows in phone_dict.items(): + if len(phone_rows) == 1: + final_pass_rows.append(phone_rows[0]) + else: + merged_row = merge_rows(phone_rows) + final_pass_rows.append(merged_row) + duplicate_rows.extend(phone_rows) + total_duplicates_removed += (len(phone_rows) - 1) + + # Sort the final deduplicated rows for easier viewing + final_pass_rows.sort(key=lambda x: (x['Business Name'], x['GBP Location Municipality'], x['GBP Location State'], x['Root Domain'])) + + # Write the final deduplicated rows to the deduplicated CSV file + with open(deduplicated_csv_path, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + writer.writerows(final_pass_rows) + + # Sort the duplicate rows for easier viewing + duplicate_rows.sort(key=lambda x: (x['Business Name'], x['GBP Location Municipality'], x['GBP Location State'], x['Root Domain'])) + + # Write the duplicate rows to the duplicates CSV file + with open(duplicates_csv_path, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + writer.writerows(duplicate_rows) + + # Print the total number of duplicates removed + print(f'Total duplicates removed: {total_duplicates_removed}') + +if __name__ == '__main__': + copy_merged_to_deduplicated() + deduplicate_and_merge() + diff --git a/bin/stage-6/package-lock.json b/bin/stage-6/package-lock.json new file mode 100644 index 0000000..3845da6 --- /dev/null +++ b/bin/stage-6/package-lock.json @@ -0,0 +1,1640 @@ +{ + "name": "stage-6", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "lighthouse": "^12.0.0" + } + }, + "node_modules/@formatjs/ecma402-abstract": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-2.0.0.tgz", + "integrity": "sha512-rRqXOqdFmk7RYvj4khklyqzcfQl9vEL/usogncBHRZfZBDOwMGuSRNFl02fu5KGHXdbinju+YXyuR+Nk8xlr/g==", + "dependencies": { + "@formatjs/intl-localematcher": "0.5.4", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/ecma402-abstract/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, + "node_modules/@formatjs/fast-memoize": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@formatjs/fast-memoize/-/fast-memoize-2.2.0.tgz", + "integrity": "sha512-hnk/nY8FyrL5YxwP9e4r9dqeM6cAbo8PeU9UjyXojZMNvVad2Z06FAVHyR3Ecw6fza+0GH7vdJgiKIVXTMbSBA==", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/fast-memoize/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, + "node_modules/@formatjs/icu-messageformat-parser": { + "version": "2.7.8", + "resolved": "https://registry.npmjs.org/@formatjs/icu-messageformat-parser/-/icu-messageformat-parser-2.7.8.tgz", + "integrity": "sha512-nBZJYmhpcSX0WeJ5SDYUkZ42AgR3xiyhNCsQweFx3cz/ULJjym8bHAzWKvG5e2+1XO98dBYC0fWeeAECAVSwLA==", + "dependencies": { + "@formatjs/ecma402-abstract": "2.0.0", + "@formatjs/icu-skeleton-parser": "1.8.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-messageformat-parser/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, + "node_modules/@formatjs/icu-skeleton-parser": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@formatjs/icu-skeleton-parser/-/icu-skeleton-parser-1.8.2.tgz", + "integrity": "sha512-k4ERKgw7aKGWJZgTarIcNEmvyTVD9FYh0mTrrBMHZ1b8hUu6iOJ4SzsZlo3UNAvHYa+PnvntIwRPt1/vy4nA9Q==", + "dependencies": { + "@formatjs/ecma402-abstract": "2.0.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-skeleton-parser/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, + "node_modules/@formatjs/intl-localematcher": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.5.4.tgz", + "integrity": "sha512-zTwEpWOzZ2CiKcB93BLngUX59hQkuZjT2+SAQEscSm52peDW/getsawMcWF1rGRpMCX6D7nSJA3CzJ8gn13N/g==", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl-localematcher/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, + "node_modules/@paulirish/trace_engine": { + "version": "0.0.19", + "resolved": "https://registry.npmjs.org/@paulirish/trace_engine/-/trace_engine-0.0.19.tgz", + "integrity": "sha512-3tjEzXBBtU83DkCJAdU2UwBBunspiwTCn+Y5jOxm592cfEuLr/T7Lcn+QhRerVqkSik2mnjN4X6NgHZjI9Biwg==" + }, + "node_modules/@puppeteer/browsers": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.2.3.tgz", + "integrity": "sha512-bJ0UBsk0ESOs6RFcLXOt99a3yTDcOKlzfjad+rhFwdaG1Lu/Wzq58GHYCDTlZ9z6mldf4g+NTb+TXEfe0PpnsQ==", + "dependencies": { + "debug": "4.3.4", + "extract-zip": "2.0.1", + "progress": "2.0.3", + "proxy-agent": "6.4.0", + "semver": "7.6.0", + "tar-fs": "3.0.5", + "unbzip2-stream": "1.4.3", + "yargs": "17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@puppeteer/browsers/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@puppeteer/browsers/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@puppeteer/browsers/node_modules/semver": { + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", + "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/core": { + "version": "6.19.7", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.19.7.tgz", + "integrity": "sha512-tOfZ/umqB2AcHPGbIrsFLcvApdTm9ggpi/kQZFkej7kMphjT+SGBiQfYtjyg9jcRW+ilAR4JXC9BGKsdEQ+8Vw==", + "dependencies": { + "@sentry/hub": "6.19.7", + "@sentry/minimal": "6.19.7", + "@sentry/types": "6.19.7", + "@sentry/utils": "6.19.7", + "tslib": "^1.9.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@sentry/hub": { + "version": "6.19.7", + "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.19.7.tgz", + "integrity": "sha512-y3OtbYFAqKHCWezF0EGGr5lcyI2KbaXW2Ik7Xp8Mu9TxbSTuwTe4rTntwg8ngPjUQU3SUHzgjqVB8qjiGqFXCA==", + "dependencies": { + "@sentry/types": "6.19.7", + "@sentry/utils": "6.19.7", + "tslib": "^1.9.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@sentry/minimal": { + "version": "6.19.7", + "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.19.7.tgz", + "integrity": "sha512-wcYmSJOdvk6VAPx8IcmZgN08XTXRwRtB1aOLZm+MVHjIZIhHoBGZJYTVQS/BWjldsamj2cX3YGbGXNunaCfYJQ==", + "dependencies": { + "@sentry/hub": "6.19.7", + "@sentry/types": "6.19.7", + "tslib": "^1.9.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@sentry/node": { + "version": "6.19.7", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-6.19.7.tgz", + "integrity": "sha512-gtmRC4dAXKODMpHXKfrkfvyBL3cI8y64vEi3fDD046uqYcrWdgoQsffuBbxMAizc6Ez1ia+f0Flue6p15Qaltg==", + "dependencies": { + "@sentry/core": "6.19.7", + "@sentry/hub": "6.19.7", + "@sentry/types": "6.19.7", + "@sentry/utils": "6.19.7", + "cookie": "^0.4.1", + "https-proxy-agent": "^5.0.0", + "lru_map": "^0.3.3", + "tslib": "^1.9.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@sentry/types": { + "version": "6.19.7", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.19.7.tgz", + "integrity": "sha512-jH84pDYE+hHIbVnab3Hr+ZXr1v8QABfhx39KknxqKWr2l0oEItzepV0URvbEhB446lk/S/59230dlUUIBGsXbg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/@sentry/utils": { + "version": "6.19.7", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.19.7.tgz", + "integrity": "sha512-z95ECmE3i9pbWoXQrD/7PgkBAzJYR+iXtPuTkpBjDKs86O3mT+PXOT3BAn79w2wkn7/i3vOGD2xVr1uiMl26dA==", + "dependencies": { + "@sentry/types": "6.19.7", + "tslib": "^1.9.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, + "node_modules/@types/node": { + "version": "20.14.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.2.tgz", + "integrity": "sha512-xyu6WAMVwv6AKFLB+e/7ySZVr/0zLCzOa7rSpq6jNwpqOrUbcACDWC+53d4n2QHOnDou0fbIsg8wZu/sxrnI4Q==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ast-types/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, + "node_modules/axe-core": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.9.1.tgz", + "integrity": "sha512-QbUdXJVTpvUTHU7871ppZkdOLBeGUKBQWHkHrvN2V9IQWGMt61zf3B45BtzjxEJzYuj0JBjBZP/hmYS/R9pmAw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/b4a": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", + "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==" + }, + "node_modules/bare-events": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", + "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", + "optional": true + }, + "node_modules/bare-fs": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.1.tgz", + "integrity": "sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA==", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "node_modules/bare-os": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.3.0.tgz", + "integrity": "sha512-oPb8oMM1xZbhRQBngTgpcQ5gXw6kjOaRsSWsIeNyRxGed2w/ARyP7ScBYpWR1qfX2E5rS3gBw6OWcSQo+s+kUg==", + "optional": true + }, + "node_modules/bare-path": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "optional": true, + "dependencies": { + "bare-os": "^2.1.0" + } + }, + "node_modules/bare-stream": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.1.2.tgz", + "integrity": "sha512-az/7TFOh4Gk9Tqs1/xMFq5FuFoeZ9hZ3orsM2x69u8NXVUDXZnpdhG8mZY/Pv6DF954MGn+iIt4rFrG34eQsvg==", + "optional": true, + "dependencies": { + "streamx": "^2.18.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, + "node_modules/chrome-launcher": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-1.1.2.tgz", + "integrity": "sha512-YclTJey34KUm5jB1aEJCq807bSievi7Nb/TU4Gu504fUYi3jw3KCIaH6L7nFWQhdEgH3V+wCh+kKD1P5cXnfxw==", + "dependencies": { + "@types/node": "*", + "escape-string-regexp": "^4.0.0", + "is-wsl": "^2.2.0", + "lighthouse-logger": "^2.0.1" + }, + "bin": { + "print-chrome-path": "bin/print-chrome-path.js" + }, + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/chromium-bidi": { + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.5.23.tgz", + "integrity": "sha512-1o/gLU9wDqbN5nL2MtfjykjOuighGXc3/hnWueO1haiEoFgX8h5vbvcA4tgdQfjw1mkZ1OEF4x/+HVeqEX6NoA==", + "dependencies": { + "mitt": "3.0.1", + "urlpattern-polyfill": "10.0.0", + "zod": "3.23.8" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/configstore": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "dependencies": { + "dot-prop": "^5.2.0", + "graceful-fs": "^4.1.2", + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cookie": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/csp_evaluator": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/csp_evaluator/-/csp_evaluator-1.1.1.tgz", + "integrity": "sha512-N3ASg0C4kNPUaNxt1XAvzHIVuzdtr8KLgfk1O8WDyimp1GisPAHESupArO2ieHk9QWbrJ/WkQODyh21Ps/xhxw==" + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "engines": { + "node": ">=8" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1232444", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1232444.tgz", + "integrity": "sha512-pM27vqEfxSxRkTMnF+XCmxSEb6duO5R+t8A9DEEJgy4Wz2RVanje2mmj99B6A3zv2r/qGfYlOvYznUhuokizmg==" + }, + "node_modules/dot-prop": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", + "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/enquirer": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", + "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", + "dependencies": { + "ansi-colors": "^4.1.1", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/escalade": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", + "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4", + "fs-extra": "^11.2.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "node_modules/http-link-header": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/http-link-header/-/http-link-header-1.1.3.tgz", + "integrity": "sha512-3cZ0SRL8fb9MUlU3mKM61FcQvPfXx2dBrZW3Vbg5CXa8jFlK8OaEpePenLe1oEXQduhz8b0QjsqfS59QP4AJDQ==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/image-ssim": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/image-ssim/-/image-ssim-0.2.0.tgz", + "integrity": "sha512-W7+sO6/yhxy83L0G7xR8YAc5Z5QFtYEXXRV6EaE8tuYBZJnA3gVgp3q7X7muhLZVodeb9UfvjSbwt9VJwjIYAg==" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/intl-messageformat": { + "version": "10.5.14", + "resolved": "https://registry.npmjs.org/intl-messageformat/-/intl-messageformat-10.5.14.tgz", + "integrity": "sha512-IjC6sI0X7YRjjyVH9aUgdftcmZK7WXdHeil4KwbjDnRWjnVitKpAx3rr6t6di1joFp5188VqKcobOPA6mCLG/w==", + "dependencies": { + "@formatjs/ecma402-abstract": "2.0.0", + "@formatjs/fast-memoize": "2.2.0", + "@formatjs/icu-messageformat-parser": "2.7.8", + "tslib": "^2.4.0" + } + }, + "node_modules/intl-messageformat/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==" + }, + "node_modules/js-library-detector": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/js-library-detector/-/js-library-detector-6.7.0.tgz", + "integrity": "sha512-c80Qupofp43y4cJ7+8TTDN/AsDwLi5oOm/plBrWI+iQt485vKXCco+yVmOwEgdo9VOdsYTuV0UlTeetVPTriXA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/lighthouse": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/lighthouse/-/lighthouse-12.0.0.tgz", + "integrity": "sha512-8lZJ/D/fH/bQMyAmVUjts1IS0NOMa0SSbyce+OGQdKbaroVnontuVh9HsB9z/1ba3dEyBguOJiMkl6gkqyf7Tw==", + "dependencies": { + "@paulirish/trace_engine": "^0.0.19", + "@sentry/node": "^6.17.4", + "axe-core": "^4.9.0", + "chrome-launcher": "^1.1.1", + "configstore": "^5.0.1", + "csp_evaluator": "1.1.1", + "devtools-protocol": "0.0.1232444", + "enquirer": "^2.3.6", + "http-link-header": "^1.1.1", + "intl-messageformat": "^10.5.3", + "jpeg-js": "^0.4.4", + "js-library-detector": "^6.7.0", + "lighthouse-logger": "^2.0.1", + "lighthouse-stack-packs": "1.12.1", + "lodash": "^4.17.21", + "lookup-closest-locale": "6.2.0", + "metaviewport-parser": "0.3.0", + "open": "^8.4.0", + "parse-cache-control": "1.0.1", + "ps-list": "^8.0.0", + "puppeteer-core": "^22.6.5", + "robots-parser": "^3.0.1", + "semver": "^5.3.0", + "speedline-core": "^1.4.3", + "third-party-web": "^0.24.2", + "tldts-icann": "^6.1.16", + "ws": "^7.0.0", + "yargs": "^17.3.1", + "yargs-parser": "^21.0.0" + }, + "bin": { + "chrome-debug": "core/scripts/manual-chrome-launcher.js", + "lighthouse": "cli/index.js", + "smokehouse": "cli/test/smokehouse/frontends/smokehouse-bin.js" + }, + "engines": { + "node": ">=18.16" + } + }, + "node_modules/lighthouse-logger": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-2.0.1.tgz", + "integrity": "sha512-ioBrW3s2i97noEmnXxmUq7cjIcVRjT5HBpAYy8zE11CxU9HqlWHHeRxfeN1tn8F7OEMVPIC9x1f8t3Z7US9ehQ==", + "dependencies": { + "debug": "^2.6.9", + "marky": "^1.2.2" + } + }, + "node_modules/lighthouse-logger/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/lighthouse-logger/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/lighthouse-stack-packs": { + "version": "1.12.1", + "resolved": "https://registry.npmjs.org/lighthouse-stack-packs/-/lighthouse-stack-packs-1.12.1.tgz", + "integrity": "sha512-i4jTmg7tvZQFwNFiwB+nCK6a7ICR68Xcwo+VIVd6Spi71vBNFUlds5HiDrSbClZdkQDON2Bhqv+KKJIo5zkPeA==" + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lookup-closest-locale": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/lookup-closest-locale/-/lookup-closest-locale-6.2.0.tgz", + "integrity": "sha512-/c2kL+Vnp1jnV6K6RpDTHK3dgg0Tu2VVp+elEiJpjfS1UyY7AjOYHohRug6wT0OpoX2qFgNORndE9RqesfVxWQ==" + }, + "node_modules/lru_map": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/lru_map/-/lru_map-0.3.3.tgz", + "integrity": "sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==" + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/marky": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/marky/-/marky-1.2.5.tgz", + "integrity": "sha512-q9JtQJKjpsVxCRVgQ+WapguSbKC3SQ5HEzFGPAJMStgh3QjCawp00UKv3MTTAArTmGmmPUvllHZoNbZ3gs0I+Q==" + }, + "node_modules/metaviewport-parser": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/metaviewport-parser/-/metaviewport-parser-0.3.0.tgz", + "integrity": "sha512-EoYJ8xfjQ6kpe9VbVHvZTZHiOl4HL1Z18CrZ+qahvLXT7ZO4YTC2JMyt5FaUp9JJp6J4Ybb/z7IsCXZt86/QkQ==" + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz", + "integrity": "sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A==", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.2", + "pac-resolver": "^7.0.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/parse-cache-control": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", + "integrity": "sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==" + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", + "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.3", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/ps-list": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/ps-list/-/ps-list-8.1.1.tgz", + "integrity": "sha512-OPS9kEJYVmiO48u/B9qneqhkMvgCxT+Tm28VCEJpheTpl8cJ0ffZRRNgS5mrQRTrX5yRTpaJ+hRDeefXYmmorQ==", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer-core": { + "version": "22.11.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.11.0.tgz", + "integrity": "sha512-57YUjhRoSpZWg9lCssWsgzM1/X/1jQnkKbbspbeW0bhZTt3TD4WdNXEYI7KrFFnSvx21tyHhfWW0zlxzbwYSAA==", + "dependencies": { + "@puppeteer/browsers": "2.2.3", + "chromium-bidi": "0.5.23", + "debug": "4.3.5", + "devtools-protocol": "0.0.1299070", + "ws": "8.17.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core/node_modules/devtools-protocol": { + "version": "0.0.1299070", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1299070.tgz", + "integrity": "sha512-+qtL3eX50qsJ7c+qVyagqi7AWMoQCBGNfoyJZMwm/NSXVqLYbuitrWEEIzxfUmTNy7//Xe8yhMmQ+elj3uAqSg==" + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/robots-parser": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz", + "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.3.tgz", + "integrity": "sha512-VNegTZKhuGq5vSD6XNKlbqWhyt/40CgoEw8XxD6dhnm8Jq9IEa3nIa4HwnM8XOqU0CdB0BwWVXusqiFXfHB3+A==", + "dependencies": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.7.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/speedline-core": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/speedline-core/-/speedline-core-1.4.3.tgz", + "integrity": "sha512-DI7/OuAUD+GMpR6dmu8lliO2Wg5zfeh+/xsdyJZCzd8o5JgFUjCeLsBDuZjIQJdwXS3J0L/uZYrELKYqx+PXog==", + "dependencies": { + "@types/node": "*", + "image-ssim": "^0.2.0", + "jpeg-js": "^0.4.1" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, + "node_modules/streamx": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.18.0.tgz", + "integrity": "sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==", + "dependencies": { + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.5.tgz", + "integrity": "sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg==", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.0.tgz", + "integrity": "sha512-TmLJNj6UgX8xcUZo4UDStGQtDiTzF7BzWlzn9g7UWrjkpHr5uJTK1ld16wZ3LXb2vb6jH8qU89dW5whuMdXYdw==", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/third-party-web": { + "version": "0.24.3", + "resolved": "https://registry.npmjs.org/third-party-web/-/third-party-web-0.24.3.tgz", + "integrity": "sha512-imE6hXZyaCeGinGFCvpWsv0oelsEaufSG39qYBQhp3urGq4OLOtsuEddf3XgKxmAAczBD/I1Tnp8L3gJ3ksTuQ==" + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "node_modules/tldts-core": { + "version": "6.1.25", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.25.tgz", + "integrity": "sha512-hbSsjJOeDMV91JiqcrrFQ46D7EepH880zVmPjnBDmt3P+h0Aowz8Nh1adIcqkdhJbgpzZYQr6aM8/N3tZC6JyA==" + }, + "node_modules/tldts-icann": { + "version": "6.1.25", + "resolved": "https://registry.npmjs.org/tldts-icann/-/tldts-icann-6.1.25.tgz", + "integrity": "sha512-Qk8ylqwECYkJgetkRaG+4qLeG/73IS3ASa5Mv+eBaptgZBzpkJTcxvEMvH8nUb+aoJIAxHOP4CKu2v4L0NiRcQ==", + "dependencies": { + "tldts-core": "^6.1.25" + } + }, + "node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, + "node_modules/unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "dependencies": { + "crypto-random-string": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/urlpattern-polyfill": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", + "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dependencies": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, + "node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xdg-basedir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/bin/stage-6/package.json b/bin/stage-6/package.json new file mode 100644 index 0000000..4216912 --- /dev/null +++ b/bin/stage-6/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "lighthouse": "^12.0.0" + } +} diff --git a/bin/stage-6/pre-sort.md b/bin/stage-6/pre-sort.md new file mode 100644 index 0000000..9bc8ff1 --- /dev/null +++ b/bin/stage-6/pre-sort.md @@ -0,0 +1,134 @@ + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to create a new script called 'pre-sort'. It should use the file 01-first-merger.csv in the Stage 5 data directory as the input. And the output for this script should a file in the same directory named 02-pre-sorted.csv. + +The job of the script will be to sort the data in the input csv using the first column as the index, and output the sorted data to the output file. + +Respect the headings in the CSV file as well. + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 +I got the following error while running this script: + + +Error: The file /home/ld/mgk-scrapes/current-data/.data/stage-5/01-first-merger.csv does not exist. + + +This script needs to operate in the Stage 6 directory, not the Stage 5 directory. + +Also, please have this script output a total tally of all data records found in the CSV file after the new file has been created. diff --git a/bin/stage-6/pre-sort.py b/bin/stage-6/pre-sort.py new file mode 100755 index 0000000..3963bc0 --- /dev/null +++ b/bin/stage-6/pre-sort.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# Script Name: pre-sort + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") +MERGED_CSV = os.path.join(STAGE_6_DIRECTORY, "01-first-merger.csv") +SORTED_CSV = os.path.join(STAGE_6_DIRECTORY, "02-pre-sorted.csv") + +def sort_csv_file(input_file, output_file): + """Sort the CSV file by the first column and return the number of records.""" + with open(input_file, 'r') as file: + reader = csv.reader(file) + headers = next(reader) + sorted_rows = sorted(reader, key=lambda row: row[0]) + + with open(output_file, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(sorted_rows) + + return len(sorted_rows) + +def main(): + figlet = Figlet(font='slant') + script_name = "pre-sort".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(MERGED_CSV): + print(f"Error: The file {MERGED_CSV} does not exist.") + sys.exit(1) + + print(f"Sorting {MERGED_CSV}...") + spinner = Halo(text='Sorting data', spinner='dots') + spinner.start() + total_records = sort_csv_file(MERGED_CSV, SORTED_CSV) + spinner.succeed("Sorting complete.") + + print(f"Sorted data saved to {SORTED_CSV}") + print(f"Total records in sorted file: {total_records}") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/prepare-stage-6.md b/bin/stage-6/prepare-stage-6.md new file mode 100644 index 0000000..c01ca81 --- /dev/null +++ b/bin/stage-6/prepare-stage-6.md @@ -0,0 +1,197 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +Now I need a prepare-stage-6 script. + +This script is different than the rest of the prepare-stage-# scripts though. + +This script needs to check if Stage 6 exists first, and if it exists, delete it, and then re-create the directory. In the same way as the previous prepare-stage-# scripts. + +However, instead of directly copying over the directories and files from stage-5, we need to create a new csv file in the Stage 6 directory called "01-first-merger.csv". + +This contents of this CSV file will be populated by iterating through every CSV file in Stage 5. The script needs to start by grabbing a heading from the first CSV file encountered, adding that to 01-first-merger.csv, and then appending data to that CSV file after that. Every CSV file in Stage 5 should be appended to 01-first-merger.csv. Minus the heading. We need to be certain that we only ever include a single heading in 01-first-merger.csv. We do not want mulitple headings to appear in the data, corrupting the data set. + +Please be sure to name all script output to accurately describe what the script is doing as it does it. ANd use spinners to show us each State being processed. We do not need a new line of reporting for every county. By state is sufficient. + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +I am having some difficulty with the 'prepare-stage-6' script: + + +#!/usr/bin/env python +# Script Name: prepare-stage-6 + +import os +import shutil +import csv +from halo import Halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") +MERGED_CSV = os.path.join(STAGE_6_DIRECTORY, "01-first-merger.csv") + +def create_stage_6_directory(): + """Create the stage-6 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_6_DIRECTORY): + print("Existing Stage 6 found, deleting and replacing...") + shutil.rmtree(STAGE_6_DIRECTORY) + os.makedirs(STAGE_6_DIRECTORY) + print("Stage 6 directory created.") + +def merge_csv_files(): + """Merge all CSV files from stage-5 into a single CSV file in stage-6.""" + first_file = True + with open(MERGED_CSV, 'w', newline='') as outfile: + writer = None + for state_dir in os.listdir(STAGE_5_DIRECTORY): + state_path = os.path.join(STAGE_5_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + with open(file_path, 'r') as infile: + reader = csv.reader(infile) + header = next(reader) + if first_file: + writer = csv.writer(outfile) + writer.writerow(header) + first_file = False + for row in reader: + writer.writerow(row) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 6...") + + create_stage_6_directory() + + print("Merging CSV files to Stage 6...") + spinner = Halo(text='Merging data', spinner='dots') + spinner.start() + merge_csv_files() + spinner.succeed("Data merged.") + + print("Stage 6 preparation complete.") + +if __name__ == "__main__": + main() + + +As it right now, when I run that script the resulting 01-first-merger.csv file has a lot of length mismatches between the header column count and data rows. + +I am wondering if we can combat this by taking a different approach when merging the data. + +What if we simply create the 01-first-merger.csv with the correct headers first: + + +Business Name +GBP Review Rating +GBP Review Count +GBP Business Category +YiB +GBP Matching Service +GBP Business Phone +GBP Business Website +GBP Location Municipality +GBP Location State +Root Domain +. + +Then we begin to iterate through the CSV files in the Stage 5 data directory, copying over the data from those CSV files into our 01-first-merger.csv. But we do it in a row by row, cell by cell fashion. + +For each data row in the CSV files we iterate through we copy over the Business Name data to the Business Name column, then the GBP Review Rating data from the GBP Review Rating cell in the source CSV to the GBP Review Rating cell in the destination CSV. + +And proceed like this until all cells, which data in them, for that given row have been copied over to the correct column in the destination CSV. + +I think doing a cell by cell approach like this, rather than just bulk copying all lines over, may result in less errors. + +What are your thoughts on this solution to our problem of too many length mismatches? diff --git a/bin/stage-6/prepare-stage-6.py b/bin/stage-6/prepare-stage-6.py new file mode 100755 index 0000000..b8ddba7 --- /dev/null +++ b/bin/stage-6/prepare-stage-6.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# Script Name: prepare-stage-6 + +import os +import shutil +import csv +from halo import Halo +from tqdm import tqdm +from pyfiglet import Figlet + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_5_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-5") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") +MERGED_CSV = os.path.join(STAGE_6_DIRECTORY, "01-first-merger.csv") + +HEADERS = [ + "Business Name", + "GBP Review Rating", + "GBP Review Count", + "GBP Business Category", + "YiB", + "GBP Matching Service", + "GBP Business Phone", + "GBP Business Website", + "GBP Location Municipality", + "GBP Location State", + "Root Domain", + "MD5 for Scrape" # Added the new column header +] + +def create_stage_6_directory(): + """Create the stage-6 directory, replacing it if it already exists.""" + if os.path.exists(STAGE_6_DIRECTORY): + print("Existing Stage 6 found, deleting and replacing...") + shutil.rmtree(STAGE_6_DIRECTORY) + os.makedirs(STAGE_6_DIRECTORY) + print("Stage 6 directory created.") + +def merge_csv_files(): + """Merge all CSV files from stage-5 into a single CSV file in stage-6.""" + with open(MERGED_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=HEADERS) + writer.writeheader() + + for state_dir in os.listdir(STAGE_5_DIRECTORY): + state_path = os.path.join(STAGE_5_DIRECTORY, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + with open(file_path, 'r') as infile: + reader = csv.DictReader(infile) + for row in reader: + data = {header: row.get(header, '').strip() for header in HEADERS} + writer.writerow(data) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "prepare-stage-6".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Preparing Stage 6...") + + create_stage_6_directory() + + print("Merging CSV files to Stage 6...") + spinner = Halo(text='Merging data', spinner='dots') + spinner.start() + merge_csv_files() + spinner.succeed("Data merged.") + + print("Stage 6 preparation complete.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/process-stage-3.md b/bin/stage-6/process-stage-3.md new file mode 100644 index 0000000..35e5c75 --- /dev/null +++ b/bin/stage-6/process-stage-3.md @@ -0,0 +1,166 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please refactor the process-stage-3 script below to be in accordance with the above. + +This script will not need to check for the existence of Stage 3, or worry about creating it. This task is handled elsewhere now. + +Please create the process-stage-3 in the same style as the process-stage-2 script below: + + +#!/usr/bin/env python +# Script Name: process-stage-2 + +import os +import subprocess +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +BIN_STAGE_2_DIR = os.path.join(PROJECT_ROOT, 'bin', 'stage-2') +BIN_DIR = os.path.join(PROJECT_ROOT, 'bin') + +def run_scripts(): + """Run the specified scripts in order.""" + scripts = [ + os.path.join(BIN_STAGE_2_DIR, "prepare-stage-2.py"), + os.path.join(BIN_STAGE_2_DIR, "remove-utm.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-review-count.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-phone-data.py"), + os.path.join(BIN_STAGE_2_DIR, "sanitize-gbp-business-website.py"), + os.path.join(BIN_DIR, "remove-empty-columns.py") + ] + + # Run scripts that do not require arguments + for script in scripts[:-1]: + subprocess.run(["python", script], check=True) + + # Run script that requires the 'stage-2' argument + subprocess.run(["python", scripts[-1], "stage-2"], check=True) + +def main(): + figlet = Figlet(font='slant') + script_name = "process-stage-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + print("Running scripts for Stage 2...\n") + run_scripts() + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("All scripts completed.") + +if __name__ == '__main__': + main() + + +Here is the current version of the process-stage-3 script which needs refactoring: + + +#!/usr/bin/env python + +import os +import shutil +import subprocess +from tqdm import tqdm +from pyfiglet import Figlet + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +data_dir = os.path.join(project_root, '.data') +stage2_dir = os.path.join(data_dir, 'stage-2') +stage3_dir = os.path.join(data_dir, 'stage-3') + +def check_and_prepare_stage3(): + # Check if the stage-3 directory exists + if os.path.exists(stage3_dir): + # Remove the existing stage-3 directory + shutil.rmtree(stage3_dir) + + # Copy stage-2 directory to stage-3 + shutil.copytree(stage2_dir, stage3_dir) + +def run_scripts(): + scripts = [ + "column-search_review-count.py", + "column-search_review-rating.py", + "column-search_gbp-location.py", + "column-search_yib.py", + "column-search_gbp-business-category.py", + "column-search_gbp-matching-services.py", + "column-search_business-name.py", + ("remove-empty-columns.py", "stage-3"), + "column-search_unknowns.py" + ] + + for script in scripts: + if isinstance(script, tuple): + script_name, arg = script + subprocess.run(["python", script_name, arg], check=True) + else: + subprocess.run(["python", script], check=True) + + # Call report-unknowns.py with the "stage-3" option + subprocess.run(["python", "report-unknowns.py", "stage-3"], check=True) + +if __name__ == '__main__': + check_and_prepare_stage3() + run_scripts() + + +Item 1: The process-stage-3 script will need to call a scrip named "prepare-stage-3" script as the first script it calls. + +Item 2: The following scripts which process-stage-3 calls are located in [Stage 3 Binaries]: + + +column-search_review-count.py +column-search_review-rating.py +column-search_gbp-location.py +column-search_yib.py +column-search_gbp-business-category.py +column-search_gbp-matching-services.py +column-search_business-name.py +column-search_unknowns.py + + +The following script is located in [Binaries]. + + +remove-empty-columns.py + diff --git a/bin/stage-6/re-order-columns.md b/bin/stage-6/re-order-columns.md new file mode 100644 index 0000000..3afa22d --- /dev/null +++ b/bin/stage-6/re-order-columns.md @@ -0,0 +1,86 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 're-order-columns', which will have as input the csv file "11-uuid-business-3-by-business-name.csv", and output will have the csv file "12-re-ordered-columns.csv". These csv files will be located in [Stage 6]. + +The first thing this script will need to is actually create a new column "Location UUID". This column should be created at the end of the rest of the columns in the sheet. Care should be taken that the column count remains consistent throughout the data as we add a new heading. + +Once this new heading is added we need to re-order the columns in the spreadsheet into a more preferred order. + +Here is the complete column order as it should appear in the CSV sheet: + + +Business Name +Business UUID +Location UUID +GBP Review Rating +GBP Review Count +YiB +GBP Business Category +GBP Matching Service +GBP Location Municipality +GBP Location State +GBP Business Phone +GBP Business Website +Root Domain + + + +# Prompt 2 +I missed a column named "Location from GBP Business Name" + +Here is an updated complete order list. Please adjust the script: + + +Business Name +Business UUID +Location UUID +GBP Review Rating +GBP Review Count +YiB +GBP Business Category +GBP Matching Service +GBP Location Municipality +GBP Location State +Location from GBP Business Name +GBP Business Phone +GBP Business Website +Root Domain + diff --git a/bin/stage-6/re-order-columns.py b/bin/stage-6/re-order-columns.py new file mode 100755 index 0000000..592a518 --- /dev/null +++ b/bin/stage-6/re-order-columns.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# Script Name: re-order-columns + +import os +import csv +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '11-uuid-business-3-by-business-name.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '12-re-ordered-columns.csv') + +# Define the correct column order +CORRECT_COLUMN_ORDER = [ + "Business Name", + "Business UUID", + "Location UUID", + "GBP Review Rating", + "GBP Review Count", + "YiB", + "GBP Business Category", + "GBP Matching Service", + "GBP Location Municipality", + "GBP Location State", + "Location from GBP Business Name", + "GBP Business Phone", + "GBP Business Website", + "Root Domain", + "MD5 for Scrape" +] + +def add_location_uuid_and_reorder_columns(): + """Add 'Location UUID' column and reorder columns.""" + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + + if "Location UUID" not in headers: + headers.append("Location UUID") + + # Ensure all expected columns are present + for column in CORRECT_COLUMN_ORDER: + if column not in headers: + headers.append(column) + + rows = list(reader) + + # Write the reordered columns to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=CORRECT_COLUMN_ORDER) + writer.writeheader() + for row in rows: + # Ensure consistent column count by adding the "Location UUID" if missing + if "Location UUID" not in row: + row["Location UUID"] = "" + writer.writerow(row) + +def main(): + figlet = Figlet(font='slant') + script_name = "re-order-columns".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Reordering columns and adding Location UUID', spinner='dots') + spinner.start() + add_location_uuid_and_reorder_columns() + spinner.succeed('Columns reordered and Location UUID added successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/services-gbp-optimization.md b/bin/stage-6/services-gbp-optimization.md new file mode 100644 index 0000000..fec195b --- /dev/null +++ b/bin/stage-6/services-gbp-optimization.md @@ -0,0 +1,49 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I would like to make a new script named "services-gbp-optimization". As input this script will use the CSV file "18-services-gbp-reputation-management.csv", and as output it will use the CSV file "19-services-gbp-optimization.csv". + +This script will create a new column named "Services: GBP Optimization". + +The script will set the value of this column for every row to either "Yes" or "No". + +If a row has data missing in either the "GBP Matching Service" or "GBP Location Municipality" columns, we will set the value of "Services: GBP Optimization" to "Yes". + +Otherwise if a business has both the "GBP Matching Service" and "GBP Location Municipality" columns containing data, we can set the value of "Services: GBP Optimization" to "No". diff --git a/bin/stage-6/services-gbp-optimization.py b/bin/stage-6/services-gbp-optimization.py new file mode 100755 index 0000000..fff46fa --- /dev/null +++ b/bin/stage-6/services-gbp-optimization.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Script Name: services-gbp-optimization + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") + +INPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "18-services-gbp-reputation-management.csv") +OUTPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "19-services-gbp-optimization.csv") + +def process_file(): + """Process the input CSV file and write the output CSV file with the new column.""" + with open(INPUT_FILE, 'r') as infile: + reader = csv.reader(infile) + rows = list(reader) + + if not rows: + print("Input file is empty.") + sys.exit(1) + + headers = rows[0] + data_rows = rows[1:] + + new_column = "Services: GBP Optimization" + headers.append(new_column) + + # Indices of relevant columns + matching_service_index = headers.index("GBP Matching Service") + municipality_index = headers.index("GBP Location Municipality") + + for row in data_rows: + matching_service = row[matching_service_index] + municipality = row[municipality_index] + + if not matching_service or not municipality: + row.append("Yes") + else: + row.append("No") + + with open(OUTPUT_FILE, 'w', newline='') as outfile: + writer = csv.writer(outfile) + writer.writerow(headers) + writer.writerows(data_rows) + +def main(): + figlet = Figlet(font='slant') + script_name = "services-gbp-optimization".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_FILE): + print(f"Error: The input file {INPUT_FILE} does not exist.") + sys.exit(1) + + print(f"Processing file {INPUT_FILE} and generating output {OUTPUT_FILE}...") + spinner = Halo(text='Processing', spinner='dots') + spinner.start() + + try: + process_file() + spinner.succeed("File processed successfully.") + except Exception as e: + spinner.fail(f"Error: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/services-gbp-reputation-management.md b/bin/stage-6/services-gbp-reputation-management.md new file mode 100644 index 0000000..bb21cd1 --- /dev/null +++ b/bin/stage-6/services-gbp-reputation-management.md @@ -0,0 +1,161 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I would like to create a new script named 'services-gbp-reputation-management'. It will use the CSV file 17-uniqify-pass-2.csv as the input, and as the output it will use the CSV file 18-services-gbp-reputation-management.csv + +This script needs to add the following new columns to the output CSV file: + + +GBP Industry Avg Review Rating +GBP Industry Avg Review Count +GBP Review Rating Offset fr. Avg +GBP Review Count Offset fr. Avg +GBP Review Rating Health +GBP Review Count Health +Services: Reputation Management + + +After these columns have been added to the spreadsheet, we now need to populate them with data by doing a little data processing. + +I would like each of these columns to be filled in by the script one column at a time. So we worry about the first column that needs data, and do what we need to in order to fill in that cell's data for every row. Then after that is done, we move on to the next column which needs processing, and so forth. + +Many of these instructions will involve calculating averages, and when doing so it is important that we ignore empty cells when doing these calculations. If we are calculatint the average number of reviews, we don't include the count of 0 for businesses in that industry with no reviews when we are calculating the average. + + +Here is how to populate the "GBP Industry Avg Review Rating": + +This column is calculated by determining the average "GBP Review Rating". Which will always be a number 1-5 with a decimal. Such as "#.#". Some examples might be "5.0, 4.9, 3.5, 2.1" etc. + +We need to determine the average "GBP Review Rating" for every group of rows that have the same "GBP Business Category" as one another. + +Each row will have a "GBP Business Category", and that is our industry we are calculating the average for. + +When we have that average calculated we use it to populate the "GBP Industry Avg Review Rating" cell for all rows in that industry. + + +Here is how to populate the "GBP Industry Avg Review Count": + +This is the same thing as the "GBP Industry Avg Review Rating", except we are calculating the average in the "GBP Review Count" column instead of the "GBP Review Rating" column. + + +Here is how to populate the "GBP Review Rating Offset fr. Avg" column: + +This column would be the difference between the "GBP Industry Avg Review Rating" and the rows "GBP Review Rating" column. It should be presented as prefixed with a positive or negative depending on whether the offset is positive or negative. + +Here is how to populate the "GBP Review Count Offset fr. Avg" column: + +This column is the same thing as the "GBP Review Rating Offset fr. Avg" column, excep that we are determining the value based upon the difference beween the "GBP Industry Avg Review Count" column and the "GBP Review Count" column instead. + + +Here is how to populate the "GBP Review Rating Health" column: +This is going to be either "Above Average" or "Below Average" depending on whether or not the rows "GBP Review Rating Offset fr. Avg" is a positive or negative number. Below Average is a negative number, Above Average is a positive number. + +Here is how to populate the "GBP Review Count Health" column: +This is the same thing as the "GBP Review Rating Health" column, but we are basing the "Above Average" and "Below Average" declaration upon the rows "GBP Review Count Offset fr. Avg" column instead. + +Here is how to populate the "Services: Reputation Management" column: +If either the "GBP Review Rating Health" or "GBP Review Count Health" is "Below "Average" this column should be set to "Yes". + +If both the "GBP Review Rating Health" and "GBP Review Count Health" is "Above Average" this column should be set to "No". + + +# Prompt 2 +I got the following error running the script: + + +Processing file /home/ld/mgk-scrapes/current-data/.data/stage-6/17-uniqify-pass-2.csv and generating output /home/ld/mgk-scrapes/current-data/.data/stage-6/18-services-gbp-reputation-management.csv... +✖ Error: mean requires at least one data point + + + +# Prompt 3 +Ok this is working nicely now. + +There are some changes I would like to make though still. + +Change 1: I would like the averages to only be calculated to the hundredths. + +Change 2: I would like the column order to be resorted as a final task. + +I would like the final column order for the output CSV to be the following list, in the order I give them: + + +Business Name +Business UUID +Location UUID +YiB +GBP Business Category +GBP Matching Service +GBP Review Rating +GBP Review Count +GBP Industry Avg Review Rating +GBP Industry Avg Review Count +GBP Review Rating Offset fr. Avg +GBP Review Count Offset fr. Avg +GBP Review Rating Health +GBP Review Count Health +GBP Location Municipality +GBP Location State +Location from GBP Business Name +GBP Business Phone +GBP Business Website +Root Domain +Services: Reputation Management + + + +# Prompt 4 +I ran into a problem with the script. + +Some of the entries in the "GBP Review Count Offset fr. Avg" columns have more than 2 decimal places. This should only be shown to the thousandths, or two decimal places. + + +# Prompt 5 +Ok that decimal place issue is fixed. + +There is one more issue I would like to address as well. + +This new issue is that I would like the "GBP Review Rating Offset fr. Avg" and the "GBP Review Count Offset fr. Avg" columns to be populated for rows that have no "GBP Review Rating" or "GBP Review Count" if we do in fact have an industry average value calculated for that rows industry. + +I think right now that some businesses have no "GBP Review Rating", or "GBP Review Count", so they were not assigned their "GBP Review Rating Offset fr. Avg" or "GBP Review Count Offset fr. Avg" cells. + +If a row is a part of a match group that we were able to calculate a "GBP Industry Avg Review Rating" or "GBP Industry Review Count" for, and that row is also missing data in the "GBP Review Rating" or "GBP Review Count" columns, we should count the "GBP Review Rating" and "GBP Review Count" for that row as 0. And we can determine the "GBP Review Rating Offset fr. Avg" and "GBP Review Count Offset fr. Avg" fields based upon that 0. + +If a row is a part of a match group we were not able to calculate a "GBP Industry Avg Review Rating" or "GBP Industry Review Count" for at all, then we can skip assigning that row either the "GBP Review Count Offset fr. Avg" or "GBP Review Rating Offset fr. Avg" entries for that row. diff --git a/bin/stage-6/services-gbp-reputation-management.py b/bin/stage-6/services-gbp-reputation-management.py new file mode 100755 index 0000000..feff033 --- /dev/null +++ b/bin/stage-6/services-gbp-reputation-management.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# Script Name: services-gbp-reputation-management + +import os +import csv +import sys +from statistics import mean +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") + +INPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "17-uniqify-pass-2.csv") +OUTPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "18-services-gbp-reputation-management.csv") + +def calculate_averages(rows, category_index, rating_index, count_index): + """Calculate the industry averages for review rating and review count.""" + category_ratings = {} + category_counts = {} + + for row in rows: + category = row[category_index] + try: + rating = float(row[rating_index]) if row[rating_index] else None + count = int(row[count_index].replace(',', '')) if row[count_index] else None + except ValueError: + continue + + if category not in category_ratings: + category_ratings[category] = [] + if category not in category_counts: + category_counts[category] = [] + + if rating is not None: + category_ratings[category].append(rating) + if count is not None: + category_counts[category].append(count) + + avg_ratings = {category: round(mean(ratings), 2) if ratings else None for category, ratings in category_ratings.items()} + avg_counts = {category: round(mean(counts), 2) if counts else None for category, counts in category_counts.items()} + + return avg_ratings, avg_counts + +def process_file(): + """Process the input CSV file and write the output CSV file with the new columns.""" + with open(INPUT_FILE, 'r') as infile: + reader = csv.reader(infile) + rows = list(reader) + + if not rows: + print("Input file is empty.") + sys.exit(1) + + headers = rows[0] + data_rows = rows[1:] + + new_columns = [ + "GBP Industry Avg Review Rating", + "GBP Industry Avg Review Count", + "GBP Review Rating Offset fr. Avg", + "GBP Review Count Offset fr. Avg", + "GBP Review Rating Health", + "GBP Review Count Health", + "Services: Reputation Management" + ] + + headers.extend(new_columns) + + # Indices of relevant columns + category_index = headers.index("GBP Business Category") + rating_index = headers.index("GBP Review Rating") + count_index = headers.index("GBP Review Count") + + # Calculate industry averages + avg_ratings, avg_counts = calculate_averages(data_rows, category_index, rating_index, count_index) + + for row in data_rows: + category = row[category_index] + try: + rating = float(row[rating_index]) if row[rating_index] else 0 + count = int(row[count_index].replace(',', '')) if row[count_index] else 0 + except ValueError: + rating = 0 + count = 0 + + avg_rating = avg_ratings.get(category, None) + avg_count = avg_counts.get(category, None) + + # Calculate new column values + rating_offset = round(rating - avg_rating, 2) if avg_rating is not None else "" + count_offset = round(count - avg_count, 2) if avg_count is not None else "" + + rating_health = "Above Average" if isinstance(rating_offset, float) and rating_offset > 0 else "Below Average" + count_health = "Above Average" if isinstance(count_offset, float) and count_offset > 0 else "Below Average" + + reputation_management = "Yes" if rating_health == "Below Average" or count_health == "Below Average" else "No" + + # Add new column values to the row + row.extend([ + avg_rating if avg_rating is not None else "", + avg_count if avg_count is not None else "", + f"{rating_offset:+.2f}" if isinstance(rating_offset, float) else "", + f"{count_offset:+.2f}" if isinstance(count_offset, float) else "", + rating_health if avg_rating is not None else "", + count_health if avg_count is not None else "", + reputation_management if avg_rating is not None or avg_count is not None else "" + ]) + + # Define final column order + final_columns = [ + "Business Name", + "Business UUID", + "Location UUID", + "YiB", + "GBP Business Category", + "GBP Matching Service", + "GBP Review Rating", + "GBP Review Count", + "GBP Industry Avg Review Rating", + "GBP Industry Avg Review Count", + "GBP Review Rating Offset fr. Avg", + "GBP Review Count Offset fr. Avg", + "GBP Review Rating Health", + "GBP Review Count Health", + "GBP Location Municipality", + "GBP Location State", + "Location from GBP Business Name", + "GBP Business Phone", + "GBP Business Website", + "Root Domain", + "MD5 for Scrape", + "Services: Reputation Management" + ] + + # Sort data rows to match the final column order + final_data_rows = [] + for row in data_rows: + final_row = [row[headers.index(col)] if col in headers else "" for col in final_columns] + final_data_rows.append(final_row) + + with open(OUTPUT_FILE, 'w', newline='') as outfile: + writer = csv.writer(outfile) + writer.writerow(final_columns) + writer.writerows(final_data_rows) + +def main(): + figlet = Figlet(font='slant') + script_name = "services-gbp-reputation-management".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_FILE): + print(f"Error: The input file {INPUT_FILE} does not exist.") + sys.exit(1) + + print(f"Processing file {INPUT_FILE} and generating output {OUTPUT_FILE}...") + spinner = Halo(text='Processing', spinner='dots') + spinner.start() + + try: + process_file() + spinner.succeed("File processed successfully.") + except Exception as e: + spinner.fail(f"Error: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/services-needs-website.md b/bin/stage-6/services-needs-website.md new file mode 100644 index 0000000..e4115e0 --- /dev/null +++ b/bin/stage-6/services-needs-website.md @@ -0,0 +1,49 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I would like to make a new script named "services-needs-website". As input this script will use the CSV file "19-services-gbp-optimization.csv", and as output it will use the CSV file "20-services-needs-website.csv". + +This script will create a new column named "Services: Needs Website". + +The script will set the value of this column for every row to either "Yes" or "No". + +If a row has data missing in the "Root Domain" column, we will set the value of "Services: Needs Website" to "Yes". + +Otherwise we will set the value of "Services: Needs Website" to "No". diff --git a/bin/stage-6/services-needs-website.py b/bin/stage-6/services-needs-website.py new file mode 100755 index 0000000..30cabde --- /dev/null +++ b/bin/stage-6/services-needs-website.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Script Name: services-needs-website + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") + +INPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "19-services-gbp-optimization.csv") +OUTPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "20-services-needs-website.csv") + +def process_file(): + """Process the input CSV file and write the output CSV file with the new column.""" + with open(INPUT_FILE, 'r') as infile: + reader = csv.reader(infile) + rows = list(reader) + + if not rows: + print("Input file is empty.") + sys.exit(1) + + headers = rows[0] + data_rows = rows[1:] + + new_column = "Services: Needs Website" + headers.append(new_column) + + # Index of the relevant column + root_domain_index = headers.index("Root Domain") + + for row in data_rows: + root_domain = row[root_domain_index] + + if not root_domain: + row.append("Yes") + else: + row.append("No") + + with open(OUTPUT_FILE, 'w', newline='') as outfile: + writer = csv.writer(outfile) + writer.writerow(headers) + writer.writerows(data_rows) + +def main(): + figlet = Figlet(font='slant') + script_name = "services-needs-website".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_FILE): + print(f"Error: The input file {INPUT_FILE} does not exist.") + sys.exit(1) + + print(f"Processing file {INPUT_FILE} and generating output {OUTPUT_FILE}...") + spinner = Halo(text='Processing', spinner='dots') + spinner.start() + + try: + process_file() + spinner.succeed("File processed successfully.") + except Exception as e: + spinner.fail(f"Error: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/services-website-audit.md b/bin/stage-6/services-website-audit.md new file mode 100644 index 0000000..00c5a0d --- /dev/null +++ b/bin/stage-6/services-website-audit.md @@ -0,0 +1,127 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I would like to make a new script named "services-website-audit". As input this script will use the CSV file "20-services-needs-website.csv", and as output it will use the CSV file "21-website-audit.csv". + +The script should first add some new columns to the output CSV file: + + +Services: Needs SSL +Services: SSL Repair +Services: Website Repair +Performance Score +Accessibility Score +Best Practices Score +SEO Score +Services: Website Speed Optimization + + +After the script has created these columns, it will be responsible for filling in the details of each. + +To do this we will be doing some tests on remote websites that we find in the "Root Domain" column of the CSV file. + +For every unique "Root Domain" we will perform a few tests, and use the results of those tests to populate the columns we just created. + +After we have gathered the data to insert into the new columns, we need to be sure that every duplicate copy of that "Root Domain" also gets the same data inserted into their rows. The CSV file will have many entries that share a "Root Domain", and I need all of those entries to have this data, but we only want to run these tests once. + +The "Services: Needs SSL", "Services: SSL Repair" and "Services: Website Repair" column is the first test. + +We need to contact the "Root Domain" we are testing, and see if the page loads successfully. It is ok if the page redirects first, but if we detect that the page loaded successfully, we will set "Services: Website Repair" to "No". If however trying to contact the "Root Domain" results in an error, and the page does not load successfully, then we will set the "Services: Website Repair" to "Yes". + +During this same test, we need to attempt to connect to the "Root Domain" using HTTPS, and verify that HTTPS works correctly. If the website does not support HTTPS, then we need to set "Services: Needs SSL" to "Yes". Otherwise if the site loads HTTPS just fine, we can set "Services: Needs SSL" to "No". + +If the "Root Domain" loads the HTTPS version of the website correctly, then we have one more test to do. And that is to attempt to connect to the website using HTTP instead of HTTPS. A correctly configured website will redirect the HTTP request to an HTTPS version and load the website. However, an incorrectly configured website will allow the HTTP connection to load as well as the HTTPS version. + +If our attempt to connect to the website using standard HTTP does not result in the connection getting redirected to an HTTPS version of the website then we can set the "Services: SSL Repair" to "Yes". Otherwise if our HTTP connection results in a redirect to the HTTPS version of the site we can set the "Services: SSL Repair" column for the row to "No". + +If we set "Services: Website Repair" to "No" with our prior test, we can proceed to fill in the remaining 5 columns: + + +Performance Score +Accessibility Score +Best Practices Score +SEO Score +Services: Website Speed Optimization + + +These columns will be filled in by using the Lighthouse CLI, and running that test against the "Root Domain". + +Earlier, I was given this example by you on how the Lighthouse CLI could be used: + + +import requests + +API_KEY = 'YOUR_API_KEY' +URL = 'https://example.com' + +def get_page_speed_insights(url, api_key): + api_url = f'https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url={url}&key={api_key}' + response = requests.get(api_url) + return response.json() + +def analyze_page_speed(data): + # Extract relevant metrics and suggestions + fcp = data['lighthouseResult']['audits']['first-contentful-paint']['displayValue'] + lcp = data['lighthouseResult']['audits']['largest-contentful-paint']['displayValue'] + suggestions = data['lighthouseResult']['categories']['performance']['score'] + + print(f"First Contentful Paint: {fcp}") + print(f"Largest Contentful Paint: {lcp}") + print(f"Performance Score: {suggestions}") + + # Determine if optimization is needed based on thresholds + if suggestions < 0.9: + print("Page needs optimization") + else: + print("Page is well optimized") + +if __name__ == '__main__': + data = get_page_speed_insights(URL, API_KEY) + analyze_page_speed(data) + + + +I would like something like that done for each entry "Root Domain" we already successfully connected to once. + +In that example lighthouse script, we have a threshold test that will print out either "Page needs optimization" or "Page is well optimized". + +I would like that same threshold system to be used to deterimine what we put into the "Services: Website Speed Optimization" field. If the page is determined to be "Page is well optimized" then I want to set "Services: Website Speed Optimization" to "No". However if the page is detected as "Page needs optimization", then I want the "Services: Website Speed Optimization" to be set to "Yes". + +I would like this script to be threaded, and to attempt to connect to 100 "Root Domains" at a time to do its tests, so I do not have to wait all day for results. diff --git a/bin/stage-6/services-website-audit.py b/bin/stage-6/services-website-audit.py new file mode 100755 index 0000000..5cb56de --- /dev/null +++ b/bin/stage-6/services-website-audit.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +# Script Name: services-website-audit + +import os +import csv +import requests +import subprocess +import threading +import json +from queue import Queue +from pyfiglet import Figlet +from halo import Halo +from tqdm import tqdm + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") + +INPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "20-services-needs-website.csv") +OUTPUT_FILE = os.path.join(STAGE_6_DIRECTORY, "21-website-audit.csv") + +# Path to the locally installed Lighthouse CLI +LIGHTHOUSE_PATH = os.path.join(PROJECT_ROOT, "bin", "node_modules", ".bin", "lighthouse") + +# Initialize a lock for threading +lock = threading.Lock() + +# Lighthouse command +LIGHTHOUSE_CMD = LIGHTHOUSE_PATH + " {url} --output=json --output-path={output_path} --quiet --no-enable-error-reporting --chrome-flags='--headless'" + +def get_lighthouse_data(url): + """Run Lighthouse CLI and return the parsed JSON data.""" + output_path = "lighthouse_report.json" + cmd = LIGHTHOUSE_CMD.format(url=url, output_path=output_path) + try: + subprocess.run(cmd, shell=True, check=True) + with open(output_path, 'r') as f: + data = json.load(f) + os.remove(output_path) + return data + except subprocess.CalledProcessError: + return None + +def analyze_lighthouse_data(data): + """Extract relevant metrics from Lighthouse JSON data.""" + performance_score = data['categories']['performance']['score'] + accessibility_score = data['categories']['accessibility']['score'] + best_practices_score = data['categories']['best-practices']['score'] + seo_score = data['categories']['seo']['score'] + + return { + "Performance Score": performance_score, + "Accessibility Score": accessibility_score, + "Best Practices Score": best_practices_score, + "SEO Score": seo_score, + "Services: Website Speed Optimization": "Yes" if performance_score < 0.9 else "No" + } + +def test_ssl_and_repair(domain): + """Test SSL and website repair needs.""" + try: + response = requests.get(f"https://{domain}", timeout=10) + ssl_needs = "No" + website_repair = "No" + except requests.exceptions.RequestException: + ssl_needs = "Yes" + website_repair = "Yes" + + ssl_repair = "No" + if ssl_needs == "No": + try: + response = requests.get(f"http://{domain}", timeout=10, allow_redirects=False) + if response.status_code == 200: + ssl_repair = "Yes" + except requests.exceptions.RequestException: + ssl_repair = "Yes" + + return ssl_needs, ssl_repair, website_repair + +def process_domain(domain, results, progress_bar): + """Process a single domain to gather all required data.""" + if not domain.startswith(("http://", "https://")): + domain = "http://" + domain + + ssl_needs, ssl_repair, website_repair = test_ssl_and_repair(domain) + + if website_repair == "No": + lighthouse_data = get_lighthouse_data(domain) + if lighthouse_data: + lighthouse_results = analyze_lighthouse_data(lighthouse_data) + else: + lighthouse_results = { + "Performance Score": "", + "Accessibility Score": "", + "Best Practices Score": "", + "SEO Score": "", + "Services: Website Speed Optimization": "" + } + else: + lighthouse_results = { + "Performance Score": "", + "Accessibility Score": "", + "Best Practices Score": "", + "SEO Score": "", + "Services: Website Speed Optimization": "" + } + + with lock: + results[domain] = { + "Services: Needs SSL": ssl_needs, + "Services: SSL Repair": ssl_repair, + "Services: Website Repair": website_repair, + **lighthouse_results + } + progress_bar.update(1) + +def worker(queue, results, progress_bar): + """Worker function for threading.""" + while True: + domain = queue.get() + if domain is None: + break + process_domain(domain, results, progress_bar) + queue.task_done() + +def main(): + figlet = Figlet(font='slant') + script_name = "services-website-audit".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_FILE): + print(f"Error: The input file {INPUT_FILE} does not exist.") + sys.exit(1) + + with open(INPUT_FILE, 'r') as infile: + reader = csv.reader(infile) + rows = list(reader) + + headers = rows[0] + data_rows = rows[1:] + + new_columns = [ + "Services: Needs SSL", + "Services: SSL Repair", + "Services: Website Repair", + "Performance Score", + "Accessibility Score", + "Best Practices Score", + "SEO Score", + "Services: Website Speed Optimization" + ] + + headers.extend(new_columns) + + # Gather all unique domains + domains = set(row[headers.index("Root Domain")] for row in data_rows if row[headers.index("Root Domain")]) + + # Dictionary to hold results + results = {} + + # Threading setup + queue = Queue() + num_threads = 100 + threads = [] + + # Progress bar setup + progress_bar = tqdm(total=len(domains), desc="Processing domains", unit="domain") + + for _ in range(num_threads): + t = threading.Thread(target=worker, args=(queue, results, progress_bar)) + t.start() + threads.append(t) + + for domain in domains: + queue.put(domain) + + queue.join() + + for _ in range(num_threads): + queue.put(None) + for t in threads: + t.join() + + progress_bar.close() + + for row in data_rows: + domain = row[headers.index("Root Domain")] + if domain in results: + result = results[domain] + row.extend([ + result["Services: Needs SSL"], + result["Services: SSL Repair"], + result["Services: Website Repair"], + result["Performance Score"], + result["Accessibility Score"], + result["Best Practices Score"], + result["SEO Score"], + result["Services: Website Speed Optimization"] + ]) + + with open(OUTPUT_FILE, 'w', newline='') as outfile: + writer = csv.writer(outfile) + writer.writerow(headers) + writer.writerows(data_rows) + + spinner = Halo(text='Processing complete', spinner='dots') + spinner.succeed("File processed successfully.") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/standardize-capital-letters.md b/bin/stage-6/standardize-capital-letters.md new file mode 100644 index 0000000..ac94c86 --- /dev/null +++ b/bin/stage-6/standardize-capital-letters.md @@ -0,0 +1,166 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'standardize-capital-letters' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage6_dir = os.path.join(project_root, '.data', 'stage-6') +csv_path = os.path.join(stage6_dir, '02-initial-deduplication.csv') + +def title_case(text): + return text.title() + +def standardize_capital_letters(): + with open(csv_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + headers = reader.fieldnames + + # Ensure that the required columns exist + if 'GBP Business Category' not in headers or 'GBP Matching Service' not in headers: + print("Error: Missing required columns.") + return + + rows = [] + for row in reader: + # Normalize capitalization for the specified columns + row['GBP Business Category'] = title_case(row.get('GBP Business Category', '').strip()) + row['GBP Matching Service'] = title_case(row.get('GBP Matching Service', '').strip()) + rows.append(row) + + # Write the modified rows back to the original CSV file + with open(csv_path, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + writer.writerows(rows) + + print(f"Capitalization standardized for columns 'GBP Business Category' and 'GBP Matching Service' in the file {csv_path}") + +if __name__ == '__main__': + standardize_capital_letters() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +The 'standardize-capital-letters' script should use as input, the csv file: "04-deleted-bad-matching-services.csv". And as output, the csv file: "05-standardized-capital-letters.csv". + diff --git a/bin/stage-6/standardize-capital-letters.py b/bin/stage-6/standardize-capital-letters.py new file mode 100755 index 0000000..607c74c --- /dev/null +++ b/bin/stage-6/standardize-capital-letters.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# Script Name: standardize-capital-letters + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") +INPUT_CSV_PATH = os.path.join(STAGE_6_DIRECTORY, '04-deleted-bad-matching-services.csv') +OUTPUT_CSV_PATH = os.path.join(STAGE_6_DIRECTORY, '05-standardized-capital-letters.csv') + +def title_case(text): + """Convert text to title case.""" + return text.title() + +def process_csv_file_standardize_capitals(input_path, output_path): + """Standardize capitalization for specific columns in the CSV file.""" + with open(input_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + headers = reader.fieldnames + + if 'GBP Business Category' not in headers or 'GBP Matching Service' not in headers: + print("Error: Missing required columns.") + sys.exit(1) + + rows = [] + for row in reader: + row['GBP Business Category'] = title_case(row.get('GBP Business Category', '').strip()) + row['GBP Matching Service'] = title_case(row.get('GBP Matching Service', '').strip()) + rows.append(row) + + with open(output_path, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + writer.writerows(rows) + + return len(rows) + +def main(): + figlet = Figlet(font='slant') + script_name = "standardize-capital-letters".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_CSV_PATH): + print(f"Error: The file {INPUT_CSV_PATH} does not exist.") + sys.exit(1) + + print(f"Processing {INPUT_CSV_PATH}...") + spinner = Halo(text='Standardizing capital letters', spinner='dots') + spinner.start() + total_rows = process_csv_file_standardize_capitals(INPUT_CSV_PATH, OUTPUT_CSV_PATH) + spinner.succeed(f"Standardization complete. Total rows processed: {total_rows}") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/standardize-yib.mb b/bin/stage-6/standardize-yib.mb new file mode 100644 index 0000000..765d062 --- /dev/null +++ b/bin/stage-6/standardize-yib.mb @@ -0,0 +1,61 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 'standardize-yib' which will use the csv file "14-uuid-business-4-single-location-entries.csv" as input, and will output to the csv file "15-yib-standardized.csv" inside of [Stage 6]. + +This script is intended to iterate through match groups of rows that have the "Business UUID" column in common. + +Rows with the same "Business UUID" should all have the same "YiB". Which stands for "Years in Business.". + +The "YiB" column is either empty, or contains a string such as: + + +5+ years in business +3+ years in business +10+ years in business + + +It is always in the format of "# years in business" where # is a number. + +What I need this script to do is iterate through all the YiB values in each match group, and to determine the YiB with the largest number. + +That YiB then needs to copied over to the YiB column for all other entries in that match group. Standardizing the YiB value in that match group. Setting each row to have the same years in business as all other members. + diff --git a/bin/stage-6/standardize-yib.py b/bin/stage-6/standardize-yib.py new file mode 100755 index 0000000..140e31a --- /dev/null +++ b/bin/stage-6/standardize-yib.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# Script Name: standardize-yib + +import os +import csv +import re +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '14-uuid-business-4-single-location-entries.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '15-yib-standardized.csv') + +def extract_years_in_business(yib_str): + """Extract the number of years from the YiB string.""" + match = re.search(r'(\d+)', yib_str) + if match: + return int(match.group(1)) + return 0 + +def standardize_yib(): + """Standardize YiB values for each match group by Business UUID.""" + business_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Business UUID + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business UUID' not in headers or 'YiB' not in headers: + print("Error: 'Business UUID' or 'YiB' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + business_uuid = row['Business UUID'].strip() + if business_uuid: + business_to_rows[business_uuid].append(row) + + print(f"Total groups by Business UUID: {len(business_to_rows)}") + + for business_uuid, group in business_to_rows.items(): + max_yib = 0 + for row in group: + yib_str = row['YiB'].strip() + if yib_str: + max_yib = max(max_yib, extract_years_in_business(yib_str)) + + standardized_yib = f"{max_yib}+ years in business" if max_yib > 0 else '' + + for row in group: + row['YiB'] = standardized_yib + + print(f"Total groups standardized: {len(business_to_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "standardize-yib".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Standardizing YiB values by Business UUID', spinner='dots') + spinner.start() + standardize_yib() + spinner.succeed('YiB standardization completed successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/tally-initial-deduplication.py b/bin/stage-6/tally-initial-deduplication.py new file mode 100755 index 0000000..9514ca2 --- /dev/null +++ b/bin/stage-6/tally-initial-deduplication.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import os +import csv + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage6_dir = os.path.join(project_root, '.data', 'stage-6') +deduplicated_csv_path = os.path.join(stage6_dir, '02-initial-deduplication.csv') +data_dir = os.path.join(project_root, '.data') +categories_file = os.path.join(data_dir, 'gbp-business-categories.txt') + +def load_categories(): + with open(categories_file, 'r') as file: + categories = [line.strip().lower() for line in file if line.strip()] + return categories + +def tally_records_by_category(): + categories = load_categories() + category_counts = {category: 0 for category in categories} + total_records = 0 + + with open(deduplicated_csv_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + for row in reader: + total_records += 1 + category = row.get('GBP Business Category', '').strip().lower() + if category in category_counts: + category_counts[category] += 1 + + return category_counts, total_records + +def print_tally_results(category_counts, total_records): + for category, count in category_counts.items(): + print(f'Records in category "{category}": {count}') + print(f'Total Records: {total_records}') + +if __name__ == '__main__': + category_counts, total_records = tally_records_by_category() + print_tally_results(category_counts, total_records) + diff --git a/bin/stage-6/trim-invalid-rows-from-merger.md b/bin/stage-6/trim-invalid-rows-from-merger.md new file mode 100644 index 0000000..0bd1bc2 --- /dev/null +++ b/bin/stage-6/trim-invalid-rows-from-merger.md @@ -0,0 +1,172 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +Please be sure to reference project details above when factoring code, or answering questions. + +I wish to refactor the script 'trim-invalid-rows-from-merger' to be in accordance with the above, and to adjust the reporting style now: + + +#!/usr/bin/env python + +import os +import csv +from tqdm import tqdm + +# Get the project root directory +project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'Concrete Sealing Company')) +stage6_dir = os.path.join(project_root, '.data', 'stage-6') +merged_csv_path = os.path.join(stage6_dir, '01-first-merger.csv') + +def trim_invalid_rows_from_merger(): + valid_rows = [] + total_rows_before = 0 + + with open(merged_csv_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + headers = reader.fieldnames + rows = list(reader) + total_rows_before = len(rows) + + # Use tqdm to display a progress bar + for row in tqdm(rows, desc="Processing rows", unit="row"): + business_name = row.get('Business Name', '').strip() + business_phone = row.get('GBP Business Phone', '').strip() + business_website = row.get('GBP Business Website', '').strip() + + if business_name and (business_phone or business_website): + valid_rows.append(row) + + # Write the valid rows back to the original CSV file + with open(merged_csv_path, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + writer.writerows(valid_rows) + + # Calculate the total number of rows removed + total_rows_after = len(valid_rows) + total_rows_removed = total_rows_before - total_rows_after + + print(f'Total rows removed: {total_rows_removed}') + +if __name__ == '__main__': + trim_invalid_rows_from_merger() + + +Please make this script have the same kind of by-state reporting as the 'remove-utm' script: + + +#!/usr/bin/env python +# Script Name: remove-utm + +import os +import csv +import sys +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +CURRENT_DATASET = os.path.join(PROJECT_ROOT, "current-data") +DATA_DIRECTORY = os.path.join(CURRENT_DATASET, ".data") +STAGE_2_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-2") + +def process_csv_file_remove_utm(file_path): + """Remove UTM parameters from URLs in the given CSV file.""" + with open(file_path, 'r') as file: + reader = csv.reader(file) + rows = list(reader) + + if not rows: + return + + modified_rows = [] + for row in rows: + modified_row = [] + for cell in row: + if "?utm" in cell: + modified_cell = cell.split("?utm")[0] + else: + modified_cell = cell + modified_row.append(modified_cell) + modified_rows.append(modified_row) + + with open(file_path, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerows(modified_rows) + +def process_csv_files(stage_directory): + """Process all CSV files in the given stage directory to remove UTM parameters from URLs.""" + for state_dir in os.listdir(stage_directory): + state_path = os.path.join(stage_directory, state_dir) + if os.path.isdir(state_path): + spinner = Halo(text=f'Processing {state_dir}', spinner='dots') + spinner.start() + for county_dir in os.listdir(state_path): + county_path = os.path.join(state_path, county_dir) + if os.path.isdir(county_path): + csv_files = [file for file in os.listdir(county_path) if file.endswith('.csv')] + for file in csv_files: + file_path = os.path.join(county_path, file) + process_csv_file_remove_utm(file_path) + spinner.succeed(f'Finished processing {state_dir}') + +def main(): + figlet = Figlet(font='slant') + script_name = "remove-utm".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(STAGE_2_DIRECTORY): + print(f"Error: The directory {STAGE_2_DIRECTORY} does not exist.") + sys.exit(1) + + print(f"Removing UTM parameters in {STAGE_2_DIRECTORY}...") + process_csv_files(STAGE_2_DIRECTORY) + + final_spinner = Halo(spinner='dots', color='green') + final_spinner.start() + final_spinner.succeed("UTM parameters removed.") + +if __name__ == "__main__": + main() + + + +# Prompt 2 +I forget to mention that 'trim-invalid-rows-from-merger' should use as input, the csv file: "02-pre-sorted.csv". And as output, the csv file: "03-trimmed-invalid-rows.csv". + +Also, this script can just make use of halo. It does need a tqdm progress bar. diff --git a/bin/stage-6/trim-invalid-rows-from-merger.py b/bin/stage-6/trim-invalid-rows-from-merger.py new file mode 100755 index 0000000..0388918 --- /dev/null +++ b/bin/stage-6/trim-invalid-rows-from-merger.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# Script Name: trim-invalid-rows-from-merger + +import os +import csv +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +DATA_DIRECTORY = os.path.join(PROJECT_ROOT, "current-data", ".data") +STAGE_6_DIRECTORY = os.path.join(DATA_DIRECTORY, "stage-6") +INPUT_CSV_PATH = os.path.join(STAGE_6_DIRECTORY, '02-pre-sorted.csv') +OUTPUT_CSV_PATH = os.path.join(STAGE_6_DIRECTORY, '03-trimmed-invalid-rows.csv') + +def process_csv_file_trim_invalid_rows(input_path, output_path): + """Trim invalid rows from the CSV file.""" + valid_rows = [] + total_rows_before = 0 + + with open(input_path, 'r') as csv_file: + reader = csv.DictReader(csv_file) + headers = reader.fieldnames + rows = list(reader) + total_rows_before = len(rows) + + for row in rows: + business_name = row.get('Business Name', '').strip() + business_phone = row.get('GBP Business Phone', '').strip() + business_website = row.get('GBP Business Website', '').strip() + + if business_name and (business_phone or business_website): + valid_rows.append(row) + + # Write the valid rows to the output CSV file + with open(output_path, 'w', newline='') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=headers) + writer.writeheader() + writer.writerows(valid_rows) + + # Calculate the total number of rows removed + total_rows_after = len(valid_rows) + total_rows_removed = total_rows_before - total_rows_after + + return total_rows_removed + +def main(): + figlet = Figlet(font='slant') + script_name = "trim-invalid-rows-from-merger".replace("-", " ").title() + print(figlet.renderText(script_name)) + + if not os.path.exists(INPUT_CSV_PATH): + print(f"Error: The file {INPUT_CSV_PATH} does not exist.") + sys.exit(1) + + print(f"Trimming invalid rows in {INPUT_CSV_PATH}...") + spinner = Halo(text='Processing data', spinner='dots') + spinner.start() + total_rows_removed = process_csv_file_trim_invalid_rows(INPUT_CSV_PATH, OUTPUT_CSV_PATH) + spinner.succeed("Processing complete.") + + print(f"Total rows removed: {total_rows_removed}") + +if __name__ == "__main__": + main() + diff --git a/bin/stage-6/uniqify-pass-1.md b/bin/stage-6/uniqify-pass-1.md new file mode 100644 index 0000000..c473d85 --- /dev/null +++ b/bin/stage-6/uniqify-pass-1.md @@ -0,0 +1,567 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 'uniqify-pass-1' which will use the csv file "14-uuid-business-4-single-location-entries.csv" as input, and will output to the csv file "15-uniqify-pass-1.csv" inside of [Stage 6]. + +The goal of this script is to iterate through all of the rows which have a common "Business UUID" and remove duplicates. + +For each match group, I want to keep only unique entries. Unique entries from each match group should also be assigned a new UUID to their "Location UUID" column. + +These unique entries we call Locations, or a Location. + +A Location will have either a different "GBP Business Phone" or "GBP Business Website" fields than other entries. + +If multiple members of a match group have the same "GBP Business Phone" and same "GBP Business Website" they are duplicates and we only need to keep one of the members. + +If multiple members of a match group have the same "GBP Business Phone" and different "GBP Business Website"'s they are unique and we want to keep both. members. + +If multiple members of a match group have different "GBP Business Phone" entries and different "GBP Business Website" entries, they are unique and we want to keep both. + +If multiple members of a match group have different "GBP Business Phone" entries but the same "GBP Business Website" entries, they are unique and we want to keep both. + +If multiple members of a match group have no "GBP Business Phone" entries but the same "GBP Business Website" entries, they are duplicates and we only want to keep one. + +If multiple members of a match group have the same "GBP Business Phone" entries but empty "GBP Business Website" entries, they are duplicates and we only want to keep one. + +If multiple members of a match group have no "GBP Business Phone" entries but different "GBP Business Website" entries, they are unique and we want to keep both. + +If multiple members of a match group have different "GBP Business Phone" entries but no "GBP Business Website" entries, they are unique and we want to keep both. + +If multiple entries have no "GBP Business Phone" entries, but other entries have different "GBP Business Phone" entries, we want to get rid of the entries with no "GBP Business Phone" entries, and keep the entries with the different "GBP Business Phone Entries". + +If multiple entries have no "GBP Business Phone" entries, but other entries have different "GBP Business Phone" entries, we want to get rid of the entries with no "GBP Business Phone" entries, and keep the entries with the different "GBP Business Phone Entries". + +When we decide that we have duplicates and need to delete rows and select an entry to keep, we want to be sure we do not lose any data from certain rows, and want to be sure that we retain information from certain rows. + +The row we want to keep is going to typically be the row with the most non-empty cells out of every column in the CSV. + +Once we know which row we have chosen to keep, we will then double check cells in certain columns of that row for missing data. If we find cells that have missing data in them, we will scan the other rows in the match group for the missing data, and copy it over to the winning row's cell for that column. + +The columns we want to try and retain data for are: + + +GBP Business Phone +GBP Business Website +Root Domain +GBP Business Category +GBP Matching Service +YiB + + +After we have our unique records selected for each "Business UUID" match group, we can then assign a new UUID to that unique location's "Location UUID" column. + + +# Prompt 2 + +I wish to adjust the script 'uniqify-pass-1' which to use the csv file "15-yib-standardized.csv" as input, and will output to the csv file "uniqify-pass-1.csv" inside of [Stage 6]. + + +#!/usr/bin/env python +# Script Name: uniqify-pass-1 + +import os +import csv +import uuid +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '14-uuid-business-4-single-location-entries.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '15-uniqify-pass-1.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def merge_row_data(winning_row, other_row, columns_to_keep): + """Merge data from other_row into winning_row for specified columns.""" + for column in columns_to_keep: + if not winning_row[column].strip() and other_row[column].strip(): + winning_row[column] = other_row[column] + +def is_unique(row1, row2): + """Check if two rows are unique based on the given rules.""" + phone1, phone2 = row1['GBP Business Phone'].strip(), row2['GBP Business Phone'].strip() + website1, website2 = row1['GBP Business Website'].strip(), row2['GBP Business Website'].strip() + + if not phone1 and not phone2 and website1 == website2: + return False + if phone1 == phone2 and not website1 and not website2: + return False + if not phone1 and phone2: + return False + if phone1 and not phone2: + return False + return True + +def uniqify_pass_1(): + """Remove duplicate rows and assign new Location UUIDs.""" + business_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Business UUID + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business UUID' not in headers or 'Location UUID' not in headers: + print("Error: 'Business UUID' or 'Location UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + business_uuid = row['Business UUID'].strip() + if business_uuid: + business_to_rows[business_uuid].append(row) + + print(f"Total groups by Business UUID: {len(business_to_rows)}") + + columns_to_keep = [ + 'GBP Business Phone', 'GBP Business Website', 'Root Domain', + 'GBP Business Category', 'GBP Matching Service', 'YiB' + ] + + unique_rows = [] + + for business_uuid, group in business_to_rows.items(): + if len(group) == 1: + row = group[0] + row['Location UUID'] = generate_uuid() + unique_rows.append(row) + continue + + # Determine unique rows + group.sort(key=lambda x: sum(1 for v in x.values() if v.strip()), reverse=True) + seen = set() + for i, row in enumerate(group): + if i == 0: + unique_rows.append(row) + row['Location UUID'] = generate_uuid() + seen.add(i) + for j, other_row in enumerate(group): + if j != i and is_unique(row, other_row): + if j not in seen: + unique_rows.append(other_row) + other_row['Location UUID'] = generate_uuid() + seen.add(j) + elif j != i: + merge_row_data(row, other_row, columns_to_keep) + + print(f"Total unique rows: {len(unique_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in unique_rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(unique_rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uniqify-pass-1".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Removing duplicates and assigning new Location UUIDs', spinner='dots') + spinner.start() + uniqify_pass_1() + spinner.succeed('Uniqify process completed successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + + + +# Prompt 3 +That script needs to output to a csv file named "16-uniqify-pass-1.csv" not "15-uniqify-pass-1.csv" + + +# Prompt 4 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +The 'uniqify-pass-1' script did not work as expected. + +It did not seem to delete too much, but did fill in a lot of data I did not want it to. + +Please examine this example CSV data from before the script parses it: + + +Business Name,Business UUID,Location UUID,GBP Review Rating,GBP Review Count,YiB,GBP Business Category,GBP Matching Service,GBP Location Municipality,GBP Location State,Location from GBP Business Name,GBP Business Phone,GBP Business Website,Root Domain +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,,4.9,183,65+ years in business,Waterproofing Service,Concrete Walls,Thomasville,NC,,,https://58foundations.com/service-areas/north-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,,4.9,263,65+ years in business,Waterproofing Service,Concrete Walls,Spartanburg,SC,,,https://58foundations.com/service-areas/south-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,,4.9,264,65+ years in business,Waterproofing Service,Concrete Walls,Spartanburg,SC,,,https://58foundations.com/service-areas/south-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,,4.9,266,65+ years in business,Waterproofing Service,Concrete Walls,Spartanburg,SC,,,https://58foundations.com/service-areas/south-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,,4.9,270,65+ years in business,Waterproofing Service,Concrete Walls,Spartanburg,SC,,,https://58foundations.com/service-areas/south-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,,5.0,4,65+ years in business,Waterproofing Service,Bowing Basement Wall Repair,,,,+1 804-409-8953,https://58foundations.com/58-foundations-of-richmond/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,,4.7,179,65+ years in business,Waterproofing Service,Concrete,Woodbridge,VA,,,https://58foundations.com/service-areas/northern-virginia/,58foundations.com + + + +Now please examine this example CSV data from after the script parsed it: + + +Business Name,Business UUID,Location UUID,GBP Review Rating,GBP Review Count,YiB,GBP Business Category,GBP Matching Service,GBP Location Municipality,GBP Location State,Location from GBP Business Name,GBP Business Phone,GBP Business Website,Root Domain +58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,40286da0-fb0b-4d9b-b1ae-ce242f5705c9,4.7,18,65+ years in business,Waterproofing Service,Structural Repair Services,Charlotte,NC,,+1 704-850-6343,https://58foundations.com/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,83fd0409-b662-4353-a878-d22d482bcdd9,5.0,4,65+ years in business,Waterproofing Service,Bowing Basement Wall Repair,,,,+1 804-409-8953,https://58foundations.com/58-foundations-of-richmond/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,825b9093-ceae-4a26-abef-3bd025847414,5.0,4,65+ years in business,Waterproofing Service,Bowing Basement Wall Repair,,,,+1 804-409-8953,https://58foundations.com/58-foundations-of-richmond/,58foundations.com +’58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,684e5c97-d2c9-4eca-b400-d95ef5a00422,4.8,54,65+ years in business,Waterproofing Service,Structural Repair Services,,,,+1 423-460-9468,https://58foundations.com/58-foundations-of-chattanooga/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,1e7b4686-623c-455b-a289-764cb23bed09,4.9,183,65+ years in business,Waterproofing Service,Concrete Walls,Thomasville,NC,,+1 704-850-6343,https://58foundations.com/service-areas/north-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,157de877-8252-4985-8c49-32f0c7851869,4.9,263,65+ years in business,Waterproofing Service,Concrete Walls,Spartanburg,SC,,+1 704-850-6343,https://58foundations.com/service-areas/south-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,8917e6b2-1b33-4bb7-bfc1-3664b4e0e74c,4.9,264,65+ years in business,Waterproofing Service,Concrete Walls,Spartanburg,SC,,+1 704-850-6343,https://58foundations.com/service-areas/south-carolina/,58foundations.com + + + +Now please examine thie example CSV data of what I expected the data to look like after being parsed: + + +Business Name,Business UUID,Location UUID,GBP Review Rating,GBP Review Count,YiB,GBP Business Category,GBP Matching Service,GBP Location Municipality,GBP Location State,Location from GBP Business Name,GBP Business Phone,GBP Business Website,Root Domain +58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,40286da0-fb0b-4d9b-b1ae-ce242f5705c9,4.7,18,65+ years in business,Waterproofing Service,Structural Repair Services,Charlotte,NC,,+1 704-850-6343,https://58foundations.com/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,83fd0409-b662-4353-a878-d22d482bcdd9,5,4,65+ years in business,Waterproofing Service,Bowing Basement Wall Repair,,,,+1 804-409-8953,https://58foundations.com/58-foundations-of-richmond/,58foundations.com +’58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,684e5c97-d2c9-4eca-b400-d95ef5a00422,4.8,54,65+ years in business,Waterproofing Service,Structural Repair Services,,,,+1 423-460-9468,https://58foundations.com/58-foundations-of-chattanooga/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,1e7b4686-623c-455b-a289-764cb23bed09,4.9,183,65+ years in business,Waterproofing Service,Concrete Walls,Thomasville,NC,,+1 704-850-6343,https://58foundations.com/service-areas/north-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,157de877-8252-4985-8c49-32f0c7851869,4.9,263,65+ years in business,Waterproofing Service,Concrete Walls,Spartanburg,SC,,+1 704-850-6343,https://58foundations.com/service-areas/south-carolina/,58foundations.com +'58 Foundations & Waterproofing,9a2d6c18-67a5-4f9c-8d21-ea345ab05691,94487996-7aa7-41fc-a02c-fd800811ebb2,4.7,179,65+ years in business,Waterproofing Service,Concrete,Woodbridge,VA,,+1 704-850-6343,https://58foundations.com/service-areas/northern-virginia/,58foundations.com +(Associates in groups) DBA Concrete Loyalty,2d7b5cc5-6945-4308-8180-306dcd8ab5f4,46ef3121-2836-4451-8f40-f27f36c1b50c,,,,Concrete Contractor,Repair Flooring,,,,+1 864-450-6496,, + + + +THere were entries in the CSV data after the parsing which were duplicates of each other. I saw entries with the same "GBP Business Website" + +Are you able to discern the problem I am trying to bring to light through examination of the above CSV data? + +I looked at this list match group and saw that many of them had unique "GBP Business Website" entries, but some of them were the same ones. + +Such as seeing: + +"https://58foundations.com/service-areas/south-carolina/" occur twice. + +Or seeing: + +"https://58foundations.com/58-foundations-of-richmond/" occur twice as well. + +The rows that shared those duplicate "GBP Business Website" entries, also had the same "GBP Business Phone". Both of those factors together clearly label these rows as duplicates. + + +# Prompt 5 +Ok I think this script is too complex. + +I would like you to re-create the entire thing from scratch + +This time though, the only rule I want to check for in each match group (same match group) is which rows have the same "GBP Business Phone" and "GBP Business Website" lines. + +Rows which have both of those entries as the same as each other should be considered duplicates. + +The row with the most data accross all columns is chosen as the winner, and the losing duplicates get deleted. + +For each column in the list of "Columns to Keep", check if the winning row is missing data there or not. If that cell is missing data, try to copy it over from other cells in other rows that contain the missing data before deleting the losing rows. + + +# Prompt 6 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output llwhat they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I ran into a problem with the 'uniqify-pass-1' script: + + +#!/usr/bin/env python +# Script Name: uniqify-pass-1 + +import os +import csv +import uuid +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '15-yib-standardized.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '16-uniqify-pass-1.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def merge_row_data(winning_row, other_row, columns_to_keep): + """Merge data from other_row into winning_row for specified columns.""" + for column in columns_to_keep: + if not winning_row[column].strip() and other_row[column].strip(): + winning_row[column] = other_row[column] + +def row_has_more_data(row): + """Calculate the number of non-empty cells in the row.""" + return sum(1 for v in row.values() if v.strip()) + +def uniqify_pass_1(): + """Remove duplicate rows and assign new Location UUIDs.""" + business_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Business UUID + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business UUID' not in headers or 'Location UUID' not in headers: + print("Error: 'Business UUID' or 'Location UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + business_uuid = row['Business UUID'].strip() + if business_uuid: + business_to_rows[business_uuid].append(row) + + print(f"Total groups by Business UUID: {len(business_to_rows)}") + + columns_to_keep = [ + 'GBP Business Phone', 'GBP Business Website', 'Root Domain', + 'GBP Business Category', 'GBP Matching Service', 'YiB' + ] + + unique_rows = [] + + for business_uuid, group in business_to_rows.items(): + if len(group) == 1: + row = group[0] + row['Location UUID'] = generate_uuid() + unique_rows.append(row) + continue + + seen_pairs = set() + for row in group: + phone_website_pair = (row['GBP Business Phone'].strip(), row['GBP Business Website'].strip()) + if phone_website_pair not in seen_pairs: + seen_pairs.add(phone_website_pair) + duplicates = [r for r in group if (r['GBP Business Phone'].strip(), r['GBP Business Website'].strip()) == phone_website_pair] + + # Determine the winning row (with most data) + winning_row = max(duplicates, key=row_has_more_data) + + # Merge missing data from duplicates into the winning row + for other_row in duplicates: + if other_row != winning_row: + merge_row_data(winning_row, other_row, columns_to_keep) + + # Assign new Location UUID to the winning row + winning_row['Location UUID'] = generate_uuid() + unique_rows.append(winning_row) + + print(f"Total unique rows: {len(unique_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in unique_rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(unique_rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uniqify-pass-1".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Removing duplicates and assigning new Location UUIDs', spinner='dots') + spinner.start() + uniqify_pass_1() + spinner.succeed('Uniqify process completed successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + +The script is intended to remove duplicates, and isolate the unique locations within each business. + +It does this primarily by examining and comparing "GBP Business Phone" and "GBP Business Website" fields. + +I found 3 entries in the parsed CSV data though which should have been all matched as duplicates, and only 1 should have been selected from the group. + +I notice that one of the entries in this selection begins with "http://" while others begin with "https://". So the first modification I know I need to make to 'uniqify-pass-1' is that it treats both "http://" and "https://" the same when trying to determine maches between "GBP Business Website" entries. + +Finally we have a 3rd entry in this group with no "GBP Business Phone" at all, and only a "GBP Business Website". While the other 2 entries in this match group have the same phone number , and same website. Meaning that this entire set of 3 is only 1 location. We should only keep 1 entry from this entire list of 3, and 2 of them should be completely deleted. + +Here is the data: + + +Business Name,Business UUID,Location UUID,GBP Review Rating,GBP Review Count,YiB,GBP Business Category,GBP Matching Service,GBP Location Municipality,GBP Location State,Location from GBP Business Name,GBP Business Phone,GBP Business Website,Root Domain +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,0f46ce7c-186b-4eb7-87b8-9bc0c81a9ea2,,,10+ years in business,Contractor,Repair Flooring,Spring City,PA,,+1 610-467-4107,https://permanentconcretesolutions.com/contact,permanentconcretesolutions.com +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,88377e04-fdfc-438e-9471-303f1f85e2e0,,,10+ years in business,Contractor,Repair Flooring,Spring City,PA,,+1 610-467-4107,https://permanentconcretesolutions.com/,permanentconcretesolutions.com +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,2e7a7c51-370a-4af8-8ab4-8b77e1b8381e,,,10+ years in business,Concrete Contractor,Drywall Repair,,,,,https://permanentconcretesolutions.com/,permanentconcretesolutions.com + + +Please attempt to compare my claims above against the data. Let me know if you see what I am describing, and if I described it accurately. If I feel we have an understanding on this issue, i will ask you to attempt to adjust the code to address it. + + +# Prompt 7 +Alright, it seems we ran into the same problem for some reason. + +I ran the script and the 3 example options from before are still in the list as a match group of 3. They were note deduplicated at all. + +Here is the data after having ran the script: + + +Business Name,Business UUID,Location UUID,GBP Review Rating,GBP Review Count,YiB,GBP Business Category,GBP Matching Service,GBP Location Municipality,GBP Location State,Location from GBP Business Name,GBP Business Phone,GBP Business Website,Root Domain +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,57a80dd8-4ce1-4e81-a745-7bc7f6eef4f0,,,10+ years in business,Contractor,Repair Flooring,Spring City,PA,,+1 610-467-4107,https://permanentconcretesolutions.com/contact,permanentconcretesolutions.com +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,239ff200-0428-4023-9ef7-a919f1b61cac,,,10+ years in business,Contractor,Repair Flooring,Spring City,PA,,+1 610-467-4107,https://permanentconcretesolutions.com/,permanentconcretesolutions.com +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,921eaac1-bc13-4b8a-aa06-04c53c8ebe15,,,10+ years in business,Concrete Contractor,Drywall Repair,,,,,https://permanentconcretesolutions.com/,permanentconcretesolutions.com + + +After I run this script, there should only be 1 of that group left over. The other 2 should be gone. + + +# Prompt 8 +Ok we are closer. That version of the script was close. + +We are left with the following data now: + + +Business Name,Business UUID,Location UUID,GBP Review Rating,GBP Review Count,YiB,GBP Business Category,GBP Matching Service,GBP Location Municipality,GBP Location State,Location from GBP Business Name,GBP Business Phone,GBP Business Website,Root Domain +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,422c4c40-56ff-44a7-b65d-201b5c3cbb97,,,10+ years in business,Contractor,Repair Flooring,Spring City,PA,,+1 610-467-4107,https://permanentconcretesolutions.com/contact,permanentconcretesolutions.com +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,9fb62824-04ba-40b5-97a6-32ea04b47ff5,,,10+ years in business,Concrete Contractor,Drywall Repair,,,,,https://permanentconcretesolutions.com/,permanentconcretesolutions.com + + +This is almost correct. + +However, the last entry without any "GBP Business Phone" should be removed as well. + +If have two locations in a business group, and one of them has no "GBP Phone Number" but has the same "GBP Business Website" as other entries in the business group which do have "GBP Business Phone" numbers, we want to consider those duplicates, and we want to keep the row with the most data. Merging over the "columns_to_keep" into the row we are keeping. + + +# Prompt 9 +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,5f77e73f-7244-473e-9470-5d6d9b9230e0,,,10+ years in business,Contractor,Repair Flooring,Spring City,PA,,+1 610-467-4107,https://permanentconcretesolutions.com/contact,permanentconcretesolutions.com +Permanent Concrete Solutions,fd8d60bc-fa57-4591-ad20-d6adf82425df,b1dfcc8b-9c61-412d-bb75-490201bfd4dd,,,10+ years in business,Concrete Contractor,Drywall Repair,,,,,https://permanentconcretesolutions.com/,permanentconcretesolutions.com diff --git a/bin/stage-6/uniqify-pass-1.py b/bin/stage-6/uniqify-pass-1.py new file mode 100755 index 0000000..c058451 --- /dev/null +++ b/bin/stage-6/uniqify-pass-1.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# Script Name: uniqify-pass-1 + +import os +import csv +import uuid +from collections import defaultdict +from urllib.parse import urlparse +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '15-yib-standardized.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '16-uniqify-pass-1.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def merge_row_data(winning_row, other_row, columns_to_keep): + """Merge data from other_row into winning_row for specified columns.""" + for column in columns_to_keep: + if not winning_row[column].strip() and other_row[column].strip(): + winning_row[column] = other_row[column] + +def row_has_more_data(row): + """Calculate the number of non-empty cells in the row.""" + return sum(1 for v in row.values() if v.strip()) + +def normalize_url(url): + """Normalize URLs by stripping the scheme (http or https), trailing slashes, and ignoring '/contact' or '/contact-us'.""" + parsed_url = urlparse(url) + netloc = parsed_url.netloc + path = parsed_url.path.rstrip('/') + if path.endswith('/contact'): + path = path[:-8] + elif path.endswith('/contact-us'): + path = path[:-11] + return netloc + path + +def uniqify_pass_1(): + """Remove duplicate rows and assign new Location UUIDs.""" + business_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Business UUID + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business UUID' not in headers or 'Location UUID' not in headers: + print("Error: 'Business UUID' or 'Location UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + business_uuid = row['Business UUID'].strip() + if business_uuid: + business_to_rows[business_uuid].append(row) + + print(f"Total groups by Business UUID: {len(business_to_rows)}") + + columns_to_keep = [ + 'GBP Business Phone', 'GBP Business Website', 'Root Domain', + 'GBP Business Category', 'GBP Matching Service', 'YiB' + ] + + unique_rows = [] + + for business_uuid, group in business_to_rows.items(): + if len(group) == 1: + row = group[0] + row['Location UUID'] = generate_uuid() + unique_rows.append(row) + continue + + # Dictionary to hold unique entries + seen = {} + rows_without_phone = [] + + for row in group: + normalized_website = normalize_url(row['GBP Business Website'].strip()) + phone = row['GBP Business Phone'].strip() + phone_website_pair = (phone, normalized_website) + + if phone_website_pair not in seen: + seen[phone_website_pair] = row + row['Location UUID'] = generate_uuid() + else: + winning_row = seen[phone_website_pair] + merge_row_data(winning_row, row, columns_to_keep) + + # Track rows without phone numbers separately + if not phone: + rows_without_phone.append(row) + + # Handle rows without phone numbers + for row in rows_without_phone: + normalized_website = normalize_url(row['GBP Business Website'].strip()) + duplicates_found = False + + for key, existing_row in seen.items(): + if normalize_url(existing_row['GBP Business Website'].strip()) == normalized_website and existing_row['GBP Business Phone'].strip(): + merge_row_data(existing_row, row, columns_to_keep) + duplicates_found = True + break + + if not duplicates_found: + row['Location UUID'] = generate_uuid() + seen[("", normalized_website)] = row + + unique_rows.extend(seen.values()) + + print(f"Total unique rows: {len(unique_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in unique_rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(unique_rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uniqify-pass-1".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Removing duplicates and assigning new Location UUIDs', spinner='dots') + spinner.start() + uniqify_pass_1() + spinner.succeed('Uniqify process completed successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/uniqify-pass-2.md b/bin/stage-6/uniqify-pass-2.md new file mode 100644 index 0000000..dac9ca9 --- /dev/null +++ b/bin/stage-6/uniqify-pass-2.md @@ -0,0 +1,16 @@ +# Prompt 1 +I wish to create another version of this script which we will call 'uniqify-pass-2'. + +It will have the same normalization done for the "GBP Business Website" when doing its checks. + +It will use input the csv file '16-uniqify-pass-1.csv' and as output will use the csv file '17-uniqify-pass-2.csv'. + +This script will do much the same thing as 'uniqify-pass-1'. It will create match groups based upon rows that have the same "Business UUID". + +These match groups will be examined to see if any of the match groups contain a mix of rows that have the same "GBP Business Website" field (Noramlized in the same way uniqify-pass-1 does.), but some entries have "GBP Business Phone" numbers while other rows do not have a "GBP Business Phone". + +When we find such a set, if any of the rows in the match group which are missing the "GBP Business Phone" data, also have a "GBP Business Website" that matches the "GBP Business Website" of other entries in the match group that do have entries in their "GBP Business Phone" field, we want to consider these duplicates and delete the row with the missing 'GBP Business Phone" data. We want to make sure though that we merge missing data from the row being deleted into the rows being preserved if need be. + +The rows to perserve are the same as in the uniqify-pass-1 script. + +Please create this script for me now. diff --git a/bin/stage-6/uniqify-pass-2.py b/bin/stage-6/uniqify-pass-2.py new file mode 100755 index 0000000..d564b0b --- /dev/null +++ b/bin/stage-6/uniqify-pass-2.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# Script Name: uniqify-pass-2 + +import os +import csv +import uuid +from urllib.parse import urlparse +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '16-uniqify-pass-1.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '17-uniqify-pass-2.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def normalize_url(url): + """Normalize the URL for comparison purposes.""" + parsed_url = urlparse(url) + netloc = parsed_url.netloc + path = parsed_url.path.rstrip('/') + if path.endswith('/contact'): + path = path[:-8] + elif path.endswith('/contact-us'): + path = path[:-11] + return netloc + path + +def merge_row_data(winning_row, other_row, columns_to_keep): + """Merge data from other_row into winning_row for specified columns.""" + for column in columns_to_keep: + if not winning_row[column].strip() and other_row[column].strip(): + winning_row[column] = other_row[column] + +def row_has_more_data(row): + """Calculate the number of non-empty cells in the row.""" + return sum(1 for v in row.values() if v.strip()) + +def uniqify_pass_2(): + """Remove duplicate rows and assign new Location UUIDs.""" + business_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Business UUID + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business UUID' not in headers or 'Location UUID' not in headers: + print("Error: 'Business UUID' or 'Location UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + business_uuid = row['Business UUID'].strip() + if business_uuid: + business_to_rows[business_uuid].append(row) + + print(f"Total groups by Business UUID: {len(business_to_rows)}") + + columns_to_keep = [ + 'GBP Business Phone', 'GBP Business Website', 'Root Domain', + 'GBP Business Category', 'GBP Matching Service', 'YiB' + ] + + unique_rows = [] + + for business_uuid, group in business_to_rows.items(): + if len(group) == 1: + row = group[0] + unique_rows.append(row) + continue + + seen_pairs = set() + for row in group: + normalized_website = normalize_url(row['GBP Business Website'].strip()) + phone = row['GBP Business Phone'].strip() + phone_website_pair = (phone, normalized_website) + + if not phone: + duplicates_found = False + for other_row in group: + if other_row['GBP Business Phone'].strip() and normalize_url(other_row['GBP Business Website'].strip()) == normalized_website: + merge_row_data(other_row, row, columns_to_keep) + duplicates_found = True + break + if not duplicates_found: + row['Location UUID'] = generate_uuid() + unique_rows.append(row) + else: + if phone_website_pair not in seen_pairs: + seen_pairs.add(phone_website_pair) + row['Location UUID'] = generate_uuid() + unique_rows.append(row) + + print(f"Total unique rows: {len(unique_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in unique_rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(unique_rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uniqify-pass-2".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Removing duplicates and assigning new Location UUIDs', spinner='dots') + spinner.start() + uniqify_pass_2() + spinner.succeed('Uniqify process completed successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/uuid-business-1-by-gbp-business-phone.md b/bin/stage-6/uuid-business-1-by-gbp-business-phone.md new file mode 100644 index 0000000..eaed855 --- /dev/null +++ b/bin/stage-6/uuid-business-1-by-gbp-business-phone.md @@ -0,0 +1,216 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 'uuid-business-1-by-gbp-business-phone' which will use the csv file "08-business-names-with-locations.csv" as input, and will output to the csv file "09-uuid-business-1-by-gbp-business-phone.csv". + +This script will begin by creating a new column named "Business UUID" at the end of the existing rows. Be sure that the script maintains uniform column count all the way down the data as the new column is added. + +Then the script is going to group together rows of data according to their "GBP Business Phone" entries. Rows that have the same "GBP Business Phone" as one another we are going to consider as being the same business. + +When such a group is found like that, a UUID will be generated and assigned to every member of that group. + + +# Prompt 2 +I forgot to mention that empty cells are not considered matches. + +And neither are rows that do not have other entries with the same "GBP Business Phone". There must exist a group of similar entries for a UUID to be generated and assigned to them. + + +# Prompt 3 +Ok I ran into some unexpected behaviour with this script. + +There was only one record in the output csv file. + +There should be no missing records. + +We simply copy over the input file to the output file. Then from there we add the new column. After that we try to find cells that have the same "GBP Business Phone", and if we find rows that share a "GBP Business Phone" we assign them the same UUID as one another. + +This process should not result in a CSV file with fewer records than the input csv file. + + +# Prompt 4 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to modify the script 'uuid-business-1-by-gbp-business-phone': + + +#!/usr/bin/env python +# Script Name: uuid-business-1-by-gbp-business-phone + +import os +import csv +import uuid +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '08-business-names-with-locations.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '09-uuid-business-1-by-gbp-business-phone.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def add_business_uuid(): + """Add Business UUID to each row grouped by GBP Business Phone.""" + phone_to_rows = defaultdict(list) + + # Read the input CSV and group rows by GBP Business Phone + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'GBP Business Phone' not in headers: + print("Error: 'GBP Business Phone' column is missing.") + return + headers.append('Business UUID') + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + phone = row['GBP Business Phone'].strip() + if phone: # Ignore empty cells + phone_to_rows[phone].append(row) + + print(f"Total groups by phone: {len(phone_to_rows)}") + + # Assign UUIDs to groups with more than one entry + phone_to_uuid = {} + for phone, grouped_rows in phone_to_rows.items(): + if len(grouped_rows) > 1: + phone_to_uuid[phone] = generate_uuid() + + print(f"Total groups assigned UUIDs: {len(phone_to_uuid)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + phone = row['GBP Business Phone'].strip() + if phone in phone_to_uuid: + row['Business UUID'] = phone_to_uuid[phone] + else: + row['Business UUID'] = '' # Leave the UUID column empty if no match + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-1-by-gbp-business-phone".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by GBP Business Phone', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + +I would like to extend this script by have it merge some data as it goes. + +In the rows that make up match groups in this script, I would like to check if any of those rows have entries in certain columns that other rows in the match group might be missing. + +I would like to fill in missing data in each row for certain columns, using data from other rows in the match group which may have the data we are missing the other rows. + +The data I would like to be able to merge into all rows in a match group are the following columns: + + +YiB +GBP Business Category +GBP Matching Service +GBP Business Website +Root Domain + + +To be sure I described my desire for this script well enough, please describe the modifications back to me so that I can confirm I communicated my needs well. Then I'll ask you to extend the script after. + diff --git a/bin/stage-6/uuid-business-1-by-gbp-business-phone.py b/bin/stage-6/uuid-business-1-by-gbp-business-phone.py new file mode 100755 index 0000000..853f6cc --- /dev/null +++ b/bin/stage-6/uuid-business-1-by-gbp-business-phone.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# Script Name: uuid-business-1-by-gbp-business-phone + +import os +import csv +import uuid +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '08-business-names-with-locations.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '09-uuid-business-1-by-gbp-business-phone.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def merge_data(grouped_rows, columns_to_merge): + """Merge missing data within match groups.""" + merged_data = {column: '' for column in columns_to_merge} + + # Collect data from all rows in the group + for row in grouped_rows: + for column in columns_to_merge: + if row[column].strip(): + merged_data[column] = row[column] + + # Fill missing data in each row + for row in grouped_rows: + for column in columns_to_merge: + if not row[column].strip(): + row[column] = merged_data[column] + +def add_business_uuid(): + """Add Business UUID to each row grouped by GBP Business Phone and merge missing data.""" + phone_to_rows = defaultdict(list) + columns_to_merge = ['YiB', 'GBP Business Category', 'GBP Matching Service', 'GBP Business Website', 'Root Domain'] + + # Read the input CSV and group rows by GBP Business Phone + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'GBP Business Phone' not in headers: + print("Error: 'GBP Business Phone' column is missing.") + return + if 'Business UUID' not in headers: + headers.append('Business UUID') + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + phone = row['GBP Business Phone'].strip() + if phone: # Ignore empty cells + phone_to_rows[phone].append(row) + + print(f"Total groups by phone: {len(phone_to_rows)}") + + # Assign UUIDs to groups with more than one entry and merge data + phone_to_uuid = {} + for phone, grouped_rows in phone_to_rows.items(): + if len(grouped_rows) > 1: + # Generate or assign existing UUID + if phone not in phone_to_uuid: + phone_to_uuid[phone] = generate_uuid() + uuid_to_assign = phone_to_uuid[phone] + + # Merge data within the group + merge_data(grouped_rows, columns_to_merge) + + # Assign the UUID to all rows in the group + for row in grouped_rows: + row['Business UUID'] = uuid_to_assign + + print(f"Total groups assigned UUIDs: {len(phone_to_uuid)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-1-by-gbp-business-phone".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by GBP Business Phone', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/uuid-business-2-by-root-domain.md b/bin/stage-6/uuid-business-2-by-root-domain.md new file mode 100644 index 0000000..95a0796 --- /dev/null +++ b/bin/stage-6/uuid-business-2-by-root-domain.md @@ -0,0 +1,437 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 'uuid-business-2-by-root-domain' which will use the csv file "09-uuid-business-1-by-gbp-business-phone.csv" as input, and will output to the csv file "10-uuid-business-2-by-root-domain.csv". + +This script will perform a similar grouping and uuid assignment task as the "uuid-business-1-by-gbp-business-phone" script: + + +#!/usr/bin/env python +# Script Name: uuid-business-1-by-gbp-business-phone + +import os +import csv +import uuid +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '08-business-names-with-locations.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '09-uuid-business-1-by-gbp-business-phone.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def add_business_uuid(): + """Add Business UUID to each row grouped by GBP Business Phone.""" + phone_to_rows = defaultdict(list) + + # Read the input CSV and group rows by GBP Business Phone + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'GBP Business Phone' not in headers: + print("Error: 'GBP Business Phone' column is missing.") + return + headers.append('Business UUID') + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + phone = row['GBP Business Phone'].strip() + if phone: # Ignore empty cells + phone_to_rows[phone].append(row) + + print(f"Total groups by phone: {len(phone_to_rows)}") + + # Assign UUIDs to groups with more than one entry + phone_to_uuid = {} + for phone, grouped_rows in phone_to_rows.items(): + if len(grouped_rows) > 1: + phone_to_uuid[phone] = generate_uuid() + + print(f"Total groups assigned UUIDs: {len(phone_to_uuid)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + phone = row['GBP Business Phone'].strip() + if phone in phone_to_uuid: + row['Business UUID'] = phone_to_uuid[phone] + else: + row['Business UUID'] = '' # Leave the UUID column empty if no match + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-1-by-gbp-business-phone".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by GBP Business Phone', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + +Our new uuid-business-2-by-root-domain script can assume that the "Business UUID" column already exists though, and does not need to create it. + +This script needs to do the same grouping and uuid assignment task as the uuid-business-1-by-gbp-business-phone script but will group by the contents of the "Root Domain" column instead. + +There are also some important extra rules to follow before assigning the UUID. + +Rule 1: If a group of entries is found to have the same "Root Domain" first check if any of those entries already has a "Business UUID" assigned to it in the "Business UUID" column. If one of them does already have a "Business UUID", then use that "Business UUID" as the "Business UUID" to assign to all the members of the match group. + +Rule 2: Any members of the match group which already have a "Business UUID" should not have their existing "Business UUID" over written. + + +# Prompt 2 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to modify the script 'uuid-business-2-by-root-domain': + + +#!/usr/bin/env python +# Script Name: uuid-business-2-by-root-domain + +import os +import csv +import uuid +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '09-uuid-business-1-by-gbp-business-phone.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '10-uuid-business-2-by-root-domain.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def add_business_uuid(): + """Add Business UUID to each row grouped by Root Domain.""" + domain_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Root Domain + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Root Domain' not in headers or 'Business UUID' not in headers: + print("Error: 'Root Domain' or 'Business UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + domain = row['Root Domain'].strip() + if domain: # Ignore empty cells + domain_to_rows[domain].append(row) + + print(f"Total groups by domain: {len(domain_to_rows)}") + + # Assign UUIDs to groups with more than one entry + for domain, grouped_rows in domain_to_rows.items(): + # Find if any row already has a Business UUID + existing_uuid = None + for row in grouped_rows: + if row['Business UUID']: + existing_uuid = row['Business UUID'] + break + + # Assign the existing UUID or generate a new one + assigned_uuid = existing_uuid or generate_uuid() + + # Assign the UUID to all rows in the group + for row in grouped_rows: + if not row['Business UUID']: # Do not overwrite existing UUIDs + row['Business UUID'] = assigned_uuid + + print(f"Total groups assigned UUIDs: {len(domain_to_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-2-by-root-domain".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by Root Domain', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + +I would like to change this script so that it can overwrite "Business UUID"s. + +If we have a match group where all the "Root Domains" are the same, we can assume they are all the same business, and that they should all have the same UUID. + +We will want to use the "Business UUID" which occurs most frequently in the match group. + +If there is an equal amount of "Business UUID" to choose from amoung the match groups, we can just pick the first one and use that to assign to all of the "Business UUID"s in the match group. + + +# Prompt 3 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to modify the script 'uuid-business-2-by-root-domain': + + +#!/usr/bin/env python +# Script Name: uuid-business-2-by-root-domain + +import os +import csv +import uuid +from collections import defaultdict, Counter +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '09-uuid-business-1-by-gbp-business-phone.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '10-uuid-business-2-by-root-domain.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def add_business_uuid(): + """Add Business UUID to each row grouped by Root Domain.""" + domain_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Root Domain + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Root Domain' not in headers or 'Business UUID' not in headers: + print("Error: 'Root Domain' or 'Business UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + domain = row['Root Domain'].strip() + if domain: # Ignore empty cells + domain_to_rows[domain].append(row) + + print(f"Total groups by domain: {len(domain_to_rows)}") + + # Assign UUIDs to groups with more than one entry + for domain, grouped_rows in domain_to_rows.items(): + # Collect all existing UUIDs in the group + uuid_counter = Counter(row['Business UUID'] for row in grouped_rows if row['Business UUID']) + + if uuid_counter: + # Use the most common UUID in the group + most_common_uuid, _ = uuid_counter.most_common(1)[0] + else: + # Generate a new UUID if none exist in the group + most_common_uuid = generate_uuid() + + # Assign the most common or newly generated UUID to all rows in the group + for row in grouped_rows: + row['Business UUID'] = most_common_uuid + + print(f"Total groups assigned UUIDs: {len(domain_to_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-2-by-root-domain".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by Root Domain', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + +I would like to extend this script by have it merge some data as it goes. + +In the rows that make up match groups in this script, I would like to check if any of those rows have entries in certain columns that other rows in the match group might be missing. + +I would like to fill in missing data in each row for certain columns, using data from other rows in the match group which may have the data we are missing the other rows. + +The data I would like to be able to merge into all rows in a match group are the following columns: + + +YiB +GBP Business Category +GBP Matching Service + + + diff --git a/bin/stage-6/uuid-business-2-by-root-domain.py b/bin/stage-6/uuid-business-2-by-root-domain.py new file mode 100755 index 0000000..a5b063f --- /dev/null +++ b/bin/stage-6/uuid-business-2-by-root-domain.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# Script Name: uuid-business-2-by-root-domain + +import os +import csv +import uuid +from collections import defaultdict, Counter +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '09-uuid-business-1-by-gbp-business-phone.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '10-uuid-business-2-by-root-domain.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def merge_data(grouped_rows, columns_to_merge): + """Merge missing data within match groups.""" + merged_data = {column: '' for column in columns_to_merge} + + # Collect data from all rows in the group + for row in grouped_rows: + for column in columns_to_merge: + if row[column].strip(): + merged_data[column] = row[column] + + # Fill missing data in each row + for row in grouped_rows: + for column in columns_to_merge: + if not row[column].strip(): + row[column] = merged_data[column] + +def add_business_uuid(): + """Add Business UUID to each row grouped by Root Domain and merge missing data.""" + domain_to_rows = defaultdict(list) + columns_to_merge = ['YiB', 'GBP Business Category', 'GBP Matching Service'] + + # Read the input CSV and group rows by Root Domain + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Root Domain' not in headers or 'Business UUID' not in headers: + print("Error: 'Root Domain' or 'Business UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + domain = row['Root Domain'].strip() + if domain: # Ignore empty cells + domain_to_rows[domain].append(row) + + print(f"Total groups by domain: {len(domain_to_rows)}") + + # Assign UUIDs to groups with more than one entry and merge data + for domain, grouped_rows in domain_to_rows.items(): + # Collect all existing UUIDs in the group + uuid_counter = Counter(row['Business UUID'] for row in grouped_rows if row['Business UUID']) + + if uuid_counter: + # Use the most common UUID in the group + most_common_uuid, _ = uuid_counter.most_common(1)[0] + else: + # Generate a new UUID if none exist in the group + most_common_uuid = generate_uuid() + + # Assign the most common or newly generated UUID to all rows in the group + for row in grouped_rows: + row['Business UUID'] = most_common_uuid + + # Merge data within the group + merge_data(grouped_rows, columns_to_merge) + + print(f"Total groups assigned UUIDs: {len(domain_to_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-2-by-root-domain".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by Root Domain', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() diff --git a/bin/stage-6/uuid-business-3-by-business-name.md b/bin/stage-6/uuid-business-3-by-business-name.md new file mode 100644 index 0000000..de53b42 --- /dev/null +++ b/bin/stage-6/uuid-business-3-by-business-name.md @@ -0,0 +1,156 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 'uuid-business-3-by-business-name' which will use the csv file "10-uuid-business-2-by-root-domain.csv" as input, and will output to the csv file "11-uuid-business-3-by-business-name.csv" inside of [Stage 6]. + +This script will perform a similar grouping and uuid assignment task as the "uuid-business-2-by-root-domain" script: + + +#!/usr/bin/env python +# Script Name: uuid-business-2-by-root-domain + +import os +import csv +import uuid +from collections import defaultdict +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '09-uuid-business-1-by-gbp-business-phone.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '10-uuid-business-2-by-root-domain.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def add_business_uuid(): + """Add Business UUID to each row grouped by Root Domain.""" + domain_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Root Domain + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Root Domain' not in headers or 'Business UUID' not in headers: + print("Error: 'Root Domain' or 'Business UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + domain = row['Root Domain'].strip() + if domain: # Ignore empty cells + domain_to_rows[domain].append(row) + + print(f"Total groups by domain: {len(domain_to_rows)}") + + # Assign UUIDs to groups with more than one entry + for domain, grouped_rows in domain_to_rows.items(): + # Find if any row already has a Business UUID + existing_uuid = None + for row in grouped_rows: + if row['Business UUID']: + existing_uuid = row['Business UUID'] + break + + # Assign the existing UUID or generate a new one + assigned_uuid = existing_uuid or generate_uuid() + + # Assign the UUID to all rows in the group + for row in grouped_rows: + if not row['Business UUID']: # Do not overwrite existing UUIDs + row['Business UUID'] = assigned_uuid + + print(f"Total groups assigned UUIDs: {len(domain_to_rows)}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-2-by-root-domain".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by Root Domain', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + + + +This script needs to do the same grouping and uuid assignment task as the uuid-business-2-by-root-domain script. However uuid-business-3-by-business-name will group by the contents of the "Business Name" column instead. + +There are also some important extra rules to follow before assigning the UUID. + +Rule 1: If a group of entries is found to have the same "Business Name", then first check if any of those entries already has a "Business UUID" assigned to it in the "Business UUID" column. If one of them does already have a "Business UUID", then use that "Business UUID" as the "Business UUID" to assign to all the members of the match group. If there are multiple "Business UUID"s to choose from in the match group, then pick the "Business UUID" that occurs the most frequently in the match group. Otherwise pick the "Business UUID" from the record in the match list with the most in common to the record which needs the "Business UUID" assigned to it. This can be done by comparing the columns "YiB", "GBP Matching Service", "GBP Business Category", "GBP Review Count", and "GBP Review Rating". The more matches found between these cells in the record getting the UUID vs the records we are checking for a UUID winner from the more weight we should give to that records UUID in terms of whether or not we should pick it as the one to copy over or not. + +Rule 2: Any members of the match group which already have a "Business UUID" should not have their existing "Business UUID" over written. + +I feel as though Rule 1 above is very complex and that I may not have described it well. I did my best, but I am still unsure about how well I communicated it. Can you please iterate it back to me as you understand it so I can confirm if I got it accross well or not? I would like to confirm this before I ask you to write this script. + + +# Prompt 2 +Ok it looks like this script may be doing what I desired mostly, but there is one unexpected behaviour this script is exhibiting which I did not accont for. + +This unexpected behaviour needs to be accounted for before i can judge if this script is doing the task at hand correctly or not. + +The unexpected behaviour is that it looks like this script may be assigning "Business UUID"'s to entries which are not a part of a group. That is to say, rows that have a "Business Name" which does not have another row with the same "Business Name" somewhere have no matches, there is no match group here. Unless there is a match group, no "Business UUID" should be assigned. It might be said that some of these entries that occur on their own have a match group of 1, but that was not really what I mean or need. diff --git a/bin/stage-6/uuid-business-3-by-business-name.py b/bin/stage-6/uuid-business-3-by-business-name.py new file mode 100755 index 0000000..558c269 --- /dev/null +++ b/bin/stage-6/uuid-business-3-by-business-name.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# Script Name: uuid-business-3-by-business-name + +import os +import csv +import uuid +from collections import defaultdict, Counter +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '10-uuid-business-2-by-root-domain.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '11-uuid-business-3-by-business-name.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def find_best_uuid(existing_uuids, reference_row, candidate_rows): + """Find the best UUID to assign based on the frequency and similarity to the reference row.""" + # Count the frequency of each existing UUID + uuid_counter = Counter(existing_uuids) + most_common_uuid, most_common_count = uuid_counter.most_common(1)[0] + + if len(uuid_counter) == 1: + return most_common_uuid + + # Find the UUID from the row with the most matches + best_uuid = most_common_uuid + best_match_count = -1 + + for candidate_row in candidate_rows: + candidate_uuid = candidate_row['Business UUID'] + if candidate_uuid in uuid_counter: + match_count = sum(1 for key in ['YiB', 'GBP Matching Service', 'GBP Business Category', 'GBP Review Count', 'GBP Review Rating'] + if reference_row[key] == candidate_row[key]) + if match_count > best_match_count: + best_match_count = match_count + best_uuid = candidate_uuid + + return best_uuid + +def add_business_uuid(): + """Add Business UUID to each row grouped by Business Name.""" + name_to_rows = defaultdict(list) + + # Read the input CSV and group rows by Business Name + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business Name' not in headers or 'Business UUID' not in headers: + print("Error: 'Business Name' or 'Business UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + for row in rows: + name = row['Business Name'].strip() + if name: # Ignore empty cells + name_to_rows[name].append(row) + + print(f"Total groups by business name: {len(name_to_rows)}") + + # Assign UUIDs to groups with more than one entry + for name, grouped_rows in name_to_rows.items(): + if len(grouped_rows) > 1: # Only process groups with more than one entry + # Find if any row already has a Business UUID + existing_uuids = [row['Business UUID'] for row in grouped_rows if row['Business UUID']] + if existing_uuids: + best_uuid = find_best_uuid(existing_uuids, grouped_rows[0], grouped_rows) + else: + best_uuid = generate_uuid() + + # Assign the UUID to all rows in the group that do not already have a UUID + for row in grouped_rows: + if not row['Business UUID']: + row['Business UUID'] = best_uuid + + print(f"Total groups assigned UUIDs: {len([group for group in name_to_rows.values() if len(group) > 1])}") + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-3-by-business-name".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs by Business Name', spinner='dots') + spinner.start() + add_business_uuid() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() + diff --git a/bin/stage-6/uuid-business-4-single-location-entries.md b/bin/stage-6/uuid-business-4-single-location-entries.md new file mode 100644 index 0000000..7dbe65e --- /dev/null +++ b/bin/stage-6/uuid-business-4-single-location-entries.md @@ -0,0 +1,48 @@ +# Prompt 1 + +- [Description]: This project revolves around creating a set of python scripts which we will use to parse CSV data. The CSV data represents Google Business Profile data. It was scraped from Google Maps. Our purpose is to clean this data up and prepare it for use with a toolkit of custom made python scripts. +- [Project Root]: "/home/ld/mgk-scrapes" +- [Current Dataset]: "[Project Root]/current-data/" +- [Data Directory]: "[Current Dataset]/.data/" +- [Parent Data]: "[Current Dataset]/"(All directories that do not begin with a period ".") +- [Stage Directories]: "[Data Directory]/stage-#" (Where # is a number.) +- [State Directories]: The state directories are the directories directly under one of the [Stage Directories]. These directories store data associated with a specific state. +- [County Directories]: The coiunty directories are the directories directly under one of the [State Directories]. These directories store data associated with a specific county inside of the state the county directory is under. +- [Stage 1]: "[Data Directory]/stage-1/" +- [Stage 2]: "[Data Directory]/stage-2/" +- [Stage 3]: "[Data Directory]/stage-3/" +- [Stage 4]: "[Data Directory]/stage-4/" +- [Stage 5]: "[Data Directory]/stage-5/" +- [Stage 6]: "[Data Directory]/stage-6/" +- [Binaries]: "[Project Root]/bin/" +- [Stage 1 Binaries]: "[Binaries]/stage-1/" +- [Stage 2 Binaries]: "[Binaries]/stage-2/" +- [Stage 3 Binaries]: "[Binaries]/stage-3/" +- [Stage 4 Binaries]: "[Binaries]/stage-4/" +- [Stage 5 Binaries]: "[Binaries]/stage-5/" +- [Stage 6 Binaries]: "[Binaries]/stage-6/" +- [Bad Matching Services]: "[Data Directory]/bad-matching-services.txt" +- [GBP Business Categories]: "[Data Directory]/gbp-business-categories.txt" +- [GBP Matching Services]: "[Data Directory]/gbp-matching-services.txt" +- [Unknown Blacklist]: "[Data Directory]/unknown-blacklist.txt" +- [Rule 1]: All scripts need to be able to be ran from any directory. +- [Rule 2]: All scripts need output what they are doing, as they are doing it. +- [Rule 3]: All tasks should use halo to report successes and failures. +- [Rule 4]: The python module "tqdm" can be used to report progress when appropriate. +- [Rule 5]: Scripts must be heavily commented, describing the purpose of the script and what each code block inside the code is for. +- [Rule 6]: Every script should begin with a hashbang "#!/usr/bin/env python". +- [Rule 7]: Every script should have a name, and the name of the script should be immediately beneath the hashbang in a comment field prefixed by a string that reads "Script Name: ". +- [Rule 8]: Scripts should generally have robust error checking. +- [Rule 9]: The first output from any script should be pyfiglet outputting the name of the script. The name that pyfiglet outputs though should be a modified version of the script name. The pyfiglet script name needs to replace the hyphens with spaces in the script name, and capitalize the words in the script name after the hyphens have been replaced. + + + +The Project Details above contain locations for files, data, scripts, and rules to follow when creating project scripts and toolkit. + +I wish to create a new script named 'uuid-business-4-single-location-entries' which will use the csv file "13-filled-in-missing-data.csv" as input, and will output to the csv file "14-uuid-business-4-single-location-entries.csv" inside of [Stage 6]. + +This script should assign a UUID to all remaining businesses that have a missing "Business UUID". For rows that do not have an entry in their "Business UUID" field, create a UUID and provide it to that for for that field. + +At the same time, for the same rows, create a second UUID and assign that to the "Location UUID" field. + + diff --git a/bin/stage-6/uuid-business-4-single-location-entries.py b/bin/stage-6/uuid-business-4-single-location-entries.py new file mode 100755 index 0000000..b8d940c --- /dev/null +++ b/bin/stage-6/uuid-business-4-single-location-entries.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# Script Name: uuid-business-4-single-location-entries + +import os +import csv +import uuid +from pyfiglet import Figlet +from halo import Halo + +# Define the paths based on the project details +PROJECT_ROOT = "/home/ld/mgk-scrapes" +STAGE_6_DIRECTORY = os.path.join(PROJECT_ROOT, 'current-data', '.data', 'stage-6') +INPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '13-filled-in-missing-data.csv') +OUTPUT_CSV = os.path.join(STAGE_6_DIRECTORY, '14-uuid-business-4-single-location-entries.csv') + +def generate_uuid(): + """Generate a new UUID.""" + return str(uuid.uuid4()) + +def assign_uuids(): + """Assign UUIDs to rows with missing Business UUID and Location UUID.""" + with open(INPUT_CSV, 'r') as infile: + reader = csv.DictReader(infile) + headers = reader.fieldnames + if 'Business UUID' not in headers or 'Location UUID' not in headers: + print("Error: 'Business UUID' or 'Location UUID' column is missing.") + return + + rows = list(reader) + print(f"Total rows read from input file: {len(rows)}") + + # Assign new UUIDs to rows with missing Business UUID + for row in rows: + if not row['Business UUID'].strip(): + row['Business UUID'] = generate_uuid() + row['Location UUID'] = generate_uuid() + + # Write to the output CSV + with open(OUTPUT_CSV, 'w', newline='') as outfile: + writer = csv.DictWriter(outfile, fieldnames=headers) + writer.writeheader() + for row in rows: + writer.writerow(row) + + print(f"Total rows written to output file: {len(rows)}") + +def main(): + figlet = Figlet(font='slant') + script_name = "uuid-business-4-single-location-entries".replace("-", " ").title() + print(figlet.renderText(script_name)) + + # Check if the output file already exists and delete it if it does + if os.path.exists(OUTPUT_CSV): + os.remove(OUTPUT_CSV) + print(f"Existing output file {OUTPUT_CSV} found and deleted.") + + spinner = Halo(text='Assigning UUIDs to single location entries', spinner='dots') + spinner.start() + assign_uuids() + spinner.succeed('UUIDs assigned successfully.') + + print(f"Processed input CSV: {INPUT_CSV}") + print(f"Output CSV: {OUTPUT_CSV}") + +if __name__ == '__main__': + main() +