From b3b5d1fbc19aff7da8ef52bf3bc79d9a35bd531d Mon Sep 17 00:00:00 2001 From: Lord_Devi Date: Tue, 19 Nov 2024 08:01:59 -0500 Subject: [PATCH] Added an export-all-tables and export-all-functions script, along with their exports. --- .../export-all-functions.sh | 2 +- bin/table-admin/export-all-tables.sh | 215 ++++++++++++++++++ exports/tables/importedrows.sql | 19 ++ exports/tables/importhistory.sql | 24 ++ exports/tables/importrawdata.sql | 18 ++ exports/tables/keywords.sql | 28 +++ exports/tables/seedkeywordusage.sql | 20 ++ 7 files changed, 325 insertions(+), 1 deletion(-) rename bin/{create-functions => db-admin}/export-all-functions.sh (98%) create mode 100644 bin/table-admin/export-all-tables.sh create mode 100644 exports/tables/importedrows.sql create mode 100644 exports/tables/importhistory.sql create mode 100644 exports/tables/importrawdata.sql create mode 100644 exports/tables/keywords.sql create mode 100644 exports/tables/seedkeywordusage.sql diff --git a/bin/create-functions/export-all-functions.sh b/bin/db-admin/export-all-functions.sh similarity index 98% rename from bin/create-functions/export-all-functions.sh rename to bin/db-admin/export-all-functions.sh index 63f44b4..6800d67 100644 --- a/bin/create-functions/export-all-functions.sh +++ b/bin/db-admin/export-all-functions.sh @@ -2,7 +2,7 @@ # export-all-functions.sh # Purpose: Export all functions from the database to individual SQL files -# Location: bin/create-functions/export-all-functions.sh +# Location: bin/db-admin/export-all-functions.sh # Get the absolute path of the script directory SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/bin/table-admin/export-all-tables.sh b/bin/table-admin/export-all-tables.sh new file mode 100644 index 0000000..4d09ff0 --- /dev/null +++ b/bin/table-admin/export-all-tables.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +# export-all-tables.sh +# Purpose: Export all table definitions from the database to individual SQL files +# Location: bin/table-admin/export-all-tables.sh + +# Get the absolute path of the script directory +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/../.." && pwd )" + +# Load database configuration using absolute path +source "$PROJECT_ROOT/.database" + +# Set up logging with absolute paths +LOG_DIR="$PROJECT_ROOT/logs" +LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-export-tables.log" + +# Create log directory if it doesn't exist +mkdir -p "$LOG_DIR" + +# Function to log messages +log_message() { + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo "$timestamp - $1" | tee -a "$LOG_FILE" +} + +# Create exports directory if it doesn't exist using absolute path +EXPORT_DIR="$PROJECT_ROOT/exports/tables" +mkdir -p "$EXPORT_DIR" + +# Clean the export directory +rm -f "$EXPORT_DIR"/*.sql + +log_message "Starting table export process" + +# Use PGPASSWORD environment variable for authentication +export PGPASSWORD="$DB_PASSWORD" + +# Get list of all tables in the database and remove carriage returns +TABLES=$(psql -h "$DB_HOST" \ + -p "$DB_PORT" \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -t \ + -c "SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + ORDER BY tablename;" | \ + tr -d '\r' | tr -d ' ') + +if [ $? -ne 0 ]; then + log_message "Error: Failed to retrieve table list" + exit 1 +fi + +# Export each table +echo "$TABLES" | while read -r TABLE; do + if [ -z "$TABLE" ]; then + continue + fi + + log_message "Exporting table: $TABLE" + OUTPUT_FILE="$EXPORT_DIR/${TABLE}.sql" + + # Get table definition including constraints, indexes, and triggers + ( + # Get table structure with column definitions + psql -h "$DB_HOST" \ + -p "$DB_PORT" \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -A \ + -q \ + -c "SELECT format('CREATE TABLE public.%I ( +%s +);', + table_name, + string_agg( + format(' %I %s%s%s%s', + column_name, + udt_name, + CASE + WHEN character_maximum_length IS NOT NULL + THEN format('(%s)', character_maximum_length) + WHEN numeric_precision IS NOT NULL AND numeric_scale IS NOT NULL + THEN format('(%s,%s)', numeric_precision, numeric_scale) + ELSE '' + END, + CASE WHEN is_nullable = 'NO' THEN ' NOT NULL' ELSE '' END, + CASE WHEN column_default IS NOT NULL + THEN format(' DEFAULT %s', column_default) + ELSE '' + END + ), + E',\n' + ) + ) + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = '${TABLE}' + GROUP BY table_name;" + + # Get primary key constraints + psql -h "$DB_HOST" \ + -p "$DB_PORT" \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -A \ + -q \ + -c "SELECT format( + 'ALTER TABLE public.%I ADD CONSTRAINT %I PRIMARY KEY (%s);', + tc.table_name, + tc.constraint_name, + string_agg(kcu.column_name, ', ') + ) + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + WHERE tc.table_schema = 'public' + AND tc.table_name = '${TABLE}' + AND tc.constraint_type = 'PRIMARY KEY' + GROUP BY tc.table_name, tc.constraint_name;" + + # Get foreign key constraints + psql -h "$DB_HOST" \ + -p "$DB_PORT" \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -A \ + -q \ + -c "SELECT format( + 'ALTER TABLE public.%I ADD CONSTRAINT %I FOREIGN KEY (%s) REFERENCES public.%I (%s);', + kcu1.table_name, + kcu1.constraint_name, + string_agg(DISTINCT kcu1.column_name, ', '), + kcu2.table_name, + string_agg(DISTINCT kcu2.column_name, ', ') + ) + FROM information_schema.referential_constraints AS rc + JOIN information_schema.key_column_usage AS kcu1 + ON kcu1.constraint_catalog = rc.constraint_catalog + AND kcu1.constraint_schema = rc.constraint_schema + AND kcu1.constraint_name = rc.constraint_name + JOIN information_schema.key_column_usage AS kcu2 + ON kcu2.constraint_catalog = rc.unique_constraint_catalog + AND kcu2.constraint_schema = rc.unique_constraint_schema + AND kcu2.constraint_name = rc.unique_constraint_name + AND kcu2.ordinal_position = kcu1.ordinal_position + WHERE kcu1.table_schema = 'public' + AND kcu1.table_name = '${TABLE}' + GROUP BY kcu1.table_name, kcu1.constraint_name, kcu2.table_name;" + + # Get unique constraints + psql -h "$DB_HOST" \ + -p "$DB_PORT" \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -A \ + -q \ + -c "SELECT format( + 'ALTER TABLE public.%I ADD CONSTRAINT %I UNIQUE (%s);', + tc.table_name, + tc.constraint_name, + string_agg(kcu.column_name, ', ') + ) + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + WHERE tc.table_schema = 'public' + AND tc.table_name = '${TABLE}' + AND tc.constraint_type = 'UNIQUE' + GROUP BY tc.table_name, tc.constraint_name;" + + # Get indexes (excluding those for constraints) + psql -h "$DB_HOST" \ + -p "$DB_PORT" \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + -A \ + -q \ + -c "SELECT indexdef || ';' + FROM pg_indexes + WHERE schemaname = 'public' + AND tablename = '${TABLE}' + AND indexname NOT IN ( + SELECT constraint_name + FROM information_schema.table_constraints + WHERE table_schema = 'public' + AND table_name = '${TABLE}' + );" + ) | tr -d '\r' > "$OUTPUT_FILE" + + if [ $? -eq 0 ] && [ -s "$OUTPUT_FILE" ]; then + SIZE=$(wc -c < "$OUTPUT_FILE") + log_message "Successfully exported $TABLE to $OUTPUT_FILE (${SIZE} bytes)" + else + log_message "Error: Failed to export table $TABLE" + rm -f "$OUTPUT_FILE" # Remove empty file if export failed + fi +done + +# Unset password environment variable for security +unset PGPASSWORD + +log_message "Table export process completed" + +# List exported tables +log_message "Exported tables:" +for file in "$EXPORT_DIR"/*.sql; do + SIZE=$(wc -c < "$file") + BASENAME=$(basename "$file") + log_message " - $BASENAME ($SIZE bytes)" +done diff --git a/exports/tables/importedrows.sql b/exports/tables/importedrows.sql new file mode 100644 index 0000000..0f279a1 --- /dev/null +++ b/exports/tables/importedrows.sql @@ -0,0 +1,19 @@ +format +CREATE TABLE public.importedrows ( + row_id int4(32,0) NOT NULL, + import_id int4(32,0), + row_number int4(32,0), + row_data jsonb, + imported_to_main bool DEFAULT false +); +(1 row) +format +ALTER TABLE public.importedrows ADD CONSTRAINT importedrows_pkey PRIMARY KEY (row_id); +(1 row) +format +ALTER TABLE public.importedrows ADD CONSTRAINT importedrows_import_id_fkey FOREIGN KEY (import_id) REFERENCES public.importhistory (import_id); +(1 row) +format +(0 rows) +?column? +(0 rows) diff --git a/exports/tables/importhistory.sql b/exports/tables/importhistory.sql new file mode 100644 index 0000000..f717a67 --- /dev/null +++ b/exports/tables/importhistory.sql @@ -0,0 +1,24 @@ +format +CREATE TABLE public.importhistory ( + import_id int4(32,0) NOT NULL, + import_date timestamp DEFAULT CURRENT_TIMESTAMP, + source_type varchar(50), + filename varchar(255), + original_seed_keyword varchar(255), + confirmed_seed_keyword_id int4(32,0), + row_count int4(32,0), + import_status varchar(50), + error_details text, + import_notes varchar(1000) +); +(1 row) +format +ALTER TABLE public.importhistory ADD CONSTRAINT importhistory_pkey PRIMARY KEY (import_id); +(1 row) +format +ALTER TABLE public.importhistory ADD CONSTRAINT importhistory_confirmed_seed_keyword_id_fkey FOREIGN KEY (confirmed_seed_keyword_id) REFERENCES public.keywords (keyword_id); +(1 row) +format +(0 rows) +?column? +(0 rows) diff --git a/exports/tables/importrawdata.sql b/exports/tables/importrawdata.sql new file mode 100644 index 0000000..1c28018 --- /dev/null +++ b/exports/tables/importrawdata.sql @@ -0,0 +1,18 @@ +format +CREATE TABLE public.importrawdata ( + import_id int4(32,0) NOT NULL, + header_row text, + column_mappings jsonb, + raw_data text +); +(1 row) +format +ALTER TABLE public.importrawdata ADD CONSTRAINT importrawdata_pkey PRIMARY KEY (import_id); +(1 row) +format +ALTER TABLE public.importrawdata ADD CONSTRAINT importrawdata_import_id_fkey FOREIGN KEY (import_id) REFERENCES public.importhistory (import_id); +(1 row) +format +(0 rows) +?column? +(0 rows) diff --git a/exports/tables/keywords.sql b/exports/tables/keywords.sql new file mode 100644 index 0000000..540f544 --- /dev/null +++ b/exports/tables/keywords.sql @@ -0,0 +1,28 @@ +format +CREATE TABLE public.keywords ( + keyword_id int4(32,0) NOT NULL, + keyword varchar(255) NOT NULL, + gkwp_search_volume int4(32,0), + gkwp_competition_index int4(32,0), + gkwp_cpc_min numeric(10,2), + gkwp_cpc_max numeric(10,2), + gkwp_last_updated timestamp, + semrush_search_volume int4(32,0), + semrush_difficulty int4(32,0), + semrush_cpc numeric(10,2), + semrush_intent varchar(50), + semrush_last_updated timestamp, + first_imported timestamp DEFAULT CURRENT_TIMESTAMP, + last_updated timestamp DEFAULT CURRENT_TIMESTAMP +); +(1 row) +format +ALTER TABLE public.keywords ADD CONSTRAINT keywords_pkey PRIMARY KEY (keyword_id); +(1 row) +format +(0 rows) +format +(0 rows) +?column? +CREATE UNIQUE INDEX unique_lowercase_keyword ON public.keywords USING btree (lower((keyword)::text)); +(1 row) diff --git a/exports/tables/seedkeywordusage.sql b/exports/tables/seedkeywordusage.sql new file mode 100644 index 0000000..763accf --- /dev/null +++ b/exports/tables/seedkeywordusage.sql @@ -0,0 +1,20 @@ +format +CREATE TABLE public.seedkeywordusage ( + usage_id int4(32,0) NOT NULL, + keyword_id int4(32,0) NOT NULL, + import_id int4(32,0) NOT NULL, + source_type varchar(50), + used_date timestamp DEFAULT CURRENT_TIMESTAMP +); +(1 row) +format +ALTER TABLE public.seedkeywordusage ADD CONSTRAINT seedkeywordusage_pkey PRIMARY KEY (usage_id); +(1 row) +format +ALTER TABLE public.seedkeywordusage ADD CONSTRAINT seedkeywordusage_import_id_fkey FOREIGN KEY (import_id) REFERENCES public.importhistory (import_id); +ALTER TABLE public.seedkeywordusage ADD CONSTRAINT seedkeywordusage_keyword_id_fkey FOREIGN KEY (keyword_id) REFERENCES public.keywords (keyword_id); +(2 rows) +format +(0 rows) +?column? +(0 rows)