Added an export-all-tables and export-all-functions script, along with their exports.
This commit is contained in:
parent
fee16c6bb7
commit
b3b5d1fbc1
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# export-all-functions.sh
|
||||
# Purpose: Export all functions from the database to individual SQL files
|
||||
# Location: bin/create-functions/export-all-functions.sh
|
||||
# Location: bin/db-admin/export-all-functions.sh
|
||||
|
||||
# Get the absolute path of the script directory
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
215
bin/table-admin/export-all-tables.sh
Normal file
215
bin/table-admin/export-all-tables.sh
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
#!/bin/bash
|
||||
|
||||
# export-all-tables.sh
|
||||
# Purpose: Export all table definitions from the database to individual SQL files
|
||||
# Location: bin/table-admin/export-all-tables.sh
|
||||
|
||||
# Get the absolute path of the script directory
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/../.." && pwd )"
|
||||
|
||||
# Load database configuration using absolute path
|
||||
source "$PROJECT_ROOT/.database"
|
||||
|
||||
# Set up logging with absolute paths
|
||||
LOG_DIR="$PROJECT_ROOT/logs"
|
||||
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-export-tables.log"
|
||||
|
||||
# Create log directory if it doesn't exist
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Function to log messages
|
||||
log_message() {
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "$timestamp - $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Create exports directory if it doesn't exist using absolute path
|
||||
EXPORT_DIR="$PROJECT_ROOT/exports/tables"
|
||||
mkdir -p "$EXPORT_DIR"
|
||||
|
||||
# Clean the export directory
|
||||
rm -f "$EXPORT_DIR"/*.sql
|
||||
|
||||
log_message "Starting table export process"
|
||||
|
||||
# Use PGPASSWORD environment variable for authentication
|
||||
export PGPASSWORD="$DB_PASSWORD"
|
||||
|
||||
# Get list of all tables in the database and remove carriage returns
|
||||
TABLES=$(psql -h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-t \
|
||||
-c "SELECT tablename
|
||||
FROM pg_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY tablename;" | \
|
||||
tr -d '\r' | tr -d ' ')
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
log_message "Error: Failed to retrieve table list"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export each table
|
||||
echo "$TABLES" | while read -r TABLE; do
|
||||
if [ -z "$TABLE" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
log_message "Exporting table: $TABLE"
|
||||
OUTPUT_FILE="$EXPORT_DIR/${TABLE}.sql"
|
||||
|
||||
# Get table definition including constraints, indexes, and triggers
|
||||
(
|
||||
# Get table structure with column definitions
|
||||
psql -h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-A \
|
||||
-q \
|
||||
-c "SELECT format('CREATE TABLE public.%I (
|
||||
%s
|
||||
);',
|
||||
table_name,
|
||||
string_agg(
|
||||
format(' %I %s%s%s%s',
|
||||
column_name,
|
||||
udt_name,
|
||||
CASE
|
||||
WHEN character_maximum_length IS NOT NULL
|
||||
THEN format('(%s)', character_maximum_length)
|
||||
WHEN numeric_precision IS NOT NULL AND numeric_scale IS NOT NULL
|
||||
THEN format('(%s,%s)', numeric_precision, numeric_scale)
|
||||
ELSE ''
|
||||
END,
|
||||
CASE WHEN is_nullable = 'NO' THEN ' NOT NULL' ELSE '' END,
|
||||
CASE WHEN column_default IS NOT NULL
|
||||
THEN format(' DEFAULT %s', column_default)
|
||||
ELSE ''
|
||||
END
|
||||
),
|
||||
E',\n'
|
||||
)
|
||||
)
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = '${TABLE}'
|
||||
GROUP BY table_name;"
|
||||
|
||||
# Get primary key constraints
|
||||
psql -h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-A \
|
||||
-q \
|
||||
-c "SELECT format(
|
||||
'ALTER TABLE public.%I ADD CONSTRAINT %I PRIMARY KEY (%s);',
|
||||
tc.table_name,
|
||||
tc.constraint_name,
|
||||
string_agg(kcu.column_name, ', ')
|
||||
)
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
WHERE tc.table_schema = 'public'
|
||||
AND tc.table_name = '${TABLE}'
|
||||
AND tc.constraint_type = 'PRIMARY KEY'
|
||||
GROUP BY tc.table_name, tc.constraint_name;"
|
||||
|
||||
# Get foreign key constraints
|
||||
psql -h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-A \
|
||||
-q \
|
||||
-c "SELECT format(
|
||||
'ALTER TABLE public.%I ADD CONSTRAINT %I FOREIGN KEY (%s) REFERENCES public.%I (%s);',
|
||||
kcu1.table_name,
|
||||
kcu1.constraint_name,
|
||||
string_agg(DISTINCT kcu1.column_name, ', '),
|
||||
kcu2.table_name,
|
||||
string_agg(DISTINCT kcu2.column_name, ', ')
|
||||
)
|
||||
FROM information_schema.referential_constraints AS rc
|
||||
JOIN information_schema.key_column_usage AS kcu1
|
||||
ON kcu1.constraint_catalog = rc.constraint_catalog
|
||||
AND kcu1.constraint_schema = rc.constraint_schema
|
||||
AND kcu1.constraint_name = rc.constraint_name
|
||||
JOIN information_schema.key_column_usage AS kcu2
|
||||
ON kcu2.constraint_catalog = rc.unique_constraint_catalog
|
||||
AND kcu2.constraint_schema = rc.unique_constraint_schema
|
||||
AND kcu2.constraint_name = rc.unique_constraint_name
|
||||
AND kcu2.ordinal_position = kcu1.ordinal_position
|
||||
WHERE kcu1.table_schema = 'public'
|
||||
AND kcu1.table_name = '${TABLE}'
|
||||
GROUP BY kcu1.table_name, kcu1.constraint_name, kcu2.table_name;"
|
||||
|
||||
# Get unique constraints
|
||||
psql -h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-A \
|
||||
-q \
|
||||
-c "SELECT format(
|
||||
'ALTER TABLE public.%I ADD CONSTRAINT %I UNIQUE (%s);',
|
||||
tc.table_name,
|
||||
tc.constraint_name,
|
||||
string_agg(kcu.column_name, ', ')
|
||||
)
|
||||
FROM information_schema.table_constraints tc
|
||||
JOIN information_schema.key_column_usage kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
WHERE tc.table_schema = 'public'
|
||||
AND tc.table_name = '${TABLE}'
|
||||
AND tc.constraint_type = 'UNIQUE'
|
||||
GROUP BY tc.table_name, tc.constraint_name;"
|
||||
|
||||
# Get indexes (excluding those for constraints)
|
||||
psql -h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-A \
|
||||
-q \
|
||||
-c "SELECT indexdef || ';'
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'public'
|
||||
AND tablename = '${TABLE}'
|
||||
AND indexname NOT IN (
|
||||
SELECT constraint_name
|
||||
FROM information_schema.table_constraints
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = '${TABLE}'
|
||||
);"
|
||||
) | tr -d '\r' > "$OUTPUT_FILE"
|
||||
|
||||
if [ $? -eq 0 ] && [ -s "$OUTPUT_FILE" ]; then
|
||||
SIZE=$(wc -c < "$OUTPUT_FILE")
|
||||
log_message "Successfully exported $TABLE to $OUTPUT_FILE (${SIZE} bytes)"
|
||||
else
|
||||
log_message "Error: Failed to export table $TABLE"
|
||||
rm -f "$OUTPUT_FILE" # Remove empty file if export failed
|
||||
fi
|
||||
done
|
||||
|
||||
# Unset password environment variable for security
|
||||
unset PGPASSWORD
|
||||
|
||||
log_message "Table export process completed"
|
||||
|
||||
# List exported tables
|
||||
log_message "Exported tables:"
|
||||
for file in "$EXPORT_DIR"/*.sql; do
|
||||
SIZE=$(wc -c < "$file")
|
||||
BASENAME=$(basename "$file")
|
||||
log_message " - $BASENAME ($SIZE bytes)"
|
||||
done
|
||||
19
exports/tables/importedrows.sql
Normal file
19
exports/tables/importedrows.sql
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
format
|
||||
CREATE TABLE public.importedrows (
|
||||
row_id int4(32,0) NOT NULL,
|
||||
import_id int4(32,0),
|
||||
row_number int4(32,0),
|
||||
row_data jsonb,
|
||||
imported_to_main bool DEFAULT false
|
||||
);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.importedrows ADD CONSTRAINT importedrows_pkey PRIMARY KEY (row_id);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.importedrows ADD CONSTRAINT importedrows_import_id_fkey FOREIGN KEY (import_id) REFERENCES public.importhistory (import_id);
|
||||
(1 row)
|
||||
format
|
||||
(0 rows)
|
||||
?column?
|
||||
(0 rows)
|
||||
24
exports/tables/importhistory.sql
Normal file
24
exports/tables/importhistory.sql
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
format
|
||||
CREATE TABLE public.importhistory (
|
||||
import_id int4(32,0) NOT NULL,
|
||||
import_date timestamp DEFAULT CURRENT_TIMESTAMP,
|
||||
source_type varchar(50),
|
||||
filename varchar(255),
|
||||
original_seed_keyword varchar(255),
|
||||
confirmed_seed_keyword_id int4(32,0),
|
||||
row_count int4(32,0),
|
||||
import_status varchar(50),
|
||||
error_details text,
|
||||
import_notes varchar(1000)
|
||||
);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.importhistory ADD CONSTRAINT importhistory_pkey PRIMARY KEY (import_id);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.importhistory ADD CONSTRAINT importhistory_confirmed_seed_keyword_id_fkey FOREIGN KEY (confirmed_seed_keyword_id) REFERENCES public.keywords (keyword_id);
|
||||
(1 row)
|
||||
format
|
||||
(0 rows)
|
||||
?column?
|
||||
(0 rows)
|
||||
18
exports/tables/importrawdata.sql
Normal file
18
exports/tables/importrawdata.sql
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
format
|
||||
CREATE TABLE public.importrawdata (
|
||||
import_id int4(32,0) NOT NULL,
|
||||
header_row text,
|
||||
column_mappings jsonb,
|
||||
raw_data text
|
||||
);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.importrawdata ADD CONSTRAINT importrawdata_pkey PRIMARY KEY (import_id);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.importrawdata ADD CONSTRAINT importrawdata_import_id_fkey FOREIGN KEY (import_id) REFERENCES public.importhistory (import_id);
|
||||
(1 row)
|
||||
format
|
||||
(0 rows)
|
||||
?column?
|
||||
(0 rows)
|
||||
28
exports/tables/keywords.sql
Normal file
28
exports/tables/keywords.sql
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
format
|
||||
CREATE TABLE public.keywords (
|
||||
keyword_id int4(32,0) NOT NULL,
|
||||
keyword varchar(255) NOT NULL,
|
||||
gkwp_search_volume int4(32,0),
|
||||
gkwp_competition_index int4(32,0),
|
||||
gkwp_cpc_min numeric(10,2),
|
||||
gkwp_cpc_max numeric(10,2),
|
||||
gkwp_last_updated timestamp,
|
||||
semrush_search_volume int4(32,0),
|
||||
semrush_difficulty int4(32,0),
|
||||
semrush_cpc numeric(10,2),
|
||||
semrush_intent varchar(50),
|
||||
semrush_last_updated timestamp,
|
||||
first_imported timestamp DEFAULT CURRENT_TIMESTAMP,
|
||||
last_updated timestamp DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.keywords ADD CONSTRAINT keywords_pkey PRIMARY KEY (keyword_id);
|
||||
(1 row)
|
||||
format
|
||||
(0 rows)
|
||||
format
|
||||
(0 rows)
|
||||
?column?
|
||||
CREATE UNIQUE INDEX unique_lowercase_keyword ON public.keywords USING btree (lower((keyword)::text));
|
||||
(1 row)
|
||||
20
exports/tables/seedkeywordusage.sql
Normal file
20
exports/tables/seedkeywordusage.sql
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
format
|
||||
CREATE TABLE public.seedkeywordusage (
|
||||
usage_id int4(32,0) NOT NULL,
|
||||
keyword_id int4(32,0) NOT NULL,
|
||||
import_id int4(32,0) NOT NULL,
|
||||
source_type varchar(50),
|
||||
used_date timestamp DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.seedkeywordusage ADD CONSTRAINT seedkeywordusage_pkey PRIMARY KEY (usage_id);
|
||||
(1 row)
|
||||
format
|
||||
ALTER TABLE public.seedkeywordusage ADD CONSTRAINT seedkeywordusage_import_id_fkey FOREIGN KEY (import_id) REFERENCES public.importhistory (import_id);
|
||||
ALTER TABLE public.seedkeywordusage ADD CONSTRAINT seedkeywordusage_keyword_id_fkey FOREIGN KEY (keyword_id) REFERENCES public.keywords (keyword_id);
|
||||
(2 rows)
|
||||
format
|
||||
(0 rows)
|
||||
?column?
|
||||
(0 rows)
|
||||
Loading…
Reference in a new issue