mgk/hirejared-kw-db
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

first commit

Browse source
This commit is contained in:
Lord_Devi 2024-11-19 07:52:08 -05:00
commit fee16c6bb7
25 changed files with 5090 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.database.template Normal file
View file

@ -0,0 +1,6 @@
# .database.template
DB_USER=postgres
DB_PASSWORD=your_password
DB_HOST=localhost
DB_PORT=5432
DB_NAME=keyword_management

98
bin/create-functions/export-all-functions.sh Normal file
View file

@ -0,0 +1,98 @@
#!/bin/bash
# export-all-functions.sh
# Purpose: Export all functions from the database to individual SQL files
# Location: bin/create-functions/export-all-functions.sh
# Get the absolute path of the script directory
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/../.." && pwd )"
# Load database configuration using absolute path
source "$PROJECT_ROOT/.database"
# Set up logging with absolute paths
LOG_DIR="$PROJECT_ROOT/logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-export-functions.log"
# Create log directory if it doesn't exist
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
echo "$timestamp - $1" | tee -a "$LOG_FILE"
}
# Create exports directory if it doesn't exist using absolute path
EXPORT_DIR="$PROJECT_ROOT/exports/functions"
mkdir -p "$EXPORT_DIR"
# Clean the export directory
rm -f "$EXPORT_DIR"/*.sql
log_message "Starting function export process"
# Use PGPASSWORD environment variable for authentication
export PGPASSWORD="$DB_PASSWORD"
# Get list of all functions in the database and remove carriage returns
FUNCTIONS=$(psql -h "$DB_HOST" \
-p "$DB_PORT" \
-U "$DB_USER" \
-d "$DB_NAME" \
-t \
-c "SELECT proname
FROM pg_proc
WHERE pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public');" | \
tr -d '\r' | tr -d ' ')
if [ $? -ne 0 ]; then
log_message "Error: Failed to retrieve function list"
exit 1
fi
# Export each function
echo "$FUNCTIONS" | while read -r FUNC; do
if [ -z "$FUNC" ]; then
continue
fi
log_message "Exporting function: $FUNC"
OUTPUT_FILE="$EXPORT_DIR/${FUNC}.sql"
# Get function definition and properly handle the output
psql -h "$DB_HOST" \
-p "$DB_PORT" \
-U "$DB_USER" \
-d "$DB_NAME" \
-A \
-q \
-c "SELECT pg_get_functiondef(oid)
FROM pg_proc
WHERE proname = '$FUNC'
AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public');" | \
tail -n +2 | \
tr -d '\r' > "$OUTPUT_FILE"
if [ $? -eq 0 ] && [ -s "$OUTPUT_FILE" ]; then
SIZE=$(wc -c < "$OUTPUT_FILE")
log_message "Successfully exported $FUNC to $OUTPUT_FILE (${SIZE} bytes)"
else
log_message "Error: Failed to export function $FUNC"
rm -f "$OUTPUT_FILE" # Remove empty file if export failed
fi
done
# Unset password environment variable for security
unset PGPASSWORD
log_message "Function export process completed"
# List exported functions
log_message "Exported functions:"
for file in "$EXPORT_DIR"/*.sql; do
SIZE=$(wc -c < "$file")
BASENAME=$(basename "$file")
log_message " - $BASENAME ($SIZE bytes)"
done

192
docs/project-docs/01-overview/architecture.md Normal file
View file

@ -0,0 +1,192 @@
# System Architecture
## Overview
The HireJared Keyword Management System employs a hybrid architecture combining command-line database management through bash scripts with a GUI-based operational interface through LibreOffice Base. This design allows for robust system management while maintaining user-friendly daily operations.
## Core Components
### 1. PostgreSQL Database
- **Version**: PostgreSQL 17
- **Configuration**:
- ICU Locale Provider enabled
- UTF-8 encoding
- en-US locale setting
- Case-insensitive collation for keyword handling
- **Security**:
- Connection details managed through `.database` configuration
- Role-based access control
- Connection limiting for resource management
### 2. Bash Script Framework
- **Purpose**: System administration and data processing through SQL execution
- **Primary Function**: Pass SQL statements to PostgreSQL via psql command
- **Initial Organization**:
```
/bin
├── db-admin/ # Database creation, deletion, backup
├── table-admin/ # Table management
├── create-functions/ # Database function management
└── import-scripts/ # Data import processing
```
- **Directory Evolution**:
- Current structure represents initial categories
- New subdirectories will be created as needed
- Categories will evolve based on script purposes
- Directory structure remains flexible for growth
- **Script Design Philosophy**:
- Scripts primarily execute SQL via psql command
- Minimal bash processing, maximum database utilization
- SQL statements stored in script variables or sourced from files
- Consistent error handling and logging
- **Configuration**:
- Centralized database connection management
- Standardized error handling
- Logging framework
- Script dependencies management
### 3. LibreOffice Base Interface
- **Purpose**: Daily operations and data management
- **Components**:
- Custom database views
- Optimized queries
- Form interfaces
- Report templates
- **Integration**:
- Direct connection to PostgreSQL
- View-based data access
- Function utilization through GUI
## Data Flow
### 1. Database Administration
```mermaid
graph TB
A[Bash Scripts] -->|Create/Delete| B[Database]
A -->|Create/Modify| C[Tables]
A -->|Install| D[Functions]
A -->|Create| E[Views]
A -->|Process| F[Data Imports]
```
### 2. Daily Operations
```mermaid
graph LR
A[LibreOffice Base] -->|Read/Write| B[Views]
B -->|Access| C[Tables]
A -->|Execute| D[Functions]
D -->|Modify| C
```
## System Interfaces
### 1. Command Line Interface
- **Primary Users**: System administrators
- **Capabilities**:
- Database creation and deletion
- Schema management
- Bulk data operations
- System maintenance
- Backup and restore
### 2. LibreOffice Base Interface
- **Primary Users**: Marketing team
- **Capabilities**:
- Keyword management
- Campaign organization
- Ad group administration
- Data analysis
- Report generation
## Data Processing Pipeline
### 1. Keyword Import Process
```mermaid
graph LR
A[Import Scripts] -->|Read| B[Source Files]
B -->|Process| C[Staging]
C -->|Validate| D[Keywords Table]
D -->|Update| E[Related Tables]
```
### 2. Campaign Management Process
```mermaid
graph TB
A[LibreOffice Base] -->|Create| B[Campaigns]
A -->|Create| C[Ad Groups]
A -->|Create| D[Landing Pages]
A -->|Assign| E[Keywords]
```
## Security Architecture
### 1. Configuration Management
- Sensitive data stored in `.database` file
- File permissions properly restricted
- Environment-specific configurations
### 2. Database Access
- Role-based access control
- Connection pooling
- SSL encryption for remote connections
### 3. Operation Logging
- Script execution logging
- Database operation logging
- Error tracking and reporting
## Performance Considerations
### 1. Database Optimization
- Proper indexing strategies
- Materialized views where appropriate
- Query optimization
- Regular maintenance scheduling
### 2. Script Performance
- Batch processing for large operations
- Transaction management
- Error recovery mechanisms
- Resource usage monitoring
### 3. LibreOffice Base Optimization
- View-based access optimization
- Form interface performance
- Query template optimization
- Connection pooling
## Backup and Recovery
### 1. Backup Strategy
- Regular automated backups
- Transaction log management
- Point-in-time recovery capability
### 2. Recovery Procedures
- Database restoration scripts
- Data verification tools
- Integrity checking
## System Requirements
### 1. Software Dependencies
- PostgreSQL 15+
- LibreOffice Base
- Bash shell
- Required PostgreSQL extensions
### 2. Hardware Recommendations
- Sufficient RAM for database operations
- Adequate storage for data and backups
- Network capacity for remote access
## Future Scalability
### 1. Planned Expansions
- Additional data source integration
- Enhanced reporting capabilities
- Automated processing features
### 2. Architectural Considerations
- Modular design for easy expansion
- Standardized interfaces
- Documented extension points

79
docs/project-docs/01-overview/directory-layout.md Normal file
View file

@ -0,0 +1,79 @@
# Directory Structure Documentation
## Overview
The project follows a structured directory layout to organize scripts, documentation, and configuration files in a logical manner. This structure supports easy maintenance, clear separation of concerns, and straightforward navigation.
## Root Directory Structure
```
/
├── .database # Database connection configuration
├── bin/ # Script directories
├── doc/ # Documentation
└── logs/ # Script execution logs
```
## Script Directory Structure (`/bin`)
```
/bin
├── db-admin/ # Database administration scripts
│ ├── create-db.sh # Database creation script
│ ├── backup-db.sh # Backup utilities
│ └── restore-db.sh # Restore utilities
│
├── table-admin/ # Table management scripts
│ ├── create-tables/ # Table creation scripts
│ └── alter-tables/ # Table modification scripts
│
├── create-functions/ # Database function creation scripts
│
└── import-scripts/ # Data import scripts
├── gkwp/ # Google Keyword Planner imports
├── semrush/ # SEMRush keyword imports
├── competitor/ # Competitor data imports
└── plain-list/ # Plain keyword list imports
```
## Documentation Structure (`/doc`)
```
/doc
└── project-docs/
├── 01-overview/
├── 02-database/
├── 03-scripts/
├── 04-implementation/
└── 05-operations/
```
## Script Naming Conventions
- All scripts use `.sh` extension
- Names use kebab-case
- Names should be descriptive of function
- Example: `create-keywords-table.sh`
## Configuration Files
### .database Format
```ini
DB_USER=postgres
DB_PASSWORD=your_password
DB_HOST=localhost
DB_PORT=5432
DB_NAME=keyword_management
```
## Script Requirements
1. All scripts must source the .database configuration
2. Scripts must include error handling
3. Scripts should log operations to the logs directory
4. Scripts must check for required prerequisites
## Logging
- Each script creates dated log files
- Logs stored in `/logs` directory
- Format: `YYYY-MM-DD-script-name.log`
## Best Practices
1. Keep scripts focused on single responsibility
2. Maintain consistent error handling
3. Document script purpose in header comments
4. Include usage examples in script headers

199
docs/project-docs/01-overview/project-goals.md Normal file
View file

@ -0,0 +1,199 @@
# HireJared Keyword Management System
## Project Overview
The HireJared Keyword Management System uses a dual-interface approach: bash scripts for database administration and structure management, combined with LibreOffice Base for daily operational use. The system provides robust keyword data management, import capabilities from multiple sources, and keyword organization features through this hybrid approach of command-line scripting and GUI-based database interaction.
## Interface Strategy
### Command-Line Interface (Bash Scripts)
- Database creation and structure management
- Table creation and modification
- Data import processing
- Bulk operations and maintenance tasks
- System updates and modifications
- Database backup and restoration
### LibreOffice Base Interface
- Daily keyword management operations
- Ad group and campaign management
- Query building and data analysis
- Report generation
- Data entry and updates
- View-based data access and manipulation
### Database Views and Functions
- Create optimized views for common operations
- Implement functions for frequently used queries
- Design views for specific LibreOffice Base forms
- Maintain consistent data access patterns
- Support efficient filtering and sorting operations
## Core Objectives
### 1. Data Centralization
- Create a central repository for all campaign keywords
- Support multiple data sources (GKWP, SEMRush, plain lists)
- Maintain historical data and import records
- Eliminate duplicate keywords through case-insensitive storage
### 2. Campaign Structure Management
- Maintain hierarchical organization of campaigns, ad groups, and keywords
- Track landing pages and their associations with ad groups
- Support campaign planning and organization
- Enable bulk operations for campaign management
#### Campaign Management
- Store and track all advertising campaigns
- Maintain campaign metadata and settings
- Support campaign-level organization and reporting
#### Ad Group Management
- Create and manage ad groups within campaigns
- Associate single landing pages with ad groups
- Track ad group performance and settings
- Support bulk ad group creation during keyword imports
#### Landing Page Integration
- Maintain database of all campaign landing pages
- Track landing page assignments to ad groups
- Ensure one-to-one relationship between ad groups and landing pages
- Support landing page performance tracking
### 3. Data Import Support
- Google Keyword Planner (GKWP) data imports
- SEMRush keyword data imports
- SEMRush competitor keyword data imports
- Plain keyword list imports for existing campaign keywords
- Support for direct ad group assignment during import
- Option to create new ad groups during import process
- Bulk keyword assignment to existing ad groups
- Support for importing current campaign structure and keywords
### 4. Data Organization
- Support for ad group assignments
- Tracking of keyword metrics from multiple sources
- Ability to identify and track seed keywords
- Support for future competitor analysis features
## Technical Requirements
### Database
- PostgreSQL with ICU locale support
- UTF-8 encoding
- Case-insensitive keyword storage
- Proper handling of currency and numeric data
- Optimized views for LibreOffice Base operations
- Functions to support common data operations
### Script Organization
- Modular bash scripts organized by function
- Centralized configuration management
- Clear separation of concerns between different operations
- Comprehensive error handling and logging
- Scripts for creating and maintaining database views
- Support for LibreOffice Base integration
### LibreOffice Base Integration
- Maintained set of database views
- Optimized query designs
- Form-specific view implementations
- Support for common filtering operations
- Efficient data entry interfaces
### Configuration Management
- Database connection details stored in `.database` configuration file
- Support for multiple environment configurations
- Secure credential management
## Next Steps
### 1. Campaign Structure Implementation
- Create campaigns table
- Create ad groups table
- Create landing pages table
- Establish relationships between tables
- Create management scripts for campaigns, ad groups, and landing pages
### 2. LibreOffice Base Integration
- Create optimized views for common operations
- Design query templates for frequent operations
- Implement supporting functions
- Create view-based forms for data entry
- Establish efficient data browsing interfaces
### 3. Keyword Import and Assignment
- Develop plain keyword list import functionality
- Support for existing campaign keywords
- Ad group assignment during import
- New ad group creation during import
- Create bulk assignment tools for keywords to ad groups
### 4. Data Source Integration
- Implement GKWP data import functionality
- Develop SEMRush data import capabilities
- Create competitor data import features
### 5. Management Interface
- Create scripts for campaign management
- Develop ad group management tools
- Implement landing page association features
- Build keyword assignment utilities
## Future Enhancements
1. Web page integration for ad targeting
2. Enhanced competitor analysis features
3. Automated processing and scheduled imports
4. Advanced reporting capabilities
5. Performance tracking and optimization tools
## Initial Implementation Priority
### Phase 1: Database Foundation
1. Create core database management scripts
- Script to create keyword_management database with ICU locale support
- Script to delete keyword_management database for testing
- Scripts to verify database creation and configuration
- Database backup and restore functionality
2. Implement current working functionality
- Script to create the Keywords table
- Scripts to create established GKWP management functions
- Scripts to verify function creation and testing
- Implementation of existing unique constraints
3. Create LibreOffice Base integration
- Implement essential views for current functionality
- Verify proper function access through LibreOffice Base
- Test keyword data management through GUI
### Phase 2: Campaign Structure Implementation
1. Create campaign structure tables
- Campaigns table
- Ad Groups table
- Landing Pages table
- Relationship management between tables
2. Implement campaign structure views
- Create views for LibreOffice Base integration
- Implement necessary management functions
- Create verification and testing scripts
### Phase 3: Keyword Management
1. Develop plain keyword list import functionality
- Import existing campaign keywords
- Support ad group assignment during import
- Enable new ad group creation during import
2. Create keyword assignment tools
- Bulk assignment utilities
- Ad group association scripts
- Import verification tools
3. Implement supporting views
- Keyword management views
- Assignment verification views
- Import status views
Each phase builds upon the successful implementation and testing of the previous phase, ensuring a solid foundation for the system. The priority is to first establish and document our current working functionality before expanding to new features.
This will enable the immediate migration of existing campaign data into the new system while setting up the foundation for enhanced functionality through GKWP and SEMRush data integration.

328
docs/project-docs/02-database/conventions.md Normal file
View file

@ -0,0 +1,328 @@
# Database and Project Conventions
## Database Naming Conventions
### General Rules
- All database objects use snake_case
- All keywords stored in lowercase
- All names should be descriptive and clear
- Abbreviations avoided unless universally understood
### Table Names
- Use plural form for table names (e.g., `Keywords`, `Campaigns`)
- Compound names use underscores (e.g., `Landing_Pages`)
- Junction tables combine both entity names (e.g., `Keyword_Ad_Group_Assignments`)
- Historical/log tables append appropriate suffix (e.g., `Import_History`)
### Column Names
- Use singular form
- Include units in name if applicable (e.g., `budget_daily`)
- Use standard suffixes:
- `_id` for primary keys
- `_at` for timestamps
- `_date` for dates
- `_type` for type indicators
- `_status` for status fields
### Index Names
- Format: `idx_[table]_[column(s)]`
- Examples:
- `idx_keyword_search`
- `idx_ad_group_campaign`
- `idx_keyword_assignments`
### View Names
- Prefix with `v_`
- Describe the data being presented
- Examples:
- `v_active_keywords_per_ad_group`
- `v_landing_page_usage`
### Function Names
- Use verb_noun format
- Describe the action being performed
- Examples:
- `update_keyword_metrics`
- `calculate_campaign_stats`
## Data Type Conventions
### Text Data
- `VARCHAR(255)` for standard text fields
- `VARCHAR(2048)` for URLs
- `TEXT` for unlimited length fields
- Always specify length for VARCHAR
### Numeric Data
- `INTEGER` for whole numbers
- `DECIMAL(10,2)` for currency
- `DECIMAL(5,2)` for percentages
- Always specify precision and scale for DECIMAL
### Date/Time Data
- `TIMESTAMP` without timezone for all timestamps
- `DATE` for date-only fields
- Default `CURRENT_TIMESTAMP` where appropriate
### Boolean Data
- Use `BOOLEAN` type (not INTEGER 0/1)
- Default values specified where appropriate
## Status Fields
### Standard Status Values
- 'active'
- 'inactive'
- 'pending'
- 'archived'
- 'deleted'
### Import Status Values
- 'SUCCESS'
- 'PARTIAL'
- 'FAILED'
## Script Naming Conventions
### Bash Scripts
- All scripts end in `.sh`
- Use kebab-case for names
- Format: `action-object[-modifier].sh`
- Examples:
- `create-database.sh`
- `import-keywords-gkwp.sh`
- `update-functions.sh`
### SQL Files
- End in `.sql`
- Match associated script name where applicable
- Examples:
- `create-keywords-table.sql`
- `create-import-views.sql`
## File Organization
### Script Categories
- Keep related scripts in appropriate subdirectories
- Create new categories as needed
- Document new categories in project documentation
### Configuration
- All sensitive data in `.database` file
- Configuration format:
```ini
DB_USER=postgres
DB_PASSWORD=your_password
DB_HOST=localhost
DB_PORT=5432
DB_NAME=keyword_management
```
## Code Formatting
### SQL Formatting
- Keywords in UPPER CASE
- Indentation: 2 spaces
- One statement per line
- Commas at the end of lines
- Proper line breaks for readability
Example:
```sql
SELECT
k.keyword,
k.gkwp_search_volume,
ag.ad_group_name
FROM
Keywords k
JOIN Keyword_Ad_Group_Assignments kaga
ON k.keyword_id = kaga.keyword_id
JOIN Ad_Groups ag
ON kaga.ad_group_id = ag.ad_group_id
WHERE
k.status = 'active'
AND ag.status = 'active';
```
### Bash Script Formatting
- Two space indentation
- Clear section comments
- Consistent error handling
- Standard variable naming
Example:
```bash
#!/bin/bash
# Load database configuration
source .database
# Execute SQL command
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-f "sql/create-table.sql"
```
## Documentation Conventions
### Script Headers
- Purpose of script
- Usage example
- Dependencies
- Expected inputs/outputs
- Author and date
Example:
```bash
#!/bin/bash
# Purpose: Creates the Keywords table in the database
# Usage: ./create-keywords-table.sh
# Dependencies: .database configuration file
# Created: 2024-11-19
```
### Comment Conventions
- SQL: -- for single line comments
- Bash: # for single line comments
- Include purpose for complex queries
- Document assumptions
- Explain non-obvious choices
## Error Handling
### Database Errors
- Proper error codes
- Meaningful error messages
- Transaction management
- Rollback procedures
### Script Errors
- Exit codes for different scenarios
- Error logging
- User feedback
- Cleanup on failure
## Logging Conventions
### Log Format
- Timestamp
- Script name
- Action performed
- Status/Result
- Error details if applicable
### Log Locations
- Script logs in `/logs` directory
- Database operation logs
- Error logs
- Import logs
## Version Control
### Commit Messages
- Clear and descriptive
- Reference task/ticket numbers
- Explain significant changes
- Document breaking changes
### Branch Naming
- feature/description
- bugfix/description
- hotfix/description
These conventions ensure consistency across the project and make maintenance and collaboration easier.
## Testing Conventions
### Test Organization
- Test scripts stored in `/tests` directory
- Mirror `/bin` directory structure
- Test files named with `.test.sh` suffix
- Example:
```
/bin
└── create-functions/
└── create-keyword-function.sh
/tests
└── create-functions/
└── create-keyword-function.test.sh
```
### Test Script Structure
```bash
#!/bin/bash
# Load test utilities
source ./tests/utils/test-framework.sh
# Test specific function creation
test_keyword_function() {
# Setup - capture initial state
initial_state=$(psql -t -c "SELECT count(*) FROM pg_proc WHERE proname = 'keyword_function';")
# Execute the script being tested
../bin/create-functions/create-keyword-function.sh
# Verify function exists
exists_check=$(psql -t -c "SELECT count(*) FROM pg_proc WHERE proname = 'keyword_function';")
assert_not_equals "$initial_state" "$exists_check" "Function should be created"
# Verify function signature
signature_check=$(psql -t -c "SELECT pg_get_functiondef('keyword_function'::regproc);")
assert_contains "$signature_check" "expected_signature" "Function signature should match"
# Test function behavior
result=$(psql -t -c "SELECT keyword_function('test_input');")
assert_equals "$result" "expected_output" "Function should produce expected results"
}
# Run tests
run_test_suite
```
### Test Utilities
- Common test framework in `/tests/utils`
- Standard assertions
- Setup/teardown helpers
- Database state management
- Test reporting functions
### Running Tests
- Individual test: `./tests/path/script.test.sh`
- Category tests: `./tests/run-category.sh create-functions`
- All tests: `./tests/run-all.sh`
### Test Reporting
- Standard output format
- Success/failure counts
- Execution time
- Detailed failure information
- State difference on failures
### Database Test Environment
- Separate test database
- Reset to known state before tests
- Cleanup after test completion
- Isolation from production data
### Continuous Integration
- Tests run before deployment
- Required passing tests for merges
- Test reports archived
- Coverage tracking
This approach offers several advantages:
1. Separation of concerns
2. Easier maintenance
3. Clear test organization
4. Reusable test utilities
5. Proper test isolation
6. Easy integration with CI/CD
Rather than building tests into operation scripts, we maintain parallel test scripts that:
- Mirror the structure of operational scripts
- Have access to shared test utilities
- Can be run individually or as suites
- Maintain clear relationships with what they test

202
docs/project-docs/02-database/schema.md Normal file
View file

@ -0,0 +1,202 @@
# Database Schema Documentation
## Database Configuration
- **Name**: keyword_management
- **Encoding**: UTF8
- **ICU Locale**: en-US
- **Created with**:
```sql
CREATE DATABASE keyword_management
WITH OWNER = postgres
ENCODING = 'UTF8'
ICU_LOCALE = 'en-US'
TEMPLATE = template0
CONNECTION LIMIT = -1;
```
## Current Tables
### Keywords Table
Primary table for keyword storage and metrics.
```sql
CREATE TABLE Keywords (
keyword_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
keyword VARCHAR(255) NOT NULL,
-- GKWP Data
gkwp_search_volume INTEGER,
gkwp_competition_index INTEGER,
gkwp_cpc_min DECIMAL(10,2),
gkwp_cpc_max DECIMAL(10,2),
gkwp_last_updated TIMESTAMP,
-- SEMRush Data
semrush_search_volume INTEGER,
semrush_difficulty INTEGER,
semrush_cpc DECIMAL(10,2),
semrush_intent VARCHAR(50),
semrush_last_updated TIMESTAMP,
-- Metadata
first_imported TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Case-insensitive unique constraint
CREATE UNIQUE INDEX unique_lowercase_keyword
ON Keywords (LOWER(keyword));
```
## Planned Tables
### Campaigns
Stores advertising campaign information.
```sql
CREATE TABLE Campaigns (
campaign_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
campaign_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
start_date DATE,
end_date DATE,
budget_daily DECIMAL(10,2),
notes TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT unique_campaign_name UNIQUE(campaign_name)
);
```
### Landing Pages
Stores landing page information for ad groups.
```sql
CREATE TABLE Landing_Pages (
landing_page_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
url VARCHAR(2048) NOT NULL,
page_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
notes TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT unique_url UNIQUE(url),
CONSTRAINT unique_page_name UNIQUE(page_name)
);
```
### Ad Groups
Links campaigns, landing pages, and keywords together.
```sql
CREATE TABLE Ad_Groups (
ad_group_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
campaign_id INTEGER NOT NULL,
landing_page_id INTEGER NOT NULL,
ad_group_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
notes TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT fk_campaign
FOREIGN KEY (campaign_id)
REFERENCES Campaigns(campaign_id),
CONSTRAINT fk_landing_page
FOREIGN KEY (landing_page_id)
REFERENCES Landing_Pages(landing_page_id),
CONSTRAINT unique_ad_group_name_per_campaign
UNIQUE(campaign_id, ad_group_name)
);
```
### Keyword Ad Group Assignments
Maps keywords to ad groups with additional metadata.
```sql
CREATE TABLE Keyword_Ad_Group_Assignments (
keyword_id INTEGER NOT NULL,
ad_group_id INTEGER NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
assigned_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
notes TEXT,
PRIMARY KEY (keyword_id, ad_group_id),
FOREIGN KEY (keyword_id) REFERENCES Keywords(keyword_id),
FOREIGN KEY (ad_group_id) REFERENCES Ad_Groups(ad_group_id)
);
```
### Import History
Tracks data imports for auditing and management.
```sql
CREATE TABLE Import_History (
import_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
import_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
source_type VARCHAR(50) NOT NULL, -- 'GKWP', 'SEMRUSH', 'PLAIN_LIST', etc.
filename VARCHAR(255),
original_seed_keyword VARCHAR(255),
assigned_ad_group_id INTEGER,
row_count INTEGER,
success_count INTEGER,
error_count INTEGER,
import_status VARCHAR(50) NOT NULL, -- 'SUCCESS', 'PARTIAL', 'FAILED'
error_details TEXT,
notes TEXT,
FOREIGN KEY (assigned_ad_group_id) REFERENCES Ad_Groups(ad_group_id)
);
```
## Views for LibreOffice Base
### Active Keywords per Ad Group
```sql
CREATE VIEW v_active_keywords_per_ad_group AS
SELECT
ag.ad_group_name,
c.campaign_name,
k.keyword,
k.gkwp_search_volume,
k.gkwp_cpc_min,
k.gkwp_cpc_max,
kaga.assigned_at
FROM Ad_Groups ag
JOIN Campaigns c ON ag.campaign_id = c.campaign_id
JOIN Keyword_Ad_Group_Assignments kaga ON ag.ad_group_id = kaga.ad_group_id
JOIN Keywords k ON kaga.keyword_id = k.keyword_id
WHERE ag.status = 'active'
AND kaga.status = 'active';
```
### Landing Page Usage
```sql
CREATE VIEW v_landing_page_usage AS
SELECT
lp.page_name,
lp.url,
c.campaign_name,
ag.ad_group_name,
COUNT(kaga.keyword_id) as keyword_count
FROM Landing_Pages lp
JOIN Ad_Groups ag ON lp.landing_page_id = ag.landing_page_id
JOIN Campaigns c ON ag.campaign_id = c.campaign_id
LEFT JOIN Keyword_Ad_Group_Assignments kaga ON ag.ad_group_id = kaga.ad_group_id
WHERE lp.status = 'active'
GROUP BY lp.page_name, lp.url, c.campaign_name, ag.ad_group_name;
```
## Indexes
```sql
-- Keyword search optimization
CREATE INDEX idx_keyword_search ON Keywords USING gin (keyword gin_trgm_ops);
-- Ad Group lookups
CREATE INDEX idx_ad_group_campaign ON Ad_Groups(campaign_id);
CREATE INDEX idx_ad_group_landing_page ON Ad_Groups(landing_page_id);
-- Keyword assignment lookups
CREATE INDEX idx_keyword_assignments ON Keyword_Ad_Group_Assignments(keyword_id);
CREATE INDEX idx_ad_group_assignments ON Keyword_Ad_Group_Assignments(ad_group_id);
```
## Notes
- All timestamp fields use timezone-free TIMESTAMP type
- Status fields use VARCHAR with controlled vocabularies
- Soft deletes implemented through status fields
- Case-insensitive keyword storage enforced through index
- Proper foreign key constraints maintain referential integrity
- Views optimized for LibreOffice Base operations

266
docs/project-docs/03-scripts/db-admin/backup-restore.md Normal file
View file

@ -0,0 +1,266 @@
# Database Backup and Restore Operations
## Overview
This documentation covers the backup and restore scripts for the keyword_management database. These scripts provide functionality for creating database backups and restoring from them, essential for disaster recovery and development operations.
## Script Locations
```
/bin/db-admin/backup-db.sh
/bin/db-admin/restore-db.sh
```
## Backup Script
### Purpose
- Create compressed backups of the database
- Maintain dated backup files
- Support both full and schema-only backups
- Include or exclude specific tables
- Log all backup operations
### Configuration
Uses standard `.database` configuration plus backup-specific settings:
```ini
BACKUP_DIR="../../backups"
BACKUP_RETENTION_DAYS=30 # Number of days to keep backups
```
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../.database
# Set error handling
set -e
# Setup directories and logging
BACKUP_DIR="../../backups"
LOG_DIR="../../logs"
DATE=$(date +%Y-%m-%d-%H%M%S)
LOG_FILE="${LOG_DIR}/backup-${DATE}.log"
BACKUP_FILE="${BACKUP_DIR}/${DB_NAME}-${DATE}.sql.gz"
mkdir -p "$BACKUP_DIR" "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Parse arguments
SCHEMA_ONLY=0
while getopts "s" opt; do
case $opt in
s)
SCHEMA_ONLY=1
;;
\?)
echo "Invalid option: -$OPTARG"
exit 1
;;
esac
done
# Perform backup
log_message "Starting backup of ${DB_NAME}"
if [ $SCHEMA_ONLY -eq 1 ]; then
log_message "Creating schema-only backup"
pg_dump -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
--schema-only \
| gzip > "$BACKUP_FILE"
else
log_message "Creating full backup"
pg_dump -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
| gzip > "$BACKUP_FILE"
fi
# Verify backup
if [ $? -eq 0 ] && [ -f "$BACKUP_FILE" ]; then
log_message "Backup completed successfully: ${BACKUP_FILE}"
log_message "Backup size: $(du -h "$BACKUP_FILE" | cut -f1)"
else
log_message "Error: Backup failed"
exit 1
fi
# Cleanup old backups
find "$BACKUP_DIR" -name "${DB_NAME}-*.sql.gz" -mtime +${BACKUP_RETENTION_DAYS} -delete
```
### Usage
```bash
# Full backup
./bin/db-admin/backup-db.sh
# Schema-only backup
./bin/db-admin/backup-db.sh -s
```
## Restore Script
### Purpose
- Restore database from backup file
- Support for checking backup integrity
- Option to restore to different database name
- Proper handling of existing connections
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../logs"
DATE=$(date +%Y-%m-%d-%H%M%S)
LOG_FILE="${LOG_DIR}/restore-${DATE}.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Check for backup file argument
if [ $# -lt 1 ]; then
echo "Usage: $0 <backup_file> [target_database]"
exit 1
fi
BACKUP_FILE=$1
TARGET_DB=${2:-$DB_NAME}
# Check if backup file exists
if [ ! -f "$BACKUP_FILE" ]; then
log_message "Error: Backup file not found: ${BACKUP_FILE}"
exit 1
fi
# Terminate existing connections
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d postgres \
-c "SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = '${TARGET_DB}'
AND pid <> pg_backend_pid();"
# Drop and recreate database
log_message "Dropping database ${TARGET_DB} if exists"
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d postgres \
-c "DROP DATABASE IF EXISTS ${TARGET_DB};"
log_message "Creating fresh database ${TARGET_DB}"
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d postgres \
-c "CREATE DATABASE ${TARGET_DB}
WITH OWNER = ${DB_USER}
ENCODING = 'UTF8'
ICU_LOCALE = 'en-US'
TEMPLATE = template0;"
# Restore from backup
log_message "Restoring from backup: ${BACKUP_FILE}"
gunzip -c "$BACKUP_FILE" | \
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${TARGET_DB}"
# Verify restore
if [ $? -eq 0 ]; then
log_message "Database restored successfully"
# Log basic database statistics
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${TARGET_DB}" \
-c "\dl+" >> "$LOG_FILE"
else
log_message "Error: Database restore failed"
exit 1
fi
```
### Usage
```bash
# Restore to same database
./bin/db-admin/restore-db.sh backups/keyword_management-2024-11-19-100000.sql.gz
# Restore to different database
./bin/db-admin/restore-db.sh backups/keyword_management-2024-11-19-100000.sql.gz keyword_management_test
```
## Testing
Located in `/tests/db-admin/backup-restore.test.sh`
Test cases:
1. Full backup creation
2. Schema-only backup
3. Backup file integrity
4. Restore to same database
5. Restore to different database
6. Error handling
```bash
#!/bin/bash
source ../utils/test-framework.sh
test_backup_restore() {
# Create test data
psql -d keyword_management -c "CREATE TABLE test_backup (id serial, name text);"
psql -d keyword_management -c "INSERT INTO test_backup (name) VALUES ('test');"
# Test full backup
../bin/db-admin/backup-db.sh
assert_success "Backup should complete successfully"
# Verify backup file exists
latest_backup=$(ls -t ../../backups/keyword_management-*.sql.gz | head -1)
assert_file_exists "$latest_backup"
# Test restore to test database
../bin/db-admin/restore-db.sh "$latest_backup" "keyword_management_test"
assert_success "Restore should complete successfully"
# Verify restored data
result=$(psql -tAc "SELECT count(*) FROM test_backup;" keyword_management_test)
assert_equals "$result" "1" "Restored data should match original"
# Cleanup
psql -c "DROP DATABASE keyword_management_test;"
rm "$latest_backup"
}
run_test_suite
```
## Notes
- Backups are compressed using gzip
- Backup files include timestamp in name
- Automatic cleanup of old backup files
- Restore process recreates database with correct encoding
- All operations are logged with timestamps
- Supports development operations with different target databases

175
docs/project-docs/03-scripts/db-admin/create-db.md Normal file
View file

@ -0,0 +1,175 @@
# Database Creation Script Documentation
## Overview
The database creation script establishes the keyword_management database with proper ICU locale support and UTF-8 encoding. This is the foundational script that must be run before any other database operations.
## Script Location
```
/bin/db-admin/create-db.sh
```
## Purpose
- Create the keyword_management database
- Set proper encoding and locale settings
- Establish initial database configuration
- Verify successful creation
## Prerequisites
- PostgreSQL 17 installed and running
- User has appropriate permissions
- `.database` configuration file exists in project root
## Configuration
The script relies on the `.database` configuration file:
```ini
DB_USER=postgres
DB_PASSWORD=your_password
DB_HOST=localhost
DB_PORT=5432
DB_NAME=keyword_management
```
## SQL Command
```sql
CREATE DATABASE keyword_management
WITH
OWNER = postgres
ENCODING = 'UTF8'
ICU_LOCALE = 'en-US'
TEMPLATE = template0
CONNECTION LIMIT = -1;
```
## Script Implementation
```bash
#!/bin/bash
# Load configuration
source ../../.database
# Set error handling
set -e
# Log file setup
LOG_DIR="../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-create-db.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Check if database already exists
DB_EXISTS=$(psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -tAc "SELECT 1 FROM pg_database WHERE datname='${DB_NAME}'")
if [ "$DB_EXISTS" = "1" ]; then
log_message "Error: Database ${DB_NAME} already exists"
exit 1
fi
# Create database
log_message "Creating database ${DB_NAME}..."
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d postgres \
-c "CREATE DATABASE ${DB_NAME}
WITH
OWNER = ${DB_USER}
ENCODING = 'UTF8'
ICU_LOCALE = 'en-US'
TEMPLATE = template0
CONNECTION LIMIT = -1;"
# Verify creation
if [ $? -eq 0 ]; then
log_message "Database ${DB_NAME} created successfully"
# Verify settings
log_message "Verifying database settings..."
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-c "\l+ ${DB_NAME}" >> "$LOG_FILE"
else
log_message "Error: Failed to create database ${DB_NAME}"
exit 1
fi
```
## Usage
```bash
./bin/db-admin/create-db.sh
```
## Expected Output
```
2024-11-19 10:00:00 - Creating database keyword_management...
2024-11-19 10:00:01 - Database keyword_management created successfully
2024-11-19 10:00:01 - Verifying database settings...
```
## Error Handling
1. Checks if database already exists
2. Verifies user permissions
3. Validates creation success
4. Logs all operations and errors
5. Returns appropriate exit codes
## Testing
Located in `/tests/db-admin/create-db.test.sh`
Test cases:
1. Database creation
2. Proper encoding verification
3. Locale setting verification
4. Owner assignment verification
5. Duplicate creation attempt
6. Permission verification
```bash
#!/bin/bash
source ../utils/test-framework.sh
test_database_creation() {
# Setup
./bin/db-admin/drop-db.sh >/dev/null 2>&1 || true
# Test creation
../bin/db-admin/create-db.sh
# Verify database exists
exists_check=$(psql -tAc "SELECT 1 FROM pg_database WHERE datname='keyword_management'")
assert_equals "$exists_check" "1" "Database should exist"
# Verify encoding
encoding_check=$(psql -tAc "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname='keyword_management'")
assert_equals "$encoding_check" "UTF8" "Database should use UTF8 encoding"
# Verify locale
locale_check=$(psql -tAc "SELECT datcollate FROM pg_database WHERE datname='keyword_management'")
assert_contains "$locale_check" "en-US" "Database should use en-US locale"
# Cleanup
./bin/db-admin/drop-db.sh >/dev/null 2>&1
}
run_test_suite
```
## Related Scripts
- `/bin/db-admin/drop-db.sh` - Removes the database
- `/bin/db-admin/backup-db.sh` - Creates database backup
- `/bin/db-admin/restore-db.sh` - Restores database from backup
## Notes
- Script must be run as database superuser or user with CREATE DATABASE privilege
- The template0 database is used to ensure clean encoding setting
- Connection limit of -1 allows unlimited connections
- ICU locale provider ensures proper case-insensitive operations
- Log files are created in the logs directory with date prefix

345
docs/project-docs/03-scripts/import-scripts/competitor-import.md Normal file
View file

@ -0,0 +1,345 @@
# Competitor Import Scripts Documentation
## Overview
These scripts handle the import of competitor keyword ranking data from SEMRush competitor exports. The system tracks competitor keyword rankings over time and maintains historical position data.
## File Format Requirements
- Comma-delimited CSV files
- Filename pattern: "SEMRush Comp - {WEBSITE}.csv"
- Key fields:
- Keyword
- Position
- Volume
- Traffic %
- Costs %
- Competition
- SERP Features
- Results
- Trends
## Script Locations
```
/bin/import-scripts/competitor/
├── import-competitor-file.sh # Single file import
├── batch-import-competitor.sh # Multiple file import
├── manage-competitors.sh # Add/update competitor info
└── sql/
├── add_competitor.sql # Add new competitor
├── update_competitor.sql # Update competitor info
├── process_rankings.sql # Process ranking data
└── archive_rankings.sql # Archive historical data
```
## Import Process Flow
```mermaid
graph TD
A[Read CSV File] --> B[Extract Domain]
B --> C[Create/Update Competitor]
C --> D[Process Rankings]
D --> E[Archive Old Rankings]
E --> F[Update Current Rankings]
F --> G[Log Results]
```
## Competitor Management Script
### Location
`/bin/import-scripts/competitor/manage-competitors.sh`
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-competitor-management.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Function to add competitor
add_competitor() {
local domain=$1
local notes=$2
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-v domain="$domain" \
-v notes="$notes" \
-f "sql/add_competitor.sql"
}
# Function to update competitor
update_competitor() {
local domain=$1
local notes=$2
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-v domain="$domain" \
-v notes="$notes" \
-f "sql/update_competitor.sql"
}
# Parse command line arguments
case "$1" in
"add")
add_competitor "$2" "$3"
;;
"update")
update_competitor "$2" "$3"
;;
*)
echo "Usage: $0 {add|update} <domain> [notes]"
exit 1
;;
esac
```
## Competitor Import Script
### Location
`/bin/import-scripts/competitor/import-competitor-file.sh`
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-competitor-import.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Check arguments
if [ $# -lt 1 ]; then
echo "Usage: $0 <competitor_export_file>"
exit 1
fi
INPUT_FILE=$1
# Extract domain from filename
DOMAIN=$(basename "$INPUT_FILE" | sed -n 's/SEMRush Comp - \(.*\)\.csv/\1/p')
if [ -z "$DOMAIN" ]; then
log_message "Error: Invalid filename format. Expected: SEMRush Comp - {WEBSITE}.csv"
exit 1
fi
# Create import history record
IMPORT_ID=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "INSERT INTO Import_History
(source_type, filename, original_seed_keyword, import_status)
VALUES ('SEMRUSH_COMP', '$(basename "$INPUT_FILE")',
'${DOMAIN}', 'IN_PROGRESS')
RETURNING import_id;")
# Process file
log_message "Processing competitor export file: $(basename "$INPUT_FILE")"
log_message "Domain: ${DOMAIN}"
# Archive existing rankings
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-v domain="$DOMAIN" \
-f "sql/archive_rankings.sql"
# Import new rankings
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-v import_id="$IMPORT_ID" \
-v domain="$DOMAIN" \
-f "sql/process_rankings.sql"
# Update import history with results
RESULTS=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "SELECT success_count, error_count
FROM Import_History
WHERE import_id = ${IMPORT_ID};")
log_message "Import completed for ${INPUT_FILE}"
log_message "Results: ${RESULTS}"
```
### SQL Processing Scripts
```sql
-- sql/process_rankings.sql
-- Create temporary table for import
CREATE TEMP TABLE competitor_rankings (
keyword TEXT,
position INTEGER,
volume INTEGER,
traffic_percent DECIMAL(5,2),
costs_percent DECIMAL(5,2),
competition INTEGER,
serp_features TEXT,
results INTEGER,
trends TEXT
);
-- Import CSV data
COPY competitor_rankings FROM STDIN WITH (FORMAT csv, DELIMITER ',', HEADER true);
-- Process rankings and update competitor keywords
WITH new_keywords AS (
INSERT INTO Keywords (keyword)
SELECT DISTINCT LOWER(keyword)
FROM competitor_rankings
ON CONFLICT (LOWER(keyword)) DO NOTHING
RETURNING keyword_id, keyword
),
all_keywords AS (
SELECT keyword_id, keyword
FROM Keywords
WHERE keyword IN (SELECT LOWER(keyword) FROM competitor_rankings)
),
competitor_id AS (
SELECT competitor_id
FROM Competitors
WHERE domain = :'domain'
)
INSERT INTO CompetitorKeywords (
competitor_id,
keyword_id,
position,
import_id,
first_seen,
last_seen
)
SELECT
(SELECT competitor_id FROM competitor_id),
k.keyword_id,
cr.position,
:import_id,
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
FROM competitor_rankings cr
JOIN all_keywords k ON LOWER(cr.keyword) = k.keyword;
-- Update import history
UPDATE Import_History
SET
import_status = 'SUCCESS',
success_count = (SELECT COUNT(*) FROM competitor_rankings),
error_count = 0
WHERE import_id = :import_id;
-- Cleanup
DROP TABLE competitor_rankings;
```
## Testing
Located in `/tests/import-scripts/competitor/`
Example test:
```bash
#!/bin/bash
source ../../../utils/test-framework.sh
test_competitor_import() {
# Create test competitor
../bin/import-scripts/competitor/manage-competitors.sh add "example.com" "Test competitor"
# Create test file
cat > "test-data/SEMRush Comp - example.com.csv" << EOL
Keyword,Position,Volume,Traffic %,Costs %,Competition,SERP Features,Results,Trends
test keyword,1,1000,2.5,3.2,45,featured snippet,1000000,up
another test,5,2000,1.8,2.1,65,local pack,2000000,stable
EOL
# Run import
../bin/import-scripts/competitor/import-competitor-file.sh "test-data/SEMRush Comp - example.com.csv"
# Verify competitor
result=$(psql -tAc "SELECT COUNT(*) FROM Competitors WHERE domain = 'example.com';" "${DB_NAME}")
assert_equals "$result" "1" "Competitor should exist"
# Verify rankings
rankings=$(psql -tAc "SELECT COUNT(*) FROM CompetitorKeywords ck
JOIN Competitors c ON ck.competitor_id = c.competitor_id
WHERE c.domain = 'example.com';" "${DB_NAME}")
assert_equals "$rankings" "2" "Should import both keyword rankings"
# Clean up
rm "test-data/SEMRush Comp - example.com.csv"
}
run_test_suite
```
## Error Handling
1. File Format
- Header validation
- Data type checking
- Required field verification
2. Competitor Management
- Domain verification
- Duplicate checking
- Status tracking
3. Ranking Data
- Position validation
- Historical data preservation
- Metric range checking
## Best Practices
1. Data Integrity
- Transaction management
- Historical data archiving
- Consistent keyword matching
2. Performance
- Batch processing
- Index utilization
- Efficient updates
3. Monitoring
- Ranking changes tracking
- Import statistics
- Error reporting
4. Security
- Domain validation
- Access control
- Data protection

283
docs/project-docs/03-scripts/import-scripts/gkwp-import.md Normal file
View file

@ -0,0 +1,283 @@
# Google Keyword Planner Import Scripts Documentation
## Overview
These scripts handle the import of keyword data from Google Keyword Planner (GKWP) exports. The scripts process tab-delimited CSV files, extract seed keywords from filenames, and update the keyword database with search volume, competition, and CPC data.
## File Format Requirements
- Tab-delimited CSV files
- Filename pattern: "GKWP - {KEYWORD}.csv"
- Required columns:
- Keyword
- Avg. monthly searches
- Competition (indexed value)
- Top of page bid (low range)
- Top of page bid (high range)
## Script Locations
```
/bin/import-scripts/gkwp/
├── import-gkwp-file.sh # Single file import
├── batch-import-gkwp.sh # Multiple file import
└── sql/
├── check_existing.sql # Check for existing keywords
├── insert_keyword.sql # Insert new keywords
└── update_metrics.sql # Update existing keywords
```
## Import Process Flow
```mermaid
graph TD
A[Read CSV File] --> B[Extract Seed Keyword]
B --> C[Validate File Format]
C --> D[Process Header Row]
D --> E[Import Each Row]
E --> F[Log Results]
F --> G[Update Import History]
```
## Single File Import Script
### Location
`/bin/import-scripts/gkwp/import-gkwp-file.sh`
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-gkwp-import.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Check arguments
if [ $# -lt 1 ]; then
echo "Usage: $0 <gkwp_export_file>"
exit 1
fi
INPUT_FILE=$1
# Extract seed keyword from filename
SEED_KEYWORD=$(basename "$INPUT_FILE" | sed -n 's/GKWP - \(.*\)\.csv/\1/p')
if [ -z "$SEED_KEYWORD" ]; then
log_message "Error: Invalid filename format. Expected: GKWP - {KEYWORD}.csv"
exit 1
fi
# Create import history record
IMPORT_ID=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "INSERT INTO Import_History
(source_type, filename, original_seed_keyword, import_status)
VALUES ('GKWP', '$(basename "$INPUT_FILE")',
'${SEED_KEYWORD}', 'IN_PROGRESS')
RETURNING import_id;")
# Process file
log_message "Processing GKWP export file: $(basename "$INPUT_FILE")"
log_message "Seed keyword: ${SEED_KEYWORD}"
# Import data using psql COPY command with custom processing
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-v import_id="$IMPORT_ID" \
-f "sql/process_gkwp_import.sql"
# Update import history with results
RESULTS=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "SELECT success_count, error_count
FROM Import_History
WHERE import_id = ${IMPORT_ID};")
log_message "Import completed for ${INPUT_FILE}"
log_message "Results: ${RESULTS}"
```
### SQL Processing Script
```sql
-- sql/process_gkwp_import.sql
-- Create temporary table for import
CREATE TEMP TABLE gkwp_import (
keyword TEXT,
monthly_searches INTEGER,
competition INTEGER,
top_page_bid_low DECIMAL(10,2),
top_page_bid_high DECIMAL(10,2)
);
-- Import CSV data
COPY gkwp_import FROM STDIN WITH (FORMAT csv, DELIMITER E'\t', HEADER true);
-- Process imported data
WITH import_results AS (
INSERT INTO Keywords (
keyword,
gkwp_search_volume,
gkwp_competition_index,
gkwp_cpc_min,
gkwp_cpc_max,
gkwp_last_updated
)
SELECT
LOWER(keyword),
monthly_searches,
competition,
top_page_bid_low,
top_page_bid_high,
CURRENT_TIMESTAMP
FROM gkwp_import
ON CONFLICT (LOWER(keyword)) DO UPDATE
SET
gkwp_search_volume = EXCLUDED.gkwp_search_volume,
gkwp_competition_index = EXCLUDED.gkwp_competition_index,
gkwp_cpc_min = EXCLUDED.gkwp_cpc_min,
gkwp_cpc_max = EXCLUDED.gkwp_cpc_max,
gkwp_last_updated = CURRENT_TIMESTAMP
RETURNING 1
)
SELECT COUNT(*) as processed
FROM import_results;
-- Update import history with results
UPDATE Import_History
SET
import_status = 'SUCCESS',
success_count = (SELECT COUNT(*) FROM gkwp_import),
error_count = 0
WHERE import_id = :import_id;
-- Cleanup
DROP TABLE gkwp_import;
```
## Batch Import Script
### Location
`/bin/import-scripts/gkwp/batch-import-gkwp.sh`
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-gkwp-batch-import.log"
mkdir -p "$LOG_DIR"
# Check arguments
if [ $# -lt 1 ]; then
echo "Usage: $0 <directory_with_gkwp_files>"
exit 1
fi
INPUT_DIR=$1
# Process all GKWP files in directory
for file in "$INPUT_DIR"/GKWP*.csv; do
if [ -f "$file" ]; then
./import-gkwp-file.sh "$file"
fi
done
```
## Testing
Located in `/tests/import-scripts/gkwp/`
Example test:
```bash
#!/bin/bash
source ../../utils/test-framework.sh
test_gkwp_import() {
# Create test file
cat > "test-data/GKWP - test.csv" << EOL
Keyword Avg. monthly searches Competition Top of page bid (low range) Top of page bid (high range)
test keyword 1000 50 0.50 1.50
another test 2000 75 0.75 2.25
EOL
# Run import
../bin/import-scripts/gkwp/import-gkwp-file.sh "test-data/GKWP - test.csv"
# Verify data
result=$(psql -tAc "SELECT COUNT(*) FROM Keywords WHERE keyword IN ('test keyword', 'another test');" "${DB_NAME}")
assert_equals "$result" "2" "Should import both keywords"
# Verify metrics
volumes=$(psql -tAc "SELECT gkwp_search_volume FROM Keywords WHERE keyword = 'test keyword';" "${DB_NAME}")
assert_equals "$volumes" "1000" "Should import correct search volume"
# Clean up
rm "test-data/GKWP - test.csv"
}
run_test_suite
```
## Error Handling
1. File Validation
- Check filename format
- Verify required columns
- Validate data types
2. Import Errors
- Record failed rows
- Continue processing on row errors
- Maintain transaction integrity
3. Logging
- Detailed error messages
- Import statistics
- Warning conditions
## Best Practices
1. Data Cleaning
- Convert keywords to lowercase
- Remove leading/trailing whitespace
- Handle special characters
2. Performance
- Batch processing
- Efficient upsert operations
- Index usage optimization
3. Monitoring
- Track success rates
- Monitor processing time
- Record data anomalies
4. Recovery
- Transaction management
- Rollback capabilities
- Error state recovery

300
docs/project-docs/03-scripts/import-scripts/plain-list-import.md Normal file
View file

@ -0,0 +1,300 @@
# Plain Keyword List Import Documentation
## Overview
These scripts handle the import of plain keyword lists, typically from existing ad groups and campaigns. This process differs from GKWP and SEMRush imports as it:
- Doesn't include metric data
- Supports direct ad group assignment
- Can create new ad groups during import
- Doesn't use seed keywords
- Focuses on organization rather than data enrichment
## File Format Requirements
- One keyword per line text file
- Optional CSV format with ad group assignments
- Supported formats:
```
# Simple text file
keyword one
keyword two
keyword three
# CSV with ad group assignments
keyword,ad_group_name,campaign_name
keyword one,products,main campaign
keyword two,services,main campaign
```
## Script Locations
```
/bin/import-scripts/plain-list/
├── import-keywords.sh # Basic keyword list import
├── import-with-adgroups.sh # Import with ad group assignments
├── create-adgroup-import.sh # Import + create new ad group
└── sql/
├── import_keywords.sql # Basic import processing
├── assign_adgroups.sql # Ad group assignment
└── create_adgroup.sql # New ad group creation
```
## Import Process Flow
```mermaid
graph TD
A[Read Input File] --> B{Format Check}
B -->|Plain List| C[Basic Import]
B -->|CSV with AdGroups| D[Parse Assignments]
D --> E{AdGroup Exists?}
E -->|Yes| F[Assign Keywords]
E -->|No| G[Create AdGroup]
G --> F
C --> H[Update Import History]
F --> H
```
## Basic Import Script
### Location
`/bin/import-scripts/plain-list/import-keywords.sh`
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-plain-import.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Check arguments
if [ $# -lt 1 ]; then
echo "Usage: $0 <keyword_file> [ad_group_id]"
exit 1
fi
INPUT_FILE=$1
AD_GROUP_ID=$2
# Create import history record
IMPORT_ID=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "INSERT INTO Import_History
(source_type, filename, assigned_ad_group_id, import_status)
VALUES ('PLAIN_LIST', '$(basename "$INPUT_FILE")',
${AD_GROUP_ID:=NULL}, 'IN_PROGRESS')
RETURNING import_id;")
# Process file
log_message "Processing keyword list: $(basename "$INPUT_FILE")"
# Import keywords
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-v import_id="$IMPORT_ID" \
-v ad_group_id="$AD_GROUP_ID" \
-f "sql/import_keywords.sql"
# Get results
RESULTS=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "SELECT success_count, error_count
FROM Import_History
WHERE import_id = ${IMPORT_ID};")
log_message "Import completed: ${RESULTS}"
```
### SQL Import Script
```sql
-- sql/import_keywords.sql
-- Create temporary table for import
CREATE TEMP TABLE keyword_import (
keyword TEXT
);
-- Import data
COPY keyword_import FROM STDIN WITH (FORMAT text);
-- Process keywords
WITH inserted_keywords AS (
INSERT INTO Keywords (keyword)
SELECT DISTINCT LOWER(keyword)
FROM keyword_import
ON CONFLICT (LOWER(keyword)) DO NOTHING
RETURNING keyword_id
),
all_keywords AS (
SELECT k.keyword_id
FROM Keywords k
JOIN keyword_import ki ON LOWER(ki.keyword) = k.keyword
)
INSERT INTO Keyword_Ad_Group_Assignments (
keyword_id,
ad_group_id,
status
)
SELECT
keyword_id,
:ad_group_id,
'active'
FROM all_keywords
WHERE :ad_group_id IS NOT NULL;
-- Update import history
UPDATE Import_History
SET
import_status = 'SUCCESS',
success_count = (SELECT COUNT(*) FROM keyword_import),
error_count = 0
WHERE import_id = :import_id;
-- Cleanup
DROP TABLE keyword_import;
```
## Ad Group Import Script
### Location
`/bin/import-scripts/plain-list/import-with-adgroups.sh`
### Implementation
```bash
#!/bin/bash
# Enhanced version supporting ad group assignments
# Implementation details for ad group import...
```
## Create and Import Script
### Location
`/bin/import-scripts/plain-list/create-adgroup-import.sh`
### Usage
```bash
# Create new ad group and import keywords
./create-adgroup-import.sh keyword_file.txt "New Ad Group" campaign_id landing_page_id
# Import to existing ad group
./import-with-adgroups.sh keyword_file.csv
```
## Testing
Located in `/tests/import-scripts/plain-list/`
Example test:
```bash
#!/bin/bash
source ../../../utils/test-framework.sh
test_plain_import() {
# Create test campaign and ad group
campaign_id=$(psql -tAc "INSERT INTO Campaigns (campaign_name) VALUES ('Test Campaign') RETURNING campaign_id;")
landing_page_id=$(psql -tAc "INSERT INTO Landing_Pages (url, page_name) VALUES ('http://test.com', 'Test Page') RETURNING landing_page_id;")
ad_group_id=$(psql -tAc "INSERT INTO Ad_Groups (campaign_id, landing_page_id, ad_group_name) VALUES ($campaign_id, $landing_page_id, 'Test Group') RETURNING ad_group_id;")
# Create test file
cat > "test-data/keywords.txt" << EOL
test keyword one
test keyword two
test keyword three
EOL
# Run import
../bin/import-scripts/plain-list/import-keywords.sh "test-data/keywords.txt" "$ad_group_id"
# Verify imports
result=$(psql -tAc "SELECT COUNT(*) FROM Keywords WHERE keyword LIKE 'test keyword%';" "${DB_NAME}")
assert_equals "$result" "3" "Should import all keywords"
# Verify assignments
assignments=$(psql -tAc "SELECT COUNT(*) FROM Keyword_Ad_Group_Assignments WHERE ad_group_id = $ad_group_id;" "${DB_NAME}")
assert_equals "$assignments" "3" "Should assign all keywords to ad group"
# Clean up
rm "test-data/keywords.txt"
}
test_adgroup_creation() {
# Test creating new ad group during import
# Implementation of ad group creation test...
}
run_test_suite
```
## Error Handling
1. File Validation
- Line format checking
- Character encoding
- Duplicate detection
2. Ad Group Validation
- Existence checking
- Permission verification
- Campaign validation
3. Assignment Validation
- Duplicate assignments
- Status tracking
- Constraint checking
## Best Practices
1. Data Preparation
- Trim whitespace
- Convert to lowercase
- Remove duplicates
- Validate keywords
2. Performance
- Batch processing
- Transaction management
- Index utilization
3. Organization
- Clear naming conventions
- Consistent status values
- Proper documentation
4. Verification
- Import confirmation
- Assignment verification
- Status checking
## Integration with LibreOffice Base
### Views for Import Verification
```sql
CREATE VIEW v_recent_imports AS
SELECT
ih.import_date,
ih.filename,
ih.success_count,
ag.ad_group_name,
c.campaign_name
FROM Import_History ih
LEFT JOIN Ad_Groups ag ON ih.assigned_ad_group_id = ag.ad_group_id
LEFT JOIN Campaigns c ON ag.campaign_id = c.campaign_id
WHERE ih.source_type = 'PLAIN_LIST'
ORDER BY ih.import_date DESC;
```

311
docs/project-docs/03-scripts/import-scripts/semrush-import.md Normal file
View file

@ -0,0 +1,311 @@
# SEMRush Import Scripts Documentation
## Overview
These scripts handle the import of data from two types of SEMRush exports:
1. Standard keyword data exports (domain-focused keyword metrics)
2. Competitor keyword exports (competitor ranking data)
Each type has its own import process and data handling requirements.
## File Format Requirements
### Standard SEMRush Export
- Comma-delimited CSV files
- Filename pattern: "SEMRush - {KEYWORD}.csv"
- Key fields:
- Keyword
- Volume
- Keyword Difficulty
- CPC
- Intent
### Competitor Export
- Comma-delimited CSV files
- Filename pattern: "SEMRush Comp - {WEBSITE}.csv"
- Key fields:
- Keyword
- Position
- Volume
- Traffic %
- Costs %
- Competition
- Results
## Script Locations
```
/bin/import-scripts/semrush/
├── standard/
│ ├── import-semrush-file.sh # Single file import
│ ├── batch-import-semrush.sh # Multiple file import
│ └── sql/
│ ├── process_import.sql
│ └── update_metrics.sql
├── competitor/
│ ├── import-competitor-file.sh # Single competitor file import
│ ├── batch-import-competitor.sh # Multiple competitor file import
│ └── sql/
│ ├── process_competitor.sql
│ └── update_metrics.sql
└── common/
├── validate-headers.sh
└── error-handling.sh
```
## Standard Import Process Flow
```mermaid
graph TD
A[Read CSV File] --> B[Extract Seed Keyword]
B --> C[Validate File Format]
C --> D[Process Header Row]
D --> E[Import Keyword Data]
E --> F[Update Metrics]
F --> G[Log Results]
```
## Competitor Import Process Flow
```mermaid
graph TD
A[Read CSV File] --> B[Extract Domain]
B --> C[Validate Format]
C --> D[Process Headers]
D --> E[Create/Update Competitor]
E --> F[Import Rankings]
F --> G[Update History]
```
## Standard Import Script
### Location
`/bin/import-scripts/semrush/standard/import-semrush-file.sh`
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-semrush-import.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Check arguments
if [ $# -lt 1 ]; then
echo "Usage: $0 <semrush_export_file>"
exit 1
fi
INPUT_FILE=$1
# Extract seed keyword from filename
SEED_KEYWORD=$(basename "$INPUT_FILE" | sed -n 's/SEMRush - \(.*\)\.csv/\1/p')
if [ -z "$SEED_KEYWORD" ]; then
log_message "Error: Invalid filename format. Expected: SEMRush - {KEYWORD}.csv"
exit 1
fi
# Create import history record
IMPORT_ID=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "INSERT INTO Import_History
(source_type, filename, original_seed_keyword, import_status)
VALUES ('SEMRUSH', '$(basename "$INPUT_FILE")',
'${SEED_KEYWORD}', 'IN_PROGRESS')
RETURNING import_id;")
# Process file
log_message "Processing SEMRush export file: $(basename "$INPUT_FILE")"
log_message "Seed keyword: ${SEED_KEYWORD}"
# Import data using psql COPY command with custom processing
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-v import_id="$IMPORT_ID" \
-f "sql/process_import.sql"
# Update import history with results
RESULTS=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "SELECT success_count, error_count
FROM Import_History
WHERE import_id = ${IMPORT_ID};")
log_message "Import completed for ${INPUT_FILE}"
log_message "Results: ${RESULTS}"
```
### SQL Processing Script
```sql
-- sql/process_import.sql
-- Create temporary table for import
CREATE TEMP TABLE semrush_import (
keyword TEXT,
volume INTEGER,
keyword_difficulty INTEGER,
cpc DECIMAL(10,2),
intent VARCHAR(50)
);
-- Import CSV data
COPY semrush_import FROM STDIN WITH (FORMAT csv, DELIMITER ',', HEADER true);
-- Process imported data
WITH import_results AS (
INSERT INTO Keywords (
keyword,
semrush_search_volume,
semrush_difficulty,
semrush_cpc,
semrush_intent,
semrush_last_updated
)
SELECT
LOWER(keyword),
volume,
keyword_difficulty,
cpc,
intent,
CURRENT_TIMESTAMP
FROM semrush_import
ON CONFLICT (LOWER(keyword)) DO UPDATE
SET
semrush_search_volume = EXCLUDED.semrush_search_volume,
semrush_difficulty = EXCLUDED.semrush_difficulty,
semrush_cpc = EXCLUDED.semrush_cpc,
semrush_intent = EXCLUDED.semrush_intent,
semrush_last_updated = CURRENT_TIMESTAMP
RETURNING 1
)
SELECT COUNT(*) as processed
FROM import_results;
-- Update import history
UPDATE Import_History
SET
import_status = 'SUCCESS',
success_count = (SELECT COUNT(*) FROM semrush_import),
error_count = 0
WHERE import_id = :import_id;
-- Cleanup
DROP TABLE semrush_import;
```
## Competitor Import Script
### Location
`/bin/import-scripts/semrush/competitor/import-competitor-file.sh`
### Implementation
```bash
#!/bin/bash
# Similar structure to standard import, but with competitor-specific processing
# Implementation details for competitor import...
```
### SQL Processing Script
```sql
-- sql/process_competitor.sql
-- Create temporary table for competitor import
CREATE TEMP TABLE competitor_import (
keyword TEXT,
position INTEGER,
volume INTEGER,
traffic_percent DECIMAL(5,2),
costs_percent DECIMAL(5,2),
competition INTEGER,
results INTEGER
);
-- Import and process competitor data
-- Implementation details for competitor data processing...
```
## Testing
Located in `/tests/import-scripts/semrush/`
Example test:
```bash
#!/bin/bash
source ../../../utils/test-framework.sh
test_semrush_standard_import() {
# Create test file
cat > "test-data/SEMRush - test.csv" << EOL
Keyword,Volume,Keyword Difficulty,CPC,Intent
test keyword,1000,45,0.75,informational
another test,2000,65,1.25,commercial
EOL
# Run import
../bin/import-scripts/semrush/standard/import-semrush-file.sh "test-data/SEMRush - test.csv"
# Verify data
result=$(psql -tAc "SELECT COUNT(*) FROM Keywords WHERE keyword IN ('test keyword', 'another test');" "${DB_NAME}")
assert_equals "$result" "2" "Should import both keywords"
# Verify metrics
metrics=$(psql -tAc "SELECT semrush_search_volume, semrush_intent FROM Keywords WHERE keyword = 'test keyword';" "${DB_NAME}")
assert_contains "$metrics" "1000" "Should import correct volume"
assert_contains "$metrics" "informational" "Should import correct intent"
# Clean up
rm "test-data/SEMRush - test.csv"
}
run_test_suite
```
## Error Handling
1. File Format Validation
- Header verification
- Data type validation
- Required field checking
2. Data Processing
- Proper numeric handling
- NULL value management
- Character encoding
3. Error Recovery
- Transaction management
- Partial import handling
- Error reporting
## Best Practices
1. Data Quality
- Consistent lowercase keywords
- Proper metric scaling
- Intent standardization
2. Performance
- Batch processing
- Index utilization
- Memory management
3. Monitoring
- Import metrics tracking
- Error rate monitoring
- Performance statistics

252
docs/project-docs/03-scripts/table-admin/alter-tables.md Normal file
View file

@ -0,0 +1,252 @@
# Table Alteration Scripts Documentation
## Overview
These scripts handle modifications to existing database tables in the keyword management system. Proper version control and change tracking are essential for table alterations.
## Script Location Structure
```
/bin/table-admin/alter-tables/
├── versions/
│ ├── 20241119_01_add_column_keywords.sh
│ ├── 20241119_02_modify_column_type.sh
│ └── [YYYYMMDD]_[XX]_[description].sh
├── execute-version.sh
└── rollback-version.sh
```
## Version Naming Convention
- Format: `YYYYMMDD_XX_description.sh`
- YYYYMMDD: Date of creation
- XX: Sequential number for multiple changes on same day
- description: Brief description using kebab-case
- Example: `20241119_01_add_status_to_keywords.sh`
## Script Template Structure
### Version Script Template
```bash
#!/bin/bash
# Load configuration
source ../../../../.database
# Version metadata
VERSION_ID="20241119_01"
VERSION_DESC="Add status column to Keywords table"
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-${VERSION_ID}.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Upgrade function
upgrade() {
log_message "Executing upgrade for version ${VERSION_ID}"
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-c "ALTER TABLE Keywords ADD COLUMN status VARCHAR(50) NOT NULL DEFAULT 'active';"
# Record the change in version history
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-c "INSERT INTO version_history (version_id, description, applied_at)
VALUES ('${VERSION_ID}', '${VERSION_DESC}', CURRENT_TIMESTAMP);"
}
# Rollback function
rollback() {
log_message "Executing rollback for version ${VERSION_ID}"
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-c "ALTER TABLE Keywords DROP COLUMN status;"
# Remove version from history
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-c "DELETE FROM version_history WHERE version_id = '${VERSION_ID}';"
}
# Version check function
version_exists() {
local exists=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "SELECT 1 FROM version_history WHERE version_id = '${VERSION_ID}';")
[ "$exists" = "1" ]
}
# Execute based on command argument
case "$1" in
"upgrade")
if version_exists; then
log_message "Version ${VERSION_ID} already applied"
exit 0
fi
upgrade
;;
"rollback")
if ! version_exists; then
log_message "Version ${VERSION_ID} not applied"
exit 0
fi
rollback
;;
*)
echo "Usage: $0 {upgrade|rollback}"
exit 1
;;
esac
```
## Version Control Table
```sql
CREATE TABLE version_history (
version_id VARCHAR(50) PRIMARY KEY,
description TEXT NOT NULL,
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
rolled_back_at TIMESTAMP
);
```
## Execute Version Script
```bash
#!/bin/bash
# Load configuration
source ../../../.database
# Set error handling
set -e
# Check arguments
if [ $# -ne 2 ]; then
echo "Usage: $0 <version_id> <upgrade|rollback>"
exit 1
fi
VERSION_ID=$1
ACTION=$2
# Find version script
SCRIPT_PATH="versions/${VERSION_ID}_*.sh"
SCRIPT_FILE=$(ls $SCRIPT_PATH 2>/dev/null || true)
if [ -z "$SCRIPT_FILE" ]; then
echo "Error: Version script not found for ${VERSION_ID}"
exit 1
fi
# Execute version script
bash "$SCRIPT_FILE" "$ACTION"
```
## Example Alterations
### Add Column
```sql
-- Version: 20241119_01_add_status_to_keywords.sh
ALTER TABLE Keywords
ADD COLUMN status VARCHAR(50) NOT NULL DEFAULT 'active';
```
### Modify Column Type
```sql
-- Version: 20241119_02_modify_cpc_precision.sh
ALTER TABLE Keywords
ALTER COLUMN gkwp_cpc_min TYPE DECIMAL(12,4),
ALTER COLUMN gkwp_cpc_max TYPE DECIMAL(12,4);
```
### Add Index
```sql
-- Version: 20241119_03_add_keyword_search_index.sh
CREATE INDEX idx_keyword_search
ON Keywords USING gin (keyword gin_trgm_ops);
```
## Testing
Located in `/tests/table-admin/alter-tables/`
Example test:
```bash
#!/bin/bash
source ../../utils/test-framework.sh
test_version_execution() {
# Setup
VERSION_ID="20241119_01"
# Test upgrade
../bin/table-admin/alter-tables/execute-version.sh "$VERSION_ID" upgrade
# Verify column added
column_check=$(psql -tAc "\d keywords" "${DB_NAME}")
assert_contains "$column_check" "status" "Status column should exist"
# Verify version recorded
version_check=$(psql -tAc "SELECT 1 FROM version_history WHERE version_id = '${VERSION_ID}';" "${DB_NAME}")
assert_equals "$version_check" "1" "Version should be recorded"
# Test rollback
../bin/table-admin/alter-tables/execute-version.sh "$VERSION_ID" rollback
# Verify column removed
column_check=$(psql -tAc "\d keywords" "${DB_NAME}")
assert_not_contains "$column_check" "status" "Status column should not exist"
}
run_test_suite
```
## Best Practices
1. Version Control
- All changes tracked in version_history table
- Unique version identifiers
- Clear descriptions of changes
- Timestamp tracking
2. Reversibility
- All changes must have rollback functionality
- Rollbacks tested before deployment
- Data preservation considered
3. Safety Checks
- Version existence verification
- Dependency checking
- Backup verification
- Error handling
4. Documentation
- Clear change descriptions
- Impact assessment
- Dependencies noted
- Testing requirements
5. Testing
- Upgrade testing
- Rollback testing
- Integration testing
- Performance impact assessment

273
docs/project-docs/03-scripts/table-admin/create-tables.md Normal file
View file

@ -0,0 +1,273 @@
# Table Creation Scripts Documentation
## Overview
These scripts handle the creation of all database tables for the keyword management system. Each table has its own creation script to maintain modularity and allow for independent testing.
## Script Locations
```
/bin/table-admin/create-tables/
├── create-keywords-table.sh
├── create-campaigns-table.sh
├── create-landing-pages-table.sh
├── create-ad-groups-table.sh
├── create-keyword-assignments-table.sh
└── create-import-history-table.sh
```
## Keywords Table Script
### Location
`/bin/table-admin/create-tables/create-keywords-table.sh`
### SQL Command
```sql
CREATE TABLE Keywords (
keyword_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
keyword VARCHAR(255) NOT NULL,
-- GKWP Data
gkwp_search_volume INTEGER,
gkwp_competition_index INTEGER,
gkwp_cpc_min DECIMAL(10,2),
gkwp_cpc_max DECIMAL(10,2),
gkwp_last_updated TIMESTAMP,
-- SEMRush Data
semrush_search_volume INTEGER,
semrush_difficulty INTEGER,
semrush_cpc DECIMAL(10,2),
semrush_intent VARCHAR(50),
semrush_last_updated TIMESTAMP,
-- Metadata
first_imported TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Case-insensitive unique constraint
CREATE UNIQUE INDEX unique_lowercase_keyword
ON Keywords (LOWER(keyword));
```
### Implementation
```bash
#!/bin/bash
# Load configuration
source ../../../.database
# Set error handling
set -e
# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-create-keywords-table.log"
mkdir -p "$LOG_DIR"
# Function to log messages
log_message() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Check if table exists
TABLE_EXISTS=$(psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-tAc "SELECT to_regclass('public.keywords');")
if [ "$TABLE_EXISTS" == "keywords" ]; then
log_message "Error: Keywords table already exists"
exit 1
fi
# Create table
log_message "Creating Keywords table..."
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-f "sql/create-keywords-table.sql"
# Verify creation
if [ $? -eq 0 ]; then
log_message "Keywords table created successfully"
# Log table structure
log_message "Table structure:"
psql -h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USER}" \
-d "${DB_NAME}" \
-c "\d+ keywords" >> "$LOG_FILE"
else
log_message "Error: Failed to create Keywords table"
exit 1
fi
```
## Campaigns Table Script
### Location
`/bin/table-admin/create-tables/create-campaigns-table.sh`
### SQL Command
```sql
CREATE TABLE Campaigns (
campaign_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
campaign_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
start_date DATE,
end_date DATE,
budget_daily DECIMAL(10,2),
notes TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT unique_campaign_name UNIQUE(campaign_name)
);
```
## Landing Pages Table Script
### Location
`/bin/table-admin/create-tables/create-landing-pages-table.sh`
### SQL Command
```sql
CREATE TABLE Landing_Pages (
landing_page_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
url VARCHAR(2048) NOT NULL,
page_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
notes TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT unique_url UNIQUE(url),
CONSTRAINT unique_page_name UNIQUE(page_name)
);
```
## Ad Groups Table Script
### Location
`/bin/table-admin/create-tables/create-ad-groups-table.sh`
### SQL Command
```sql
CREATE TABLE Ad_Groups (
ad_group_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
campaign_id INTEGER NOT NULL,
landing_page_id INTEGER NOT NULL,
ad_group_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
notes TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT fk_campaign
FOREIGN KEY (campaign_id)
REFERENCES Campaigns(campaign_id),
CONSTRAINT fk_landing_page
FOREIGN KEY (landing_page_id)
REFERENCES Landing_Pages(landing_page_id),
CONSTRAINT unique_ad_group_name_per_campaign
UNIQUE(campaign_id, ad_group_name)
);
```
## Keyword Assignments Table Script
### Location
`/bin/table-admin/create-tables/create-keyword-assignments-table.sh`
### SQL Command
```sql
CREATE TABLE Keyword_Ad_Group_Assignments (
keyword_id INTEGER NOT NULL,
ad_group_id INTEGER NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
assigned_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
notes TEXT,
PRIMARY KEY (keyword_id, ad_group_id),
FOREIGN KEY (keyword_id) REFERENCES Keywords(keyword_id),
FOREIGN KEY (ad_group_id) REFERENCES Ad_Groups(ad_group_id)
);
```
## Import History Table Script
### Location
`/bin/table-admin/create-tables/create-import-history-table.sh`
### SQL Command
```sql
CREATE TABLE Import_History (
import_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
import_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
source_type VARCHAR(50) NOT NULL,
filename VARCHAR(255),
original_seed_keyword VARCHAR(255),
assigned_ad_group_id INTEGER,
row_count INTEGER,
success_count INTEGER,
error_count INTEGER,
import_status VARCHAR(50) NOT NULL,
error_details TEXT,
notes TEXT,
FOREIGN KEY (assigned_ad_group_id) REFERENCES Ad_Groups(ad_group_id)
);
```
## Testing
Located in `/tests/table-admin/create-tables/`
Each table has its own test file:
```
/tests/table-admin/create-tables/
├── create-keywords-table.test.sh
├── create-campaigns-table.test.sh
├── create-landing-pages-table.test.sh
├── create-ad-groups-table.test.sh
├── create-keyword-assignments-table.test.sh
└── create-import-history-table.test.sh
```
Example test for Keywords table:
```bash
#!/bin/bash
source ../../utils/test-framework.sh
test_keywords_table_creation() {
# Ensure table doesn't exist
psql -d "${DB_NAME}" -c "DROP TABLE IF EXISTS Keywords CASCADE;"
# Create table
../bin/table-admin/create-tables/create-keywords-table.sh
# Verify table exists
exists_check=$(psql -tAc "SELECT to_regclass('public.keywords');" "${DB_NAME}")
assert_equals "$exists_check" "keywords" "Keywords table should exist"
# Verify columns
columns=$(psql -tAc "\d keywords" "${DB_NAME}")
assert_contains "$columns" "keyword_id" "Should have keyword_id column"
assert_contains "$columns" "keyword" "Should have keyword column"
# Test unique constraint
psql -d "${DB_NAME}" -c "INSERT INTO Keywords (keyword) VALUES ('Test');"
error_check=$(psql -d "${DB_NAME}" -c "INSERT INTO Keywords (keyword) VALUES ('TEST');" 2>&1)
assert_contains "$error_check" "duplicate key value" "Should enforce case-insensitive uniqueness"
}
run_test_suite
```
## Notes
- Tables must be created in the correct order due to foreign key constraints
- Each script includes proper error handling and logging
- Case-insensitive keyword storage is enforced
- All timestamp fields use timezone-free TIMESTAMP type
- Status fields use predefined values
- Each table has appropriate indexes for performance

201
docs/project-docs/04-implementation/completed-tasks.md Normal file
View file

@ -0,0 +1,201 @@
# Completed Tasks
## Database Foundation
### Database Creation
- ✅ Created keyword_management database
- Implementation: PostgreSQL 17
- Encoding: UTF-8
- Locale: en-US (ICU)
- Command executed:
```sql
CREATE DATABASE keyword_management
WITH
OWNER = postgres
ENCODING = 'UTF8'
ICU_LOCALE = 'en-US'
TEMPLATE = template0
CONNECTION LIMIT = -1;
```
- Verification: Connection and encoding tests successful
- Date completed: 2024-11-19
### Keywords Table
- ✅ Created Keywords table with proper structure
- Implementation:
```sql
CREATE TABLE Keywords (
keyword_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
keyword VARCHAR(255) NOT NULL,
-- GKWP Data
gkwp_search_volume INTEGER,
gkwp_competition_index INTEGER,
gkwp_cpc_min DECIMAL(10,2),
gkwp_cpc_max DECIMAL(10,2),
gkwp_last_updated TIMESTAMP,
-- SEMRush Data
semrush_search_volume INTEGER,
semrush_difficulty INTEGER,
semrush_cpc DECIMAL(10,2),
semrush_intent VARCHAR(50),
semrush_last_updated TIMESTAMP,
-- Metadata
first_imported TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
```
- Added case-insensitive constraint:
```sql
CREATE UNIQUE INDEX unique_lowercase_keyword
ON Keywords (LOWER(keyword));
```
- Verification: Table created and constraints verified
- Date completed: 2024-11-19
## Project Structure
### Directory Layout
- ✅ Established project directory structure
```
/
├── .database # Configuration file
├── bin/ # Script directories
│ ├── db-admin/
│ ├── table-admin/
│ ├── create-functions/
│ └── import-scripts/
├── doc/ # Documentation
└── logs/ # Script execution logs
```
- Date completed: 2024-11-19
### Documentation Structure
- ✅ Created documentation hierarchy
```
/doc/project-docs/
├── 01-overview/
│ ├── project-goals.md
│ ├── architecture.md
│ └── directory-layout.md
├── 02-database/
│ ├── schema.md
│ └── conventions.md
├── 03-scripts/
├── 04-implementation/
└── 05-operations/
```
- Date completed: 2024-11-19
## Documentation
### Core Documentation
- ✅ Created project-goals.md
- Defined core objectives
- Outlined implementation phases
- Established success criteria
- Date completed: 2024-11-19
- ✅ Created architecture.md
- Defined system components
- Established design patterns
- Documented integration points
- Date completed: 2024-11-19
- ✅ Created directory-layout.md
- Documented directory structure
- Established naming conventions
- Defined organizational principles
- Date completed: 2024-11-19
### Technical Documentation
- ✅ Created schema.md
- Documented database schema
- Defined table relationships
- Established data types
- Date completed: 2024-11-19
- ✅ Created conventions.md
- Established coding standards
- Defined naming conventions
- Created testing framework guidelines
- Date completed: 2024-11-19
## Testing Framework
### Base Implementation
- ✅ Created test framework structure
```
/tests/
├── utils/
│ └── test-framework.sh
├── db-admin/
└── table-admin/
```
- Implementation: Bash-based testing framework
- Features:
- Standard assertions
- Test reporting
- Error capture
- Date completed: 2024-11-19
### Test Utilities
- ✅ Created basic test utilities
- Assert functions
- Setup/teardown helpers
- Database state management
- Date completed: 2024-11-19
## Configuration Management
### Database Configuration
- ✅ Created .database configuration template
```ini
DB_USER=postgres
DB_PASSWORD=your_password
DB_HOST=localhost
DB_PORT=5432
DB_NAME=keyword_management
```
- Implementation: Source-able configuration file
- Security: Proper file permissions
- Date completed: 2024-11-19
## LibreOffice Base Integration
### Database Connection
- ✅ Established LibreOffice Base connection
- Configuration: PostgreSQL direct connection
- Authentication: Password authentication
- Testing: Connection verified
- Date completed: 2024-11-19
## Verification and Testing
### Database Verification
- ✅ Tested database creation
- Encoding verification
- Locale testing
- Connection testing
- Date completed: 2024-11-19
### Table Verification
- ✅ Tested Keywords table
- Structure verification
- Constraint testing
- Index verification
- Date completed: 2024-11-19
## Notes
- All completed tasks have been tested and verified
- Documentation has been created and reviewed
- Test coverage has been established for completed components
- Each completion has been logged and dated
## Impact Analysis
- Database foundation is solid and ready for expansion
- Project structure supports future development
- Documentation provides clear guidance for next steps
- Testing framework enables reliable development

156
docs/project-docs/04-implementation/current-status.md Normal file
View file

@ -0,0 +1,156 @@
# Current Implementation Status
## Completed Items
### Database Foundation
- ✅ Created PostgreSQL 17 database with ICU locale support
- ✅ Established UTF-8 encoding
- ✅ Set up en-US locale configuration
- ✅ Created Keywords table with proper data types and constraints
- ✅ Implemented case-insensitive keyword storage
### Project Structure
- ✅ Established directory structure
- ✅ Created bash script framework
- ✅ Set up logging system
- ✅ Implemented configuration management
- ✅ Created test framework
### Documentation
- ✅ Project goals and overview
- ✅ System architecture
- ✅ Directory layout
- ✅ Database schema
- ✅ Coding conventions
- ✅ Import process documentation
## In Progress
### Script Development
- 🔄 Creating database administration scripts
- ✅ Database creation script
- ✅ Backup procedures
- ✅ Restore functionality
- ❌ Maintenance routines
- 🔄 Implementing table management scripts
- ✅ Keywords table creation
- ❌ Campaign table creation
- ❌ Ad Groups table creation
- ❌ Landing Pages table creation
- ❌ Import History table creation
- 🔄 Setting up import processing
- ❌ GKWP import scripts
- ❌ SEMRush import scripts
- ❌ Competitor import scripts
- ❌ Plain list import scripts
### Testing Implementation
- 🔄 Test framework setup
- ✅ Basic structure
- ✅ Helper functions
- ❌ Comprehensive test cases
- ❌ Automated test execution
## Immediate Next Steps
### 1. Complete Database Administration Scripts
- [ ] Finalize database creation script
- [ ] Implement database deletion script
- [ ] Create database verification script
- [ ] Add error handling and logging
### 2. Table Creation Scripts
- [ ] Create remaining core tables:
- [ ] Campaigns
- [ ] Ad Groups
- [ ] Landing Pages
- [ ] Import History
- [ ] Keyword Assignments
- [ ] Implement proper foreign key relationships
- [ ] Add necessary indexes
- [ ] Create verification procedures
### 3. GKWP Data Management
- [ ] Finalize GKWP import script
- [ ] Create data verification procedures
- [ ] Implement metric update functionality
- [ ] Add logging and error handling
### 4. LibreOffice Base Integration
- [ ] Create necessary views
- [ ] Set up common queries
- [ ] Establish data entry forms
- [ ] Create import verification views
## Upcoming Features
### Phase 1: Core Functionality
- [ ] Campaign management
- [ ] Ad group administration
- [ ] Landing page tracking
- [ ] Keyword assignment tools
### Phase 2: Import Processing
- [ ] GKWP data integration
- [ ] SEMRush data processing
- [ ] Competitor analysis
- [ ] Plain list importing
### Phase 3: Data Management
- [ ] Metric tracking
- [ ] Historical data management
- [ ] Performance analysis
- [ ] Report generation
## Known Issues
### Database
1. Need to verify ICU locale configuration
2. Need to implement proper index strategy
3. Need to establish backup procedures
### Scripts
1. Error handling needs enhancement
2. Logging system needs standardization
3. Configuration management needs security review
### Testing
1. Need comprehensive test coverage
2. Need automated test execution
3. Need test data generation
## Blockers and Dependencies
### Current Blockers
- None currently identified
### Dependencies
1. PostgreSQL 17 with ICU support
2. LibreOffice Base connection capability
3. Bash shell environment
4. Proper database permissions
## Next Implementation Sprint
### Priority Tasks
1. Complete database administration scripts
2. Implement core table creation scripts
3. Develop GKWP import functionality
4. Create essential LibreOffice Base views
### Success Criteria
- All scripts properly documented
- Test coverage established
- Error handling implemented
- Logging system functioning
- Basic functionality verified
## Notes
- Current focus is on establishing solid foundation
- Prioritizing data integrity and reliability
- Maintaining clear documentation
- Ensuring proper test coverage
Would you like me to expand on any particular aspect of the current status or provide more detail about upcoming tasks?

230
docs/project-docs/04-implementation/pending-tasks.md Normal file
View file

@ -0,0 +1,230 @@
# Pending Tasks
## Immediate Priority Tasks
### Database Administration Scripts
1. Script Creation
- [ ] Create drop-db.sh script
- Must safely terminate existing connections
- Include confirmation prompt
- Log operation details
- [ ] Complete backup-db.sh script
- Implement rotation of backups
- Add compression options
- Include selective backup capability
- [ ] Create restore-db.sh script
- Add verification steps
- Include target database option
- Support partial restores
2. Testing Implementation
- [ ] Create test suite for db-admin scripts
- [ ] Implement test data generation
- [ ] Create verification procedures
### Table Creation Scripts
1. Core Tables
- [ ] Campaigns table
```sql
CREATE TABLE Campaigns (
campaign_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
campaign_name VARCHAR(255) NOT NULL,
status VARCHAR(50) NOT NULL DEFAULT 'active',
-- additional fields...
);
```
- [ ] Landing_Pages table
```sql
CREATE TABLE Landing_Pages (
landing_page_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
url VARCHAR(2048) NOT NULL,
page_name VARCHAR(255) NOT NULL,
-- additional fields...
);
```
- [ ] Ad_Groups table
```sql
CREATE TABLE Ad_Groups (
ad_group_id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
campaign_id INTEGER NOT NULL,
landing_page_id INTEGER NOT NULL,
-- additional fields...
);
```
- [ ] Keyword_Ad_Group_Assignments table
```sql
CREATE TABLE Keyword_Ad_Group_Assignments (
keyword_id INTEGER NOT NULL,
ad_group_id INTEGER NOT NULL,
-- additional fields...
);
```
2. Supporting Tables
- [ ] Import_History table
- [ ] Version_History table
- [ ] Error_Log table
## Data Import Implementation
### GKWP Import System
1. Script Development
- [ ] Create import parser
- [ ] Implement data validation
- [ ] Add metric updating
- [ ] Create import logging
2. Supporting Functions
- [ ] Data cleaning utilities
- [ ] Validation functions
- [ ] Error handling routines
### SEMRush Import System
1. Core Development
- [ ] Standard keyword import
- [ ] Competitor data import
- [ ] Metric integration
2. Data Processing
- [ ] Format conversion
- [ ] Data validation
- [ ] Metric normalization
### Plain List Import System
1. Basic Functionality
- [ ] Simple keyword list import
- [ ] Ad group assignment support
- [ ] Bulk import capabilities
2. Enhanced Features
- [ ] New ad group creation during import
- [ ] Campaign assignment options
- [ ] Landing page integration
## LibreOffice Base Integration
### View Creation
1. Campaign Management
- [ ] Campaign overview
- [ ] Ad group listings
- [ ] Performance metrics
2. Keyword Management
- [ ] Keyword assignments
- [ ] Metric comparisons
- [ ] Historical data
### Form Development
1. Data Entry
- [ ] Campaign creation
- [ ] Ad group management
- [ ] Keyword assignment
2. Data Management
- [ ] Import verification
- [ ] Status updates
- [ ] Bulk operations
## Testing Implementation
### Unit Tests
1. Database Operations
- [ ] Table creation tests
- [ ] Data integrity tests
- [ ] Constraint verification
2. Import Processing
- [ ] GKWP import tests
- [ ] SEMRush import tests
- [ ] Plain list import tests
### Integration Tests
1. System Integration
- [ ] Full import workflow
- [ ] Campaign management
- [ ] Ad group assignments
2. Performance Testing
- [ ] Large dataset handling
- [ ] Concurrent operation testing
- [ ] Resource utilization
## Documentation Updates
### Technical Documentation
1. Script Documentation
- [ ] Usage guides
- [ ] Configuration options
- [ ] Error handling
2. Process Documentation
- [ ] Import workflows
- [ ] Management procedures
- [ ] Maintenance tasks
### User Documentation
1. Operation Guides
- [ ] Import procedures
- [ ] Campaign management
- [ ] Keyword organization
2. Troubleshooting Guides
- [ ] Common issues
- [ ] Resolution procedures
- [ ] Support escalation
## Future Enhancements
### Phase 1: Performance Optimization
1. Database Optimization
- [ ] Index optimization
- [ ] Query tuning
- [ ] Performance monitoring
2. Process Optimization
- [ ] Batch processing
- [ ] Concurrent operations
- [ ] Resource management
### Phase 2: Feature Enhancement
1. Advanced Features
- [ ] Automated imports
- [ ] Advanced reporting
- [ ] Trend analysis
2. Integration Features
- [ ] API development
- [ ] External tool integration
- [ ] Automation support
## Dependencies and Prerequisites
### System Requirements
- PostgreSQL 17 with ICU support
- LibreOffice Base compatibility
- Bash environment
- Proper permissions and access
### Development Tools
- Testing framework
- Documentation tools
- Version control system
## Timeline Considerations
- Immediate tasks: 2-3 weeks
- Core functionality: 1-2 months
- Complete system: 3-4 months
- Enhancements: Ongoing
## Success Criteria
- All tests passing
- Documentation complete
- Performance metrics met
- User acceptance verified
Would you like me to expand on any of these pending tasks or provide more detailed information about specific implementations?

280
docs/project-docs/05-operations/configuration.md Normal file
View file

@ -0,0 +1,280 @@
# System Configuration Documentation
## Overview
This document details the configuration settings and files required for the keyword management system. The primary configuration is managed through the `.database` file, with additional settings managed through script-specific configuration files.
## Database Configuration
### .database File
Located in project root directory: `/.database`
```ini
# Database Connection Settings
DB_USER=postgres
DB_PASSWORD=your_password
DB_HOST=localhost
DB_PORT=5432
DB_NAME=keyword_management
# Backup Configuration
BACKUP_DIR=backups
BACKUP_RETENTION_DAYS=30
# Logging Configuration
LOG_DIR=logs
LOG_LEVEL=INFO # DEBUG, INFO, WARN, ERROR
# Import Settings
IMPORT_BATCH_SIZE=1000
MAX_IMPORT_ERRORS=50
```
### Security Requirements
- File permissions: 600 (`-rw-------`)
- Owner: Script execution user
- Group: Script execution group
- Location: Project root only
- No version control tracking
### Environment Variables
Alternative to .database file for CI/CD environments:
```bash
export KW_DB_USER=postgres
export KW_DB_PASSWORD=your_password
export KW_DB_HOST=localhost
export KW_DB_PORT=5432
export KW_DB_NAME=keyword_management
```
## Configuration Management
### Loading Configuration
Scripts should load configuration in this order:
1. Environment variables (if present)
2. .database file
3. Default values
Example configuration loading:
```bash
#!/bin/bash
# Load configuration with fallback to defaults
load_config() {
# Try environment variables first
DB_USER=${KW_DB_USER:-}
DB_PASSWORD=${KW_DB_PASSWORD:-}
DB_HOST=${KW_DB_HOST:-}
DB_PORT=${KW_DB_PORT:-}
DB_NAME=${KW_DB_NAME:-}
# If any required env vars are missing, try .database file
if [[ -z "$DB_USER" || -z "$DB_PASSWORD" ]]; then
if [[ -f "/.database" ]]; then
source "/.database"
else
echo "Error: No configuration found"
exit 1
fi
fi
# Apply defaults if still not set
DB_HOST=${DB_HOST:-localhost}
DB_PORT=${DB_PORT:-5432}
BACKUP_RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-30}
LOG_LEVEL=${LOG_LEVEL:-INFO}
}
```
## Directory Structure Configuration
### Required Directories
```bash
# Create required directories
mkdir -p "${BACKUP_DIR}"
mkdir -p "${LOG_DIR}"
chmod 750 "${BACKUP_DIR}" "${LOG_DIR}"
```
### Directory Permissions
```
backups/ drwxr-x--- Script user:script group
logs/ drwxr-x--- Script user:script group
```
## Logging Configuration
### Log Files
- Format: `YYYY-MM-DD-script-name.log`
- Location: `${LOG_DIR}`
- Rotation: Daily
- Retention: 30 days
### Log Levels
```bash
LOG_LEVEL_DEBUG=0
LOG_LEVEL_INFO=1
LOG_LEVEL_WARN=2
LOG_LEVEL_ERROR=3
# Logging function
log_message() {
local level=$1
local message=$2
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
if [[ $level -ge ${LOG_LEVEL:-1} ]]; then
echo "${timestamp} [${level}] ${message}" >> "${LOG_FILE}"
fi
}
```
## Import Configuration
### GKWP Import Settings
```ini
# GKWP specific settings
GKWP_BATCH_SIZE=1000
GKWP_MAX_ERRORS=50
GKWP_TIMEOUT=300
```
### SEMRush Import Settings
```ini
# SEMRush specific settings
SEMRUSH_BATCH_SIZE=1000
SEMRUSH_MAX_ERRORS=50
SEMRUSH_TIMEOUT=300
```
### Plain List Import Settings
```ini
# Plain list import settings
PLAIN_BATCH_SIZE=5000
PLAIN_MAX_ERRORS=100
```
## Error Handling Configuration
### Error Levels
```bash
# Error level definitions
ERR_NONE=0
ERR_WARN=1
ERR_ERROR=2
ERR_FATAL=3
# Error handling configuration
MAX_RETRIES=3
RETRY_DELAY=5
```
### Error Handling Function
```bash
handle_error() {
local error_level=$1
local message=$2
local script_name=$(basename "$0")
case $error_level in
$ERR_WARN)
log_message "WARN" "${script_name}: ${message}"
;;
$ERR_ERROR)
log_message "ERROR" "${script_name}: ${message}"
return 1
;;
$ERR_FATAL)
log_message "FATAL" "${script_name}: ${message}"
exit 1
;;
esac
}
```
## Testing Configuration
### Test Database Settings
```ini
# Test database configuration
TEST_DB_NAME=keyword_management_test
TEST_DATA_DIR=tests/data
```
### Test Framework Configuration
```bash
# Test framework settings
TEST_TIMEOUT=30
TEST_PARALLEL=false
TEST_VERBOSE=true
```
## Backup Configuration
### Backup Settings
```ini
# Backup configuration
BACKUP_COMPRESSION=gzip
BACKUP_PREFIX=keyword_management
BACKUP_SUFFIX=.sql.gz
```
### Backup Rotation
```bash
# Backup rotation settings
DAILY_RETENTION=7
WEEKLY_RETENTION=4
MONTHLY_RETENTION=12
```
## Configuration Validation
### Validation Script
```bash
#!/bin/bash
validate_config() {
# Required variables
local required_vars=(
"DB_USER"
"DB_PASSWORD"
"DB_HOST"
"DB_PORT"
"DB_NAME"
)
# Check required variables
for var in "${required_vars[@]}"; do
if [[ -z "${!var}" ]]; then
echo "Error: Required variable $var is not set"
exit 1
fi
done
# Validate directories
if [[ ! -d "${BACKUP_DIR}" ]]; then
echo "Error: Backup directory does not exist"
exit 1
fi
if [[ ! -d "${LOG_DIR}" ]]; then
echo "Error: Log directory does not exist"
exit 1
fi
}
```
## Configuration Updates
### Update Process
1. Stop running scripts
2. Backup current configuration
3. Apply changes
4. Validate new configuration
5. Restart scripts
### Version Control
- Keep configuration templates in version control
- Document changes in changelog
- Include update instructions
Would you like me to expand on any of these configuration aspects or provide additional examples?

286
docs/project-docs/05-operations/troubleshooting.md Normal file
View file

@ -0,0 +1,286 @@
# Troubleshooting Guide
## Common Issues and Solutions
### Database Connection Issues
#### Unable to Connect to Database
```
Error: could not connect to server: Connection refused
```
**Possible Causes:**
1. PostgreSQL service not running
2. Incorrect connection details
3. Network/firewall issues
**Solutions:**
1. Check PostgreSQL service:
```bash
# Check service status
sudo systemctl status postgresql
# Start service if stopped
sudo systemctl start postgresql
```
2. Verify connection details in `.database`:
```bash
# Test connection manually
psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}"
```
3. Check firewall settings:
```bash
# Check if port is open
sudo netstat -tulpn | grep 5432
```
#### Authentication Failed
```
Error: FATAL: password authentication failed for user
```
**Solutions:**
1. Verify credentials in `.database`
2. Check PostgreSQL authentication settings:
```bash
# View pg_hba.conf location
psql -U postgres -c 'SHOW hba_file;'
# Check user permissions
psql -U postgres -c '\du'
```
### Import Process Issues
#### GKWP Import Failures
**File Format Errors:**
```
Error: Invalid filename format. Expected: GKWP - {KEYWORD}.csv
```
**Solutions:**
1. Check file naming convention
2. Verify file content format:
```bash
# View file headers
head -n 1 "GKWP - keyword.csv"
# Check file encoding
file -i "GKWP - keyword.csv"
```
**Data Processing Errors:**
```
Error: Invalid data format in row X
```
**Solutions:**
1. Examine problematic rows:
```bash
# View specific line
sed -n 'Xp' "GKWP - keyword.csv"
```
2. Check for special characters:
```bash
# Look for non-ASCII characters
grep -P '[^\x00-\x7F]' "GKWP - keyword.csv"
```
#### SEMRush Import Issues
**Duplicate Keywords:**
```
Error: duplicate key value violates unique constraint
```
**Solutions:**
1. Check existing keywords:
```sql
SELECT keyword
FROM Keywords
WHERE LOWER(keyword) = LOWER('problematic_keyword');
```
2. Use conflict resolution:
```sql
INSERT ... ON CONFLICT (LOWER(keyword)) DO UPDATE ...
```
### LibreOffice Base Connection Issues
#### Unable to Connect
**Symptoms:**
- Connection error dialog
- Database not showing in available connections
**Solutions:**
1. Verify PostgreSQL ODBC/JDBC settings
2. Check LibreOffice Base configuration:
- Database URL format
- Driver selection
- Connection parameters
#### Slow Performance
**Solutions:**
1. Check indexes:
```sql
-- View missing indexes
SELECT schemaname, tablename, reason, round(percent,2)
FROM pg_stat_user_tables
WHERE n_live_tup > 100000
ORDER BY n_live_tup DESC;
```
2. Optimize views:
```sql
-- Analyze view performance
EXPLAIN ANALYZE SELECT * FROM v_active_keywords_per_ad_group;
```
### Script Execution Issues
#### Permission Denied
```
bash: ./bin/db-admin/create-db.sh: Permission denied
```
**Solutions:**
1. Check file permissions:
```bash
# View permissions
ls -l bin/db-admin/create-db.sh
# Set correct permissions
chmod +x bin/db-admin/create-db.sh
```
2. Verify script ownership:
```bash
# Change ownership if needed
chown proper_user:proper_group bin/db-admin/create-db.sh
```
#### Path Issues
```
Error: Could not source .database file
```
**Solutions:**
1. Check script execution directory
2. Use absolute paths:
```bash
# Get script directory
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${SCRIPT_DIR}/../../.database"
```
### Data Integrity Issues
#### Orphaned Records
**Symptoms:**
- Missing relationships
- Incomplete data
**Solutions:**
1. Check for orphaned records:
```sql
-- Find keywords without ad groups
SELECT k.keyword_id, k.keyword
FROM Keywords k
LEFT JOIN Keyword_Ad_Group_Assignments kaga
ON k.keyword_id = kaga.keyword_id
WHERE kaga.ad_group_id IS NULL;
```
2. Clean up orphaned data:
```sql
-- Remove orphaned assignments
DELETE FROM Keyword_Ad_Group_Assignments
WHERE ad_group_id NOT IN (SELECT ad_group_id FROM Ad_Groups);
```
### Logging and Monitoring
#### Missing Logs
**Solutions:**
1. Check log directory permissions
2. Verify log configuration:
```bash
# Check log directory
ls -la logs/
# Create log directory if missing
mkdir -p logs/
chmod 750 logs/
```
#### Error Investigation
```bash
# Search for errors in logs
grep -r "ERROR" logs/
# View recent errors
tail -f logs/$(date +%Y-%m-%d)-*.log
```
## Preventive Measures
### Regular Maintenance
1. Database cleanup:
```sql
VACUUM ANALYZE;
```
2. Log rotation:
```bash
# Rotate logs older than 30 days
find logs/ -name "*.log" -mtime +30 -delete
```
3. Backup verification:
```bash
# Test backup integrity
./bin/db-admin/verify-backup.sh backups/latest.sql.gz
```
### Monitoring
1. Database size monitoring:
```sql
SELECT pg_size_pretty(pg_database_size('keyword_management'));
```
2. Table growth monitoring:
```sql
SELECT relname, n_live_tup
FROM pg_stat_user_tables
ORDER BY n_live_tup DESC;
```
## Emergency Procedures
### Database Recovery
1. Stop all active processes
2. Restore from latest backup:
```bash
./bin/db-admin/restore-db.sh backups/latest.sql.gz
```
### Data Corruption
1. Switch to read-only mode:
```sql
ALTER DATABASE keyword_management SET default_transaction_read_only = on;
```
2. Investigate issues:
```sql
-- Check table integrity
SELECT * FROM pg_stat_database WHERE datname = 'keyword_management';
```
## Support Resources
1. Log locations:
- Script logs: `/logs`
- PostgreSQL logs: (system-dependent)
- LibreOffice Base logs
2. Configuration files:
- `.database`
- PostgreSQL configuration
- Script configurations
Would you like me to expand on any particular troubleshooting aspect or add more examples?

254
docs/project-docs/05-operations/usage.md Normal file
View file

@ -0,0 +1,254 @@
# System Usage Documentation
## Overview
This document details how to use the keyword management system, including script execution, database operations, and LibreOffice Base interactions. The system uses a combination of command-line scripts for administration and LibreOffice Base for daily operations.
## Initial Setup
### 1. Configuration
```bash
# Copy configuration template
cp .database.template .database
# Edit configuration with your settings
nano .database
# Set proper permissions
chmod 600 .database
```
### 2. Database Initialization
```bash
# Create database
./bin/db-admin/create-db.sh
# Verify creation
psql -h localhost -U postgres -d keyword_management -c "\l"
```
### 3. LibreOffice Base Connection
1. Open LibreOffice Base
2. Select "Connect to existing database"
3. Choose "PostgreSQL"
4. Enter connection details from .database file
## Database Administration
### Creating the Database
```bash
# Create new database
./bin/db-admin/create-db.sh
# With specific locale (if needed)
./bin/db-admin/create-db.sh --locale en-US
```
### Backup and Restore
```bash
# Create backup
./bin/db-admin/backup-db.sh
# Restore from backup
./bin/db-admin/restore-db.sh backups/keyword_management-2024-11-19-100000.sql.gz
```
### Database Removal
```bash
# Remove database (requires confirmation)
./bin/db-admin/drop-db.sh
```
## Table Management
### Creating Tables
```bash
# Create all tables
./bin/table-admin/create-tables/create-all-tables.sh
# Create specific table
./bin/table-admin/create-tables/create-keywords-table.sh
```
### Table Modifications
```bash
# Apply specific version
./bin/table-admin/alter-tables/execute-version.sh 20241119_01 upgrade
# Rollback version
./bin/table-admin/alter-tables/execute-version.sh 20241119_01 rollback
```
## Data Import Operations
### GKWP Import
```bash
# Import single GKWP file
./bin/import-scripts/gkwp/import-gkwp-file.sh "GKWP - keyword.csv"
# Batch import GKWP files
./bin/import-scripts/gkwp/batch-import-gkwp.sh /path/to/gkwp/files
```
### SEMRush Import
```bash
# Import SEMRush keyword data
./bin/import-scripts/semrush/standard/import-semrush-file.sh "SEMRush - keyword.csv"
# Import competitor data
./bin/import-scripts/semrush/competitor/import-competitor-file.sh "SEMRush Comp - example.com.csv"
```
### Plain List Import
```bash
# Basic keyword list import
./bin/import-scripts/plain-list/import-keywords.sh keywords.txt
# Import with ad group assignment
./bin/import-scripts/plain-list/import-with-adgroups.sh keywords.csv ad_group_id
# Create ad group and import
./bin/import-scripts/plain-list/create-adgroup-import.sh keywords.txt "New Ad Group" campaign_id landing_page_id
```
## LibreOffice Base Operations
### Campaign Management
1. Open campaign management form
2. Create new campaign:
- Enter campaign name
- Set status
- Add notes
- Save
### Ad Group Management
1. Open ad group management form
2. Create new ad group:
- Select campaign
- Choose landing page
- Enter ad group name
- Save
### Keyword Assignment
1. Open keyword assignment form
2. Select target ad group
3. Choose keywords to assign
4. Confirm assignment
### Data Views
Available views:
- Active Keywords per Ad Group
- Landing Page Usage
- Recent Imports
- Keyword Metrics
## Testing
### Running Tests
```bash
# Run all tests
./tests/run-all.sh
# Run specific test category
./tests/run-category.sh create-functions
# Run single test
./tests/db-admin/create-db.test.sh
```
### Test Result Verification
```bash
# View test logs
cat logs/YYYY-MM-DD-test-execution.log
# Check test coverage
./tests/check-coverage.sh
```
## Common Operations
### Adding New Campaign
1. Create campaign using LibreOffice Base form
2. Add landing pages
3. Create ad groups
4. Import keywords
5. Assign keywords to ad groups
### Updating Keyword Data
1. Export current keywords
2. Get new data from source
3. Run appropriate import script
4. Verify updates in LibreOffice Base
### Managing Ad Groups
1. Open ad group management view
2. Create or modify ad groups
3. Assign landing pages
4. Add keywords through import or assignment
## Maintenance
### Regular Maintenance
```bash
# Backup database
./bin/db-admin/backup-db.sh
# Clean old logs
./bin/maintenance/clean-logs.sh
# Check database health
./bin/maintenance/check-db-health.sh
```
### Error Recovery
```bash
# View recent errors
./bin/maintenance/show-errors.sh
# Retry failed imports
./bin/maintenance/retry-imports.sh
```
## Best Practices
### Data Import
1. Always verify file format before import
2. Use appropriate import script for data source
3. Check import logs for errors
4. Verify data in LibreOffice Base after import
### Database Management
1. Regular backups
2. Monitor database size
3. Archive old data periodically
4. Maintain error logs
### Ad Group Organization
1. Consistent naming conventions
2. Clear campaign structure
3. Proper landing page assignments
4. Regular review of keyword assignments
## Troubleshooting
### Common Issues
1. Import failures
- Check file format
- Verify permissions
- Review error logs
2. Connection issues
- Verify database configuration
- Check network connectivity
- Confirm credentials
3. Performance problems
- Monitor database size
- Check index usage
- Review query patterns
### Getting Help
- Check logs in `/logs` directory
- Review documentation
- Contact system administrator
Would you like me to expand on any particular aspect of system usage or provide additional examples?

86
exports/functions/initialize_gkwp_import.sql Normal file
View file

@ -0,0 +1,86 @@
CREATE OR REPLACE FUNCTION public.initialize_gkwp_import(p_filename text, p_header_row text, p_raw_data text, p_confirmed_seed_keyword text)
RETURNS TABLE(result_import_id integer, result_status text, result_message text)
LANGUAGE plpgsql
AS $function$
DECLARE
v_seed_keyword_id INTEGER;
v_import_id INTEGER;
v_validation RECORD;
v_formatted_header TEXT;
v_formatted_data TEXT;
BEGIN
-- First validate filename
SELECT * INTO v_validation
FROM validate_gkwp_filename(p_filename);
IF NOT v_validation.is_valid THEN
RETURN QUERY VALUES (
NULL::INTEGER,
'ERROR'::TEXT,
v_validation.error_message
);
RETURN;
END IF;
-- Format header and data to ensure proper line breaks
-- Replace any existing line breaks with a standard one to avoid mixed line endings
v_formatted_header := regexp_replace(p_header_row, '\r\n|\r|\n', E'\n', 'g');
v_formatted_data := regexp_replace(p_raw_data, '\r\n|\r|\n', E'\n', 'g');
-- Get or create seed keyword ID
INSERT INTO keywords (keyword, first_imported, last_updated)
VALUES (LOWER(p_confirmed_seed_keyword), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (LOWER(keyword))
DO UPDATE SET last_updated = CURRENT_TIMESTAMP
RETURNING keyword_id INTO v_seed_keyword_id;
-- Create import history record
INSERT INTO importhistory (
source_type,
filename,
original_seed_keyword,
confirmed_seed_keyword_id,
import_status
) VALUES (
'GKWP',
p_filename,
v_validation.extracted_keyword,
v_seed_keyword_id,
'PROCESSING'
) RETURNING import_id INTO v_import_id;
-- Store raw data with proper formatting
INSERT INTO importrawdata (
import_id,
header_row,
raw_data
) VALUES (
v_import_id,
v_formatted_header,
v_formatted_data
);
RETURN QUERY VALUES (
v_import_id,
'SUCCESS'::TEXT,
'Import initialized successfully'::TEXT
);
EXCEPTION WHEN OTHERS THEN
-- If we created an import record but failed later, mark it as failed
IF v_import_id IS NOT NULL THEN
UPDATE importhistory
SET import_status = 'FAILED',
error_details = SQLERRM
WHERE import_id = v_import_id;
END IF;
RETURN QUERY VALUES (
v_import_id,
'ERROR'::TEXT,
SQLERRM::TEXT
);
END;
$function$
(1 row)

149
exports/functions/process_gkwp_import_data.sql Normal file
View file

@ -0,0 +1,149 @@
CREATE OR REPLACE FUNCTION public.process_gkwp_import_data(p_import_id integer)
RETURNS TABLE(result_status text, result_message text, rows_processed integer, debug_info text)
LANGUAGE plpgsql
AS $function$
DECLARE
v_raw_data TEXT;
v_header_row TEXT;
v_current_row TEXT;
v_rows_processed INTEGER := 0;
v_row_array TEXT[];
v_header_array TEXT[];
v_jsonb_data JSONB;
v_column_count INTEGER;
v_found_header BOOLEAN := FALSE;
v_metadata JSONB := '{}'::JSONB;
v_lines TEXT[];
v_debug_info TEXT := '';
BEGIN
-- Get the raw data for this import
SELECT raw_data, header_row
INTO v_raw_data, v_header_row
FROM importrawdata
WHERE import_id = p_import_id;
IF v_raw_data IS NULL THEN
RETURN QUERY VALUES(
'ERROR'::TEXT,
'No raw data found for import ID ' || p_import_id::TEXT,
0,
'No raw data found'::TEXT
);
RETURN;
END IF;
-- Add first debug info
v_debug_info := 'Raw data length: ' || length(v_raw_data)::TEXT;
-- First process the header row string into lines
v_lines := string_to_array(v_header_row, E'\n');
v_debug_info := v_debug_info || E'\nHeader lines count: ' || coalesce(array_length(v_lines, 1)::TEXT, 'NULL');
-- Process header lines to find the actual header row
FOR i IN 1..coalesce(array_length(v_lines, 1), 0) LOOP
v_row_array := string_to_array(v_lines[i], E'\t');
-- Store first two rows as metadata
IF i <= 2 THEN
v_metadata := v_metadata || jsonb_build_object(
'meta_row_' || i::text,
v_lines[i]
);
END IF;
-- Check if this is the header row
IF v_row_array[1] = 'Keyword' THEN
v_header_array := v_row_array;
v_column_count := array_length(v_header_array, 1);
v_found_header := TRUE;
-- Store metadata in ImportRawData
UPDATE importrawdata
SET column_mappings = v_metadata
WHERE import_id = p_import_id;
v_debug_info := v_debug_info || E'\nFound header row with ' || v_column_count::TEXT || ' columns';
EXIT; -- Found our header, exit the loop
END IF;
END LOOP;
IF NOT v_found_header THEN
RETURN QUERY VALUES(
'ERROR'::TEXT,
'Could not find header row in import data',
0,
v_debug_info
);
RETURN;
END IF;
-- Now process the actual data rows
v_lines := string_to_array(v_raw_data, E'\n');
v_debug_info := v_debug_info || E'\nData lines count: ' || coalesce(array_length(v_lines, 1)::TEXT, 'NULL');
FOR v_current_row IN SELECT unnest(v_lines) LOOP
-- Skip empty rows
IF length(trim(v_current_row)) = 0 THEN
CONTINUE;
END IF;
v_row_array := string_to_array(v_current_row, E'\t');
-- Create JSONB object with ALL columns
v_jsonb_data := '{}'::JSONB;
FOR i IN 1..least(v_column_count, array_length(v_row_array, 1)) LOOP
IF v_header_array[i] IS NOT NULL THEN
v_jsonb_data := v_jsonb_data || jsonb_build_object(
trim(v_header_array[i]),
NULLIF(trim(v_row_array[i]), '')
);
END IF;
END LOOP;
-- Insert into ImportedRows
INSERT INTO importedrows (
import_id,
row_number,
row_data,
imported_to_main
) VALUES (
p_import_id,
v_rows_processed + 1,
v_jsonb_data,
FALSE
);
v_rows_processed := v_rows_processed + 1;
END LOOP;
-- Update import history status
UPDATE importhistory
SET
row_count = v_rows_processed,
import_status = 'PROCESSED'
WHERE import_id = p_import_id;
RETURN QUERY VALUES(
'SUCCESS'::TEXT,
v_rows_processed::TEXT || ' rows processed successfully',
v_rows_processed,
v_debug_info
);
EXCEPTION WHEN OTHERS THEN
UPDATE importhistory
SET
import_status = 'PROCESSING_FAILED',
error_details = SQLERRM
WHERE import_id = p_import_id;
RETURN QUERY VALUES(
'ERROR'::TEXT,
'Error processing rows: ' || SQLERRM,
v_rows_processed,
v_debug_info || E'\nException occurred: ' || SQLERRM
);
END;
$function$
(1 row)

116
exports/functions/update_keywords_from_gkwp_import.sql Normal file
View file

@ -0,0 +1,116 @@
CREATE OR REPLACE FUNCTION public.update_keywords_from_gkwp_import(p_import_id integer)
RETURNS TABLE(result_status text, result_message text, keywords_updated integer, keywords_inserted integer)
LANGUAGE plpgsql
AS $function$
DECLARE
v_keywords_updated INTEGER := 0;
v_keywords_inserted INTEGER := 0;
v_updated_ids INTEGER[];
v_inserted_ids INTEGER[];
BEGIN
-- First insert any new keywords
WITH processed_rows AS (
SELECT
row_data->>'Keyword' as keyword,
(row_data->>'Avg. monthly searches')::INTEGER as search_volume,
(row_data->>'Competition (indexed value)')::INTEGER as competition_index,
(row_data->>'Top of page bid (low range)')::NUMERIC as cpc_min,
(row_data->>'Top of page bid (high range)')::NUMERIC as cpc_max
FROM importedrows
WHERE import_id = p_import_id
AND NOT imported_to_main
),
new_keywords AS (
INSERT INTO keywords (
keyword,
gkwp_search_volume,
gkwp_competition_index,
gkwp_cpc_min,
gkwp_cpc_max,
gkwp_last_updated,
first_imported,
last_updated
)
SELECT
pr.keyword,
pr.search_volume,
pr.competition_index,
pr.cpc_min,
pr.cpc_max,
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP,
CURRENT_TIMESTAMP
FROM processed_rows pr
WHERE NOT EXISTS (
SELECT 1 FROM keywords k
WHERE LOWER(k.keyword) = LOWER(pr.keyword)
)
RETURNING keyword_id
)
SELECT array_agg(keyword_id)
INTO v_inserted_ids
FROM new_keywords;
-- Then update existing keywords
WITH processed_rows AS (
SELECT
row_data->>'Keyword' as keyword,
(row_data->>'Avg. monthly searches')::INTEGER as search_volume,
(row_data->>'Competition (indexed value)')::INTEGER as competition_index,
(row_data->>'Top of page bid (low range)')::NUMERIC as cpc_min,
(row_data->>'Top of page bid (high range)')::NUMERIC as cpc_max
FROM importedrows
WHERE import_id = p_import_id
AND NOT imported_to_main
),
updated_keywords AS (
UPDATE keywords k
SET
gkwp_search_volume = pr.search_volume,
gkwp_competition_index = pr.competition_index,
gkwp_cpc_min = pr.cpc_min,
gkwp_cpc_max = pr.cpc_max,
gkwp_last_updated = CURRENT_TIMESTAMP,
last_updated = CURRENT_TIMESTAMP
FROM processed_rows pr
WHERE LOWER(k.keyword) = LOWER(pr.keyword)
RETURNING k.keyword_id
)
SELECT array_agg(keyword_id)
INTO v_updated_ids
FROM updated_keywords;
-- Count updates and inserts
v_keywords_updated := coalesce(array_length(v_updated_ids, 1), 0);
v_keywords_inserted := coalesce(array_length(v_inserted_ids, 1), 0);
-- Mark rows as imported
UPDATE importedrows
SET imported_to_main = TRUE
WHERE import_id = p_import_id
AND NOT imported_to_main;
-- Update import history status
UPDATE importhistory
SET import_status = 'COMPLETED'
WHERE import_id = p_import_id;
RETURN QUERY VALUES(
'SUCCESS'::TEXT,
v_keywords_updated::TEXT || ' keywords updated, ' ||
v_keywords_inserted::TEXT || ' keywords inserted',
v_keywords_updated,
v_keywords_inserted
);
EXCEPTION WHEN OTHERS THEN
RETURN QUERY VALUES(
'ERROR'::TEXT,
'Error processing keywords: ' || SQLERRM,
v_keywords_updated,
v_keywords_inserted
);
END;
$function$
(1 row)

23
exports/functions/validate_gkwp_filename.sql Normal file
View file

@ -0,0 +1,23 @@
CREATE OR REPLACE FUNCTION public.validate_gkwp_filename(filename text)
RETURNS TABLE(is_valid boolean, error_message text, extracted_keyword text)
LANGUAGE plpgsql
AS $function$
BEGIN
-- Check if filename matches pattern
IF filename !~ '^GKWP - .+\.csv$' THEN
RETURN QUERY SELECT
false,
'Invalid filename format. Expected: "GKWP - {KEYWORD}.csv"',
NULL::TEXT;
RETURN;
END IF;
-- Extract the keyword part
RETURN QUERY SELECT
true,
'Valid filename',
substring(filename from '^GKWP - (.+)\.csv$');
END;
$function$
(1 row)