hirejared-kw-db/docs/project-docs/03-scripts/import-scripts/competitor-import.md

Competitor Import Scripts Documentation

Overview

These scripts handle the import of competitor keyword ranking data from SEMRush competitor exports. The system tracks competitor keyword rankings over time and maintains historical position data.

File Format Requirements

• Comma-delimited CSV files
• Filename pattern: "SEMRush Comp - {WEBSITE}.csv"
• Key fields:
  • Keyword
  • Position
  • Volume
  • Traffic %
  • Costs %
  • Competition
  • SERP Features
  • Results
  • Trends

Script Locations

/bin/import-scripts/competitor/
├── import-competitor-file.sh      # Single file import
├── batch-import-competitor.sh     # Multiple file import
├── manage-competitors.sh          # Add/update competitor info
└── sql/
    ├── add_competitor.sql        # Add new competitor
    ├── update_competitor.sql     # Update competitor info
    ├── process_rankings.sql      # Process ranking data
    └── archive_rankings.sql      # Archive historical data

Import Process Flow

graph TD
    A[Read CSV File] --> B[Extract Domain]
    B --> C[Create/Update Competitor]
    C --> D[Process Rankings]
    D --> E[Archive Old Rankings]
    E --> F[Update Current Rankings]
    F --> G[Log Results]

Competitor Management Script

Location

/bin/import-scripts/competitor/manage-competitors.sh

Implementation

#!/bin/bash

# Load configuration
source ../../../.database

# Set error handling
set -e

# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-competitor-management.log"

mkdir -p "$LOG_DIR"

# Function to log messages
log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

# Function to add competitor
add_competitor() {
    local domain=$1
    local notes=$2
    
    psql -h "${DB_HOST}" \
         -p "${DB_PORT}" \
         -U "${DB_USER}" \
         -d "${DB_NAME}" \
         -v domain="$domain" \
         -v notes="$notes" \
         -f "sql/add_competitor.sql"
}

# Function to update competitor
update_competitor() {
    local domain=$1
    local notes=$2
    
    psql -h "${DB_HOST}" \
         -p "${DB_PORT}" \
         -U "${DB_USER}" \
         -d "${DB_NAME}" \
         -v domain="$domain" \
         -v notes="$notes" \
         -f "sql/update_competitor.sql"
}

# Parse command line arguments
case "$1" in
    "add")
        add_competitor "$2" "$3"
        ;;
    "update")
        update_competitor "$2" "$3"
        ;;
    *)
        echo "Usage: $0 {add|update} <domain> [notes]"
        exit 1
        ;;
esac

Competitor Import Script

Location

/bin/import-scripts/competitor/import-competitor-file.sh

Implementation

#!/bin/bash

# Load configuration
source ../../../.database

# Set error handling
set -e

# Setup logging
LOG_DIR="../../../logs"
LOG_FILE="${LOG_DIR}/$(date +%Y-%m-%d)-competitor-import.log"

mkdir -p "$LOG_DIR"

# Function to log messages
log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

# Check arguments
if [ $# -lt 1 ]; then
    echo "Usage: $0 <competitor_export_file>"
    exit 1
fi

INPUT_FILE=$1

# Extract domain from filename
DOMAIN=$(basename "$INPUT_FILE" | sed -n 's/SEMRush Comp - \(.*\)\.csv/\1/p')
if [ -z "$DOMAIN" ]; then
    log_message "Error: Invalid filename format. Expected: SEMRush Comp - {WEBSITE}.csv"
    exit 1
fi

# Create import history record
IMPORT_ID=$(psql -h "${DB_HOST}" \
                 -p "${DB_PORT}" \
                 -U "${DB_USER}" \
                 -d "${DB_NAME}" \
                 -tAc "INSERT INTO Import_History 
                       (source_type, filename, original_seed_keyword, import_status) 
                       VALUES ('SEMRUSH_COMP', '$(basename "$INPUT_FILE")', 
                               '${DOMAIN}', 'IN_PROGRESS') 
                       RETURNING import_id;")

# Process file
log_message "Processing competitor export file: $(basename "$INPUT_FILE")"
log_message "Domain: ${DOMAIN}"

# Archive existing rankings
psql -h "${DB_HOST}" \
     -p "${DB_PORT}" \
     -U "${DB_USER}" \
     -d "${DB_NAME}" \
     -v domain="$DOMAIN" \
     -f "sql/archive_rankings.sql"

# Import new rankings
psql -h "${DB_HOST}" \
     -p "${DB_PORT}" \
     -U "${DB_USER}" \
     -d "${DB_NAME}" \
     -v import_id="$IMPORT_ID" \
     -v domain="$DOMAIN" \
     -f "sql/process_rankings.sql"

# Update import history with results
RESULTS=$(psql -h "${DB_HOST}" \
               -p "${DB_PORT}" \
               -U "${DB_USER}" \
               -d "${DB_NAME}" \
               -tAc "SELECT success_count, error_count 
                     FROM Import_History 
                     WHERE import_id = ${IMPORT_ID};")

log_message "Import completed for ${INPUT_FILE}"
log_message "Results: ${RESULTS}"

SQL Processing Scripts

-- sql/process_rankings.sql

-- Create temporary table for import
CREATE TEMP TABLE competitor_rankings (
    keyword TEXT,
    position INTEGER,
    volume INTEGER,
    traffic_percent DECIMAL(5,2),
    costs_percent DECIMAL(5,2),
    competition INTEGER,
    serp_features TEXT,
    results INTEGER,
    trends TEXT
);

-- Import CSV data
COPY competitor_rankings FROM STDIN WITH (FORMAT csv, DELIMITER ',', HEADER true);

-- Process rankings and update competitor keywords
WITH new_keywords AS (
    INSERT INTO Keywords (keyword)
    SELECT DISTINCT LOWER(keyword)
    FROM competitor_rankings
    ON CONFLICT (LOWER(keyword)) DO NOTHING
    RETURNING keyword_id, keyword
),
all_keywords AS (
    SELECT keyword_id, keyword 
    FROM Keywords 
    WHERE keyword IN (SELECT LOWER(keyword) FROM competitor_rankings)
),
competitor_id AS (
    SELECT competitor_id 
    FROM Competitors 
    WHERE domain = :'domain'
)
INSERT INTO CompetitorKeywords (
    competitor_id,
    keyword_id,
    position,
    import_id,
    first_seen,
    last_seen
)
SELECT 
    (SELECT competitor_id FROM competitor_id),
    k.keyword_id,
    cr.position,
    :import_id,
    CURRENT_TIMESTAMP,
    CURRENT_TIMESTAMP
FROM competitor_rankings cr
JOIN all_keywords k ON LOWER(cr.keyword) = k.keyword;

-- Update import history
UPDATE Import_History
SET 
    import_status = 'SUCCESS',
    success_count = (SELECT COUNT(*) FROM competitor_rankings),
    error_count = 0
WHERE import_id = :import_id;

-- Cleanup
DROP TABLE competitor_rankings;

Testing

Located in /tests/import-scripts/competitor/

Example test:

#!/bin/bash

source ../../../utils/test-framework.sh

test_competitor_import() {
    # Create test competitor
    ../bin/import-scripts/competitor/manage-competitors.sh add "example.com" "Test competitor"
    
    # Create test file
    cat > "test-data/SEMRush Comp - example.com.csv" << EOL
Keyword,Position,Volume,Traffic %,Costs %,Competition,SERP Features,Results,Trends
test keyword,1,1000,2.5,3.2,45,featured snippet,1000000,up
another test,5,2000,1.8,2.1,65,local pack,2000000,stable
EOL

    # Run import
    ../bin/import-scripts/competitor/import-competitor-file.sh "test-data/SEMRush Comp - example.com.csv"
    
    # Verify competitor
    result=$(psql -tAc "SELECT COUNT(*) FROM Competitors WHERE domain = 'example.com';" "${DB_NAME}")
    assert_equals "$result" "1" "Competitor should exist"
    
    # Verify rankings
    rankings=$(psql -tAc "SELECT COUNT(*) FROM CompetitorKeywords ck 
                         JOIN Competitors c ON ck.competitor_id = c.competitor_id 
                         WHERE c.domain = 'example.com';" "${DB_NAME}")
    assert_equals "$rankings" "2" "Should import both keyword rankings"
    
    # Clean up
    rm "test-data/SEMRush Comp - example.com.csv"
}

run_test_suite

Error Handling

1. File Format

   • Header validation
   • Data type checking
   • Required field verification

2. Competitor Management

   • Domain verification
   • Duplicate checking
   • Status tracking

3. Ranking Data

   • Position validation
   • Historical data preservation
   • Metric range checking

Best Practices

1. Data Integrity

   • Transaction management
   • Historical data archiving
   • Consistent keyword matching

2. Performance

   • Batch processing
   • Index utilization
   • Efficient updates

3. Monitoring

   • Ranking changes tracking
   • Import statistics
   • Error reporting

4. Security

   • Domain validation
   • Access control
   • Data protection