150 lines
4.4 KiB
PL/PgSQL
150 lines
4.4 KiB
PL/PgSQL
CREATE OR REPLACE FUNCTION public.process_gkwp_import_data(p_import_id integer)
|
|
RETURNS TABLE(result_status text, result_message text, rows_processed integer, debug_info text)
|
|
LANGUAGE plpgsql
|
|
AS $function$
|
|
DECLARE
|
|
v_raw_data TEXT;
|
|
v_header_row TEXT;
|
|
v_current_row TEXT;
|
|
v_rows_processed INTEGER := 0;
|
|
v_row_array TEXT[];
|
|
v_header_array TEXT[];
|
|
v_jsonb_data JSONB;
|
|
v_column_count INTEGER;
|
|
v_found_header BOOLEAN := FALSE;
|
|
v_metadata JSONB := '{}'::JSONB;
|
|
v_lines TEXT[];
|
|
v_debug_info TEXT := '';
|
|
BEGIN
|
|
-- Get the raw data for this import
|
|
SELECT raw_data, header_row
|
|
INTO v_raw_data, v_header_row
|
|
FROM importrawdata
|
|
WHERE import_id = p_import_id;
|
|
|
|
IF v_raw_data IS NULL THEN
|
|
RETURN QUERY VALUES(
|
|
'ERROR'::TEXT,
|
|
'No raw data found for import ID ' || p_import_id::TEXT,
|
|
0,
|
|
'No raw data found'::TEXT
|
|
);
|
|
RETURN;
|
|
END IF;
|
|
|
|
-- Add first debug info
|
|
v_debug_info := 'Raw data length: ' || length(v_raw_data)::TEXT;
|
|
|
|
-- First process the header row string into lines
|
|
v_lines := string_to_array(v_header_row, E'\n');
|
|
v_debug_info := v_debug_info || E'\nHeader lines count: ' || coalesce(array_length(v_lines, 1)::TEXT, 'NULL');
|
|
|
|
-- Process header lines to find the actual header row
|
|
FOR i IN 1..coalesce(array_length(v_lines, 1), 0) LOOP
|
|
v_row_array := string_to_array(v_lines[i], E'\t');
|
|
|
|
-- Store first two rows as metadata
|
|
IF i <= 2 THEN
|
|
v_metadata := v_metadata || jsonb_build_object(
|
|
'meta_row_' || i::text,
|
|
v_lines[i]
|
|
);
|
|
END IF;
|
|
|
|
-- Check if this is the header row
|
|
IF v_row_array[1] = 'Keyword' THEN
|
|
v_header_array := v_row_array;
|
|
v_column_count := array_length(v_header_array, 1);
|
|
v_found_header := TRUE;
|
|
|
|
-- Store metadata in ImportRawData
|
|
UPDATE importrawdata
|
|
SET column_mappings = v_metadata
|
|
WHERE import_id = p_import_id;
|
|
|
|
v_debug_info := v_debug_info || E'\nFound header row with ' || v_column_count::TEXT || ' columns';
|
|
EXIT; -- Found our header, exit the loop
|
|
END IF;
|
|
END LOOP;
|
|
|
|
IF NOT v_found_header THEN
|
|
RETURN QUERY VALUES(
|
|
'ERROR'::TEXT,
|
|
'Could not find header row in import data',
|
|
0,
|
|
v_debug_info
|
|
);
|
|
RETURN;
|
|
END IF;
|
|
|
|
-- Now process the actual data rows
|
|
v_lines := string_to_array(v_raw_data, E'\n');
|
|
v_debug_info := v_debug_info || E'\nData lines count: ' || coalesce(array_length(v_lines, 1)::TEXT, 'NULL');
|
|
|
|
FOR v_current_row IN SELECT unnest(v_lines) LOOP
|
|
-- Skip empty rows
|
|
IF length(trim(v_current_row)) = 0 THEN
|
|
CONTINUE;
|
|
END IF;
|
|
|
|
v_row_array := string_to_array(v_current_row, E'\t');
|
|
|
|
-- Create JSONB object with ALL columns
|
|
v_jsonb_data := '{}'::JSONB;
|
|
FOR i IN 1..least(v_column_count, array_length(v_row_array, 1)) LOOP
|
|
IF v_header_array[i] IS NOT NULL THEN
|
|
v_jsonb_data := v_jsonb_data || jsonb_build_object(
|
|
trim(v_header_array[i]),
|
|
NULLIF(trim(v_row_array[i]), '')
|
|
);
|
|
END IF;
|
|
END LOOP;
|
|
|
|
-- Insert into ImportedRows
|
|
INSERT INTO importedrows (
|
|
import_id,
|
|
row_number,
|
|
row_data,
|
|
imported_to_main
|
|
) VALUES (
|
|
p_import_id,
|
|
v_rows_processed + 1,
|
|
v_jsonb_data,
|
|
FALSE
|
|
);
|
|
|
|
v_rows_processed := v_rows_processed + 1;
|
|
END LOOP;
|
|
|
|
-- Update import history status
|
|
UPDATE importhistory
|
|
SET
|
|
row_count = v_rows_processed,
|
|
import_status = 'PROCESSED'
|
|
WHERE import_id = p_import_id;
|
|
|
|
RETURN QUERY VALUES(
|
|
'SUCCESS'::TEXT,
|
|
v_rows_processed::TEXT || ' rows processed successfully',
|
|
v_rows_processed,
|
|
v_debug_info
|
|
);
|
|
|
|
EXCEPTION WHEN OTHERS THEN
|
|
UPDATE importhistory
|
|
SET
|
|
import_status = 'PROCESSING_FAILED',
|
|
error_details = SQLERRM
|
|
WHERE import_id = p_import_id;
|
|
|
|
RETURN QUERY VALUES(
|
|
'ERROR'::TEXT,
|
|
'Error processing rows: ' || SQLERRM,
|
|
v_rows_processed,
|
|
v_debug_info || E'\nException occurred: ' || SQLERRM
|
|
);
|
|
END;
|
|
$function$
|
|
|
|
(1 row)
|