feat: Implement batch processing and pagination for large CSV imports
This commit is contained in:
parent
7e5318dae5
commit
8115fb3ece
|
|
@ -27,7 +27,7 @@
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for data in ids_data %}
|
||||
{% for data in page_obj %}
|
||||
<tr>
|
||||
<td>{{ data.business_name }}</td>
|
||||
<td>{{ data.gbp_business_category }}</td>
|
||||
|
|
@ -48,4 +48,30 @@
|
|||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<nav aria-label="Page navigation">
|
||||
<ul class="pagination justify-content-center">
|
||||
{% if page_obj.has_previous %}
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="?page=1">« First</a>
|
||||
</li>
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="?page={{ page_obj.previous_page_number }}">Previous</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
|
||||
<li class="page-item disabled">
|
||||
<span class="page-link">Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}</span>
|
||||
</li>
|
||||
|
||||
{% if page_obj.has_next %}
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="?page={{ page_obj.next_page_number }}">Next</a>
|
||||
</li>
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="?page={{ page_obj.paginator.num_pages }}">Last »</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
</ul>
|
||||
</nav>
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
from django.shortcuts import render, redirect, get_object_or_404
|
||||
from django.db.models import Q
|
||||
from django.contrib import messages
|
||||
from django.db import transaction
|
||||
from .models import Lead, IDSData
|
||||
from .forms import LeadForm
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
from django.core.paginator import Paginator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -61,8 +63,11 @@ def data_sources(request):
|
|||
return render(request, 'leads/data_sources.html')
|
||||
|
||||
def ids_data(request):
|
||||
ids_data = IDSData.objects.all()
|
||||
return render(request, 'leads/ids_data.html', {'ids_data': ids_data})
|
||||
ids_data_list = IDSData.objects.all()
|
||||
paginator = Paginator(ids_data_list, 50) # Show 50 items per page
|
||||
page_number = request.GET.get('page')
|
||||
page_obj = paginator.get_page(page_number)
|
||||
return render(request, 'leads/ids_data.html', {'page_obj': page_obj})
|
||||
|
||||
def ids_data_import(request):
|
||||
if request.method == 'POST':
|
||||
|
|
@ -72,42 +77,52 @@ def ids_data_import(request):
|
|||
decoded_file = csv_file.read().decode('utf-8')
|
||||
io_string = io.StringIO(decoded_file)
|
||||
reader = csv.DictReader(io_string)
|
||||
imported_count = 0
|
||||
for row in reader:
|
||||
try:
|
||||
IDSData.objects.update_or_create(
|
||||
|
||||
batch_size = 1000
|
||||
batch = []
|
||||
total_imported = 0
|
||||
|
||||
with transaction.atomic():
|
||||
for row in reader:
|
||||
ids_data = IDSData(
|
||||
business_uuid=row['Business UUID'],
|
||||
location_uuid=row['Location UUID'],
|
||||
defaults={
|
||||
'business_name': row['Business Name'],
|
||||
'yib': row['YiB'],
|
||||
'gbp_business_category': row['GBP Business Category'],
|
||||
'gbp_matching_service': row['GBP Matching Service'],
|
||||
'gbp_review_rating': float(row['GBP Review Rating']) if row['GBP Review Rating'] else None,
|
||||
'gbp_review_count': int(float(row['GBP Review Count'])) if row['GBP Review Count'] else None,
|
||||
'gbp_industry_avg_review_rating': float(row['GBP Industry Avg Review Rating']) if row['GBP Industry Avg Review Rating'] else None,
|
||||
'gbp_industry_avg_review_count': float(row['GBP Industry Avg Review Count']) if row['GBP Industry Avg Review Count'] else None,
|
||||
'gbp_review_rating_offset': float(row['GBP Review Rating Offset fr. Avg']) if row['GBP Review Rating Offset fr. Avg'] else None,
|
||||
'gbp_review_count_offset': float(row['GBP Review Count Offset fr. Avg']) if row['GBP Review Count Offset fr. Avg'] else None,
|
||||
'gbp_review_rating_health': row['GBP Review Rating Health'],
|
||||
'gbp_review_count_health': row['GBP Review Count Health'],
|
||||
'gbp_location_municipality': row['GBP Location Municipality'],
|
||||
'gbp_location_state': row['GBP Location State'],
|
||||
'location_from_gbp_business_name': row['Location from GBP Business Name'],
|
||||
'gbp_business_phone': row['GBP Business Phone'],
|
||||
'gbp_business_website': row['GBP Business Website'],
|
||||
'root_domain': row['Root Domain'],
|
||||
'md5_for_scrape': row['MD5 for Scrape'],
|
||||
'services_reputation_management': row['Services: Reputation Management'] == 'Yes',
|
||||
'services_gbp_optimization': row['Services: GBP Optimization'] == 'Yes',
|
||||
'services_needs_website': row['Services: Needs Website'] == 'Yes'
|
||||
}
|
||||
business_name=row['Business Name'],
|
||||
yib=row['YiB'],
|
||||
gbp_business_category=row['GBP Business Category'],
|
||||
gbp_matching_service=row['GBP Matching Service'],
|
||||
gbp_review_rating=float(row['GBP Review Rating']) if row['GBP Review Rating'] else None,
|
||||
gbp_review_count=int(float(row['GBP Review Count'])) if row['GBP Review Count'] else None,
|
||||
gbp_industry_avg_review_rating=float(row['GBP Industry Avg Review Rating']) if row['GBP Industry Avg Review Rating'] else None,
|
||||
gbp_industry_avg_review_count=float(row['GBP Industry Avg Review Count']) if row['GBP Industry Avg Review Count'] else None,
|
||||
gbp_review_rating_offset=float(row['GBP Review Rating Offset fr. Avg']) if row['GBP Review Rating Offset fr. Avg'] else None,
|
||||
gbp_review_count_offset=float(row['GBP Review Count Offset fr. Avg']) if row['GBP Review Count Offset fr. Avg'] else None,
|
||||
gbp_review_rating_health=row['GBP Review Rating Health'],
|
||||
gbp_review_count_health=row['GBP Review Count Health'],
|
||||
gbp_location_municipality=row['GBP Location Municipality'],
|
||||
gbp_location_state=row['GBP Location State'],
|
||||
location_from_gbp_business_name=row['Location from GBP Business Name'],
|
||||
gbp_business_phone=row['GBP Business Phone'],
|
||||
gbp_business_website=row['GBP Business Website'],
|
||||
root_domain=row['Root Domain'],
|
||||
md5_for_scrape=row['MD5 for Scrape'],
|
||||
services_reputation_management=row['Services: Reputation Management'] == 'Yes',
|
||||
services_gbp_optimization=row['Services: GBP Optimization'] == 'Yes',
|
||||
services_needs_website=row['Services: Needs Website'] == 'Yes'
|
||||
)
|
||||
imported_count += 1
|
||||
except Exception as row_error:
|
||||
logger.error(f"Error importing row: {row}. Error: {str(row_error)}")
|
||||
messages.success(request, f'CSV file imported successfully. {imported_count} records imported or updated.')
|
||||
logger.info(f'CSV import completed. {imported_count} records imported or updated.')
|
||||
batch.append(ids_data)
|
||||
|
||||
if len(batch) >= batch_size:
|
||||
IDSData.objects.bulk_create(batch, ignore_conflicts=True)
|
||||
total_imported += len(batch)
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
IDSData.objects.bulk_create(batch, ignore_conflicts=True)
|
||||
total_imported += len(batch)
|
||||
|
||||
messages.success(request, f'CSV file imported successfully. {total_imported} records imported or updated.')
|
||||
logger.info(f'CSV import completed. {total_imported} records imported or updated.')
|
||||
except Exception as e:
|
||||
error_message = f'Error importing CSV file: {str(e)}'
|
||||
messages.error(request, error_message)
|
||||
|
|
|
|||
Loading…
Reference in a new issue