Batch Processing Large Datasets in Odoo: Without Crashing Your System

# ❌ DON'T DO THIS
orders = self.env['sale.order'].search([])  # Load ALL 50,000
for order in orders:
    order.process_order()  # Memory = 250GB

# Result: CRASH

# ✅ DO THIS
batch_size = 1000
offset = 0

while True:
    orders = self.env['sale.order'].search(
        [],
        offset=offset,
        limit=batch_size
    )
    
    if not orders:
        break
    
    for order in orders:
        order.process_order()
    
    # Memory released
    offset += batch_size

# Result: SUCCESS

# Formula: batch_size = 5,000 / (record_complexity)
# record_complexity = number of related records per order

# Simple record (order with 5 lines):
# Complexity = 5
# Batch size = 5,000 / 5 = 1,000 records per batch

# Complex record (order with 50 lines, 30 taxes, 10 documents):
# Complexity = 90
# Batch size = 5,000 / 90 ≈ 50 records per batch

# ❌ WRONG
def import_orders(self):
    """Crashes on large imports."""
    data = read_csv_file('/tmp/orders.csv')  # Load all 100K
    
    for row in data:
        self.env['sale.order'].create({
            'partner_id': row['customer_id'],
            'amount_total': row['amount'],
            ...
        })
    
    return "Imported %d orders" % len(data)

from itertools import islice

def import_orders(self):
    """Safely imports large CSV."""
    batch_size = 1000
    imported = 0
    failed = 0
    
    try:
        data = read_csv_file('/tmp/orders.csv')
        
        # Process in batches
        for batch in self._batch_generator(data, batch_size):
            try:
                # Create batch
                vals_list = [
                    {
                        'partner_id': row['customer_id'],
                        'amount_total': float(row['amount']),
                        'order_date': row['date'],
                    }
                    for row in batch
                ]
                
                # Batch create (more efficient than loop)
                self.env['sale.order'].create(vals_list)
                imported += len(vals_list)
                
            except Exception as e:
                # Log batch error, continue
                failed += len(batch)
                self.env.cr.rollback()
                _logger.error(f"Batch failed: {e}")
                continue
        
        return f"Imported: {imported}, Failed: {failed}"
    
    finally:
        # Always clean up
        self.env.cr.commit()

def _batch_generator(self, iterable, batch_size):
    """Yield successive batches from iterable."""
    iterator = iter(iterable)
    while True:
        batch = list(islice(iterator, batch_size))
        if not batch:
            break
        yield batch

# ❌ WRONG - Takes 10 minutes, times out
def generate_sales_report(self):
    """Generates report for all orders."""
    # Load all 500K lines at once
    lines = self.env['sale.order.line'].search([
        ('order_id.state', '=', 'done'),
    ])
    
    report_data = []
    for line in lines:
        report_data.append({
            'product': line.product_id.name,
            'qty': line.product_qty,
            'revenue': line.price_total,
        })
    
    return report_data

# ✅ RIGHT - Uses aggregation, completes in 10 seconds
def generate_sales_report(self):
    """Generates report efficiently."""
    # Use read_group instead of loading all records
    data = self.env['sale.order.line'].read_group(
        domain=[('order_id.state', '=', 'done')],
        fields=[
            'product_id',
            'product_qty:sum',
            'price_total:sum'
        ],
        groupby=['product_id']
    )
    
    # Transform to report format
    report_data = [
        {
            'product': d['product_id'][1],
            'qty': d['product_qty'],
            'revenue': d['price_total'],
        }
        for d in data
    ]
    
    return report_data
    # Result: 10 seconds, minimal memory

pip install odoo-bin-addons-queue-job

from odoo_bin_addons import queue_job

class MailCampaign(models.Model):
    _name = 'mail.campaign'
    
    def action_send_emails(self):
        """Send emails to all subscribers (using background jobs)."""
        subscribers = self.env['res.partner'].search([
            ('email', '!=', False),
            ('opt_in', '=', True),
        ])
        
        # Enqueue job for each batch
        batch_size = 500
        for i in range(0, len(subscribers), batch_size):
            batch = subscribers[i:i+batch_size]
            # Send as background job
            self.with_delay(priority=10).send_batch_emails(batch)
        
        return "Enqueued %d emails for sending" % len(subscribers)
    
    @job
    def send_batch_emails(self, subscribers):
        """Send emails to a batch of subscribers."""
        template = self.env.ref('mail.email_template_campaign')
        
        for subscriber in subscribers:
            try:
                # Send email
                template.send_mail(
                    subscriber.id,
                    force_send=True
                )
            except Exception as e:
                _logger.error(f"Failed to email {subscriber.email}: {e}")
                # Continue with next subscriber
                continue

def process_large_dataset(self):
    """Process with explicit memory management."""
    batch_size = 1000
    offset = 0
    
    while True:
        # Fetch batch
        records = self.env['sale.order'].search(
            [],
            offset=offset,
            limit=batch_size
        )
        
        if not records:
            break
        
        # Process batch
        for record in records:
            record.action_process()
        
        # CRITICAL: Clear caches to free memory
        self.env.clear()
        
        # Clean up local variables
        del records
        
        offset += batch_size

import psutil
import os

def process_with_monitoring(self):
    """Process batch with memory alerts."""
    process = psutil.Process(os.getpid())
    max_memory = 1024 * 1024 * 1024  # 1GB alert threshold
    batch_size = 1000
    offset = 0
    
    while True:
        records = self.env['sale.order'].search(
            [],
            offset=offset,
            limit=batch_size
        )
        
        if not records:
            break
        
        # Check memory
        memory_used = process.memory_info().rss
        if memory_used > max_memory:
            _logger.warning(f"High memory: {memory_used / 1024 / 1024}MB")
        
        # Process
        for record in records:
            record.action_process()
        
        self.env.clear()
        offset += batch_size