Advanced Performance Tuning

This guide covers advanced performance optimization techniques for experienced system administrators who need to squeeze maximum performance from their systems.

Kernel Tuning

Sysctl Optimization

# Apply advanced kernel settings
./performance-optimizer.sh --kernel-tune \
  --profile high-throughput \
  --apply-immediately

# Custom kernel parameters
./performance-optimizer.sh --set-sysctl \
  --params "net.core.somaxconn=65535,
           net.ipv4.tcp_max_syn_backlog=65535,
           net.core.netdev_max_backlog=65535"

Example Sysctl Configuration

# /etc/lambdasoftworks/performance/sysctl-high-performance.conf

# Network Stack
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.tcp_fin_timeout = 10
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10000 65535

# TCP Optimization
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_fastopen = 3
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_mtu_probing = 1

# TCP Memory
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 87380 16777216
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216

# File System & VM
vm.swappiness = 10
vm.dirty_ratio = 60
vm.dirty_background_ratio = 2
vm.vfs_cache_pressure = 50

Advanced Service Optimization

Nginx Advanced Configuration

# /etc/lambdasoftworks/performance/nginx-advanced.conf

# Worker Configuration
worker_processes auto;
worker_cpu_affinity auto;
worker_rlimit_nofile 65535;

events {
    worker_connections 65535;
    use epoll;
    multi_accept on;
}

http {
    # Buffering
    client_body_buffer_size 128k;
    client_max_body_size 10m;
    client_header_buffer_size 1k;
    large_client_header_buffers 4 8k;
    output_buffers 2 32k;
    postpone_output 1460;
    
    # Timeouts
    client_body_timeout 12;
    client_header_timeout 12;
    keepalive_timeout 15;
    keepalive_requests 100;
    send_timeout 10;
    
    # Compression
    gzip on;
    gzip_comp_level 5;
    gzip_min_length 256;
    gzip_proxied any;
    gzip_vary on;
    gzip_types
        application/javascript
        application/json
        application/x-javascript
        application/xml
        text/css
        text/javascript
        text/plain
        text/xml;
    
    # File Handling
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    directio 4m;
    aio threads;
    
    # SSL
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;
    ssl_session_tickets off;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers off;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256;
    
    # FastCGI
    fastcgi_buffers 256 16k;
    fastcgi_buffer_size 128k;
    fastcgi_connect_timeout 3s;
    fastcgi_send_timeout 120s;
    fastcgi_read_timeout 120s;
    fastcgi_busy_buffers_size 256k;
    fastcgi_temp_file_write_size 256k;
    reset_timedout_connection on;
}

MySQL/MariaDB Advanced Optimization

# /etc/lambdasoftworks/performance/mysql-advanced.cnf

[mysqld]
# InnoDB Settings
innodb_buffer_pool_size = 12G
innodb_buffer_pool_instances = 12
innodb_read_io_threads = 12
innodb_write_io_threads = 12
innodb_io_capacity = 5000
innodb_io_capacity_max = 10000
innodb_file_per_table = 1
innodb_flush_method = O_DIRECT
innodb_flush_neighbors = 0
innodb_log_file_size = 2G
innodb_log_buffer_size = 16M
innodb_purge_threads = 4
innodb_adaptive_hash_index = 0
innodb_random_read_ahead = 0
innodb_use_native_aio = 1

# Buffer Settings
key_buffer_size = 256M
sort_buffer_size = 4M
read_buffer_size = 3M
read_rnd_buffer_size = 4M
join_buffer_size = 4M
tmp_table_size = 64M
max_heap_table_size = 64M

# Connection Settings
max_connections = 1000
thread_cache_size = 128
table_open_cache = 4000
table_definition_cache = 2048
open_files_limit = 65535

# Query Cache (disabled for high-performance)
query_cache_type = 0
query_cache_size = 0

# Logging
slow_query_log = 1
long_query_time = 2
log_queries_not_using_indexes = 1

Advanced I/O Optimization

Storage Configuration

# Optimize disk layout
./performance-optimizer.sh --optimize-disk-layout \
  --separate-data \
  --raid-level 10 \
  --stripe-size 256K

# Configure I/O scheduler
./performance-optimizer.sh --io-scheduler \
  --scheduler none \
  --disk nvme0n1 \
  --queue-depth 128

File System Tuning

# XFS Optimization
./performance-optimizer.sh --tune-xfs \
  --mount-point /data \
  --log-size 256m \
  --inode-size 512

# EXT4 Optimization
./performance-optimizer.sh --tune-ext4 \
  --mount-point /data \
  --commit 30 \
  --journal-mode writeback

Memory Management

Huge Pages Configuration

# Configure huge pages
./performance-optimizer.sh --huge-pages \
  --size 2M \
  --count 2048 \
  --mount-point /dev/hugepages

# Application-specific huge pages
./performance-optimizer.sh --huge-pages-app \
  --application mysql \
  --size-gb 8

Memory Allocation

# Configure memory allocator
./performance-optimizer.sh --memory-allocator \
  --allocator jemalloc \
  --applications "nginx,php-fpm"

# Memory limits
./performance-optimizer.sh --memory-limits \
  --service nginx \
  --soft-limit 4G \
  --hard-limit 8G

Network Stack Optimization

TCP Stack Tuning

# Optimize TCP stack
./performance-optimizer.sh --tcp-tune \
  --congestion-control bbr \
  --memory high \
  --backlog 65535

# Network interface tuning
./performance-optimizer.sh --nic-tune \
  --interface eth0 \
  --ring-buffer max \
  --coalesce on

Example Network Configuration

# /etc/lambdasoftworks/performance/network-high-performance.yml
network:
  interfaces:
    eth0:
      ring_buffer:
        rx: 4096
        tx: 4096
      coalesce:
        adaptive_rx: on
        adaptive_tx: on
        stats_block_usecs: 1000000
      
  stack:
    tcp_mem: [131072, 262144, 524288]
    tcp_rmem: [4096, 87380, 16777216]
    tcp_wmem: [4096, 87380, 16777216]
    tcp_mtu_probing: 1
    tcp_congestion_control: bbr
    
  queues:
    tx: 2000
    rx: 2000
    
  interrupts:
    balance: rps
    cpu_mask: 0-7

Performance Monitoring

Advanced Metrics Collection

# Configure advanced monitoring
./performance-monitor.sh --advanced-metrics \
  --collectors all \
  --interval 10s \
  --retention 30d

# Custom metrics
./performance-monitor.sh --custom-metrics \
  --script /path/to/collector.sh \
  --interval 30s

Performance Alerts

# Configure advanced alerts
./performance-monitor.sh --advanced-alerts \
  --rules-file alerts.yml \
  --notification-channels "slack,email"

# Trend analysis
./performance-monitor.sh --trend-analysis \
  --metrics "cpu,memory,io" \
  --window 7d \
  --threshold-deviation 2

Troubleshooting Advanced Issues

Performance Profiling

# System-wide profiling
./performance-analyzer.sh --profile \
  --mode system-wide \
  --duration 5m \
  --stack-depth 20

# Application profiling
./performance-analyzer.sh --profile-app \
  --target nginx \
  --type cpu \
  --output flamegraph

Performance Debugging

# Debug performance issues
./performance-analyzer.sh --debug \
  --component all \
  --trace-events \
  --capture-stack

# Generate debug report
./performance-analyzer.sh --debug-report \
  --type comprehensive \
  --include-metrics \
  --format html

Best Practices

1. Testing

   • Always test in staging first
   • Document baseline performance
   • Validate changes under load

2. Monitoring

   • Use comprehensive monitoring
   • Set up trending analysis
   • Configure meaningful alerts

3. Documentation

   • Document all changes
   • Keep optimization history
   • Track performance impact

Next Steps

• Performance Benchmarking
• High Availability Setup
• Clustering Guide
• Troubleshooting Guide