第92集JVMFullGC运维监控与优化实战

1. JVM Full GC运维监控概述

Full GC（Full Garbage Collection）是JVM垃圾回收的重要指标，频繁的Full GC会导致应用暂停、性能下降。本文将详细介绍FGC监控、GC调优、内存泄漏检测和JVM性能优化的完整解决方案，帮助运维人员有效管理JVM性能。

1.1 核心挑战

FGC监控: 实时监控Full GC频率和耗时
GC调优: 优化垃圾回收策略和参数
内存泄漏检测: 及时发现和定位内存泄漏
性能优化: 优化JVM性能和减少GC停顿
故障诊断: 快速定位GC相关问题

1.2 技术架构

FGC监控 → 数据采集 → 性能分析 → 告警通知 → 自动优化
   ↓        ↓         ↓         ↓         ↓
GC指标 → 监控代理 → 数据存储 → 告警引擎 → 调优脚本
   ↓        ↓         ↓         ↓         ↓
内存分析 → 堆转储 → 泄漏检测 → 自动修复 → 性能报告

2. FGC监控系统

2.1 Maven依赖配置

<!-- pom.xml -->
<dependencies>
    <!-- Spring Boot Web -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    
    <!-- Spring Boot Data Redis -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-data-redis</artifactId>
    </dependency>
    
    <!-- Micrometer监控 -->
    <dependency>
        <groupId>io.micrometer</groupId>
        <artifactId>micrometer-registry-prometheus</artifactId>
    </dependency>
    
    <!-- MyBatis Plus -->
    <dependency>
        <groupId>com.baomidou</groupId>
        <artifactId>mybatis-plus-boot-starter</artifactId>
        <version>3.5.2</version>
    </dependency>
</dependencies>

2.2 应用配置

# application.yml
server:
  port: 8080

spring:
  redis:
    host: localhost
    port: 6379
    database: 0

# FGC监控配置
fgc-monitor:
  collection-interval: 5000         # 采集间隔(毫秒)
  fgc-frequency-threshold: 10        # FGC频率告警阈值(次/分钟)
  fgc-duration-threshold: 1000      # FGC耗时告警阈值(毫秒)
  memory-leak-threshold: 80         # 内存泄漏告警阈值(%)
  heap-dump-enabled: true           # 启用堆转储
  gc-log-enabled: true              # 启用GC日志

3. FGC监控服务

3.1 FGC监控实体类

/**
 * FGC监控数据实体类
 */
@Data
@TableName("fgc_monitor_data")
public class FgcMonitorData {
    
    @TableId(type = IdType.AUTO)
    private Long id;                    // 主键ID
    
    private String hostname;             // 主机名
    
    private String ip;                   // IP地址
    
    private String jvmName;              // JVM名称
    
    private String gcName;               // GC名称
    
    private Long fgcCount;               // Full GC次数
    
    private Long fgcTime;                // Full GC总时间
    
    private Double avgFgcDuration;       // 平均FGC耗时
    
    private Double maxFgcDuration;        // 最大FGC耗时
    
    private Long heapUsedBefore;         // GC前堆内存使用
    
    private Long heapUsedAfter;          // GC后堆内存使用
    
    private Long heapSize;               // 堆内存大小
    
    private Double heapUsage;            // 堆内存使用率
    
    private Long oldGenUsed;             // 老年代使用量
    
    private Long oldGenSize;             // 老年代大小
    
    private Double oldGenUsage;          // 老年代使用率
    
    private Long youngGenUsed;           // 新生代使用量
    
    private Long youngGenSize;           // 新生代大小
    
    private Double youngGenUsage;        // 新生代使用率
    
    private String gcCause;              // GC原因
    
    private Date collectTime;            // 采集时间
    
    private Date createTime;             // 创建时间
}

/**
 * FGC告警记录实体类
 */
@Data
@TableName("fgc_alert_record")
public class FgcAlertRecord {
    
    @TableId(type = IdType.AUTO)
    private Long id;                    // 主键ID
    
    private String hostname;             // 主机名
    
    private String jvmName;              // JVM名称
    
    private String alertType;            // 告警类型
    
    private String alertLevel;            // 告警级别
    
    private String alertMessage;         // 告警消息
    
    private String alertStatus;          // 告警状态
    
    private Date alertTime;              // 告警时间
    
    private Date resolveTime;            // 解决时间
    
    private String resolveMessage;       // 解决说明
}

3.2 FGC监控服务

/**
 * FGC监控服务
 * 负责FGC数据的采集、存储和分析
 */
@Service
public class FgcMonitorService {

    @Autowired
    private FgcMonitorDataMapper fgcMonitorDataMapper;
    
    @Autowired
    private FgcAlertRecordMapper fgcAlertRecordMapper;
    
    @Autowired
    private RedisTemplate<String, Object> redisTemplate;
    
    @Autowired
    private AlertService alertService;

    /**
     * 采集FGC数据
     * 定期采集Full GC性能指标
     */
    @Scheduled(fixedRate = 5000) // 每5秒执行一次
    public void collectFgcData() {
        try {
            // 1. 获取FGC信息
            FgcInfo fgcInfo = getFgcInfo();
            
            // 2. 创建FGC监控数据
            FgcMonitorData monitorData = createFgcMonitorData(fgcInfo);
            
            // 3. 保存到数据库
            fgcMonitorDataMapper.insert(monitorData);
            
            // 4. 更新缓存
            updateFgcCache(monitorData);
            
            // 5. 检查FGC告警
            checkFgcAlert(monitorData);
            
            log.debug("采集FGC数据: hostname={}, fgcCount={}, avgDuration={}ms", 
                    monitorData.getHostname(), monitorData.getFgcCount(), monitorData.getAvgFgcDuration());
                    
        } catch (Exception e) {
            log.error("采集FGC数据失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 获取FGC信息
     */
    private FgcInfo getFgcInfo() {
        FgcInfo fgcInfo = new FgcInfo();
        
        try {
            // 获取GC信息
            List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
            
            long totalFgcCount = 0;
            long totalFgcTime = 0;
            double maxFgcDuration = 0;
            
            for (GarbageCollectorMXBean gcBean : gcBeans) {
                String gcName = gcBean.getName();
                
                // 检查是否为Full GC
                if (isFullGc(gcName)) {
                    long fgcCount = gcBean.getCollectionCount();
                    long fgcTime = gcBean.getCollectionTime();
                    
                    totalFgcCount += fgcCount;
                    totalFgcTime += fgcTime;
                    
                    // 计算平均耗时
                    if (fgcCount > 0) {
                        double avgDuration = (double) fgcTime / fgcCount;
                        maxFgcDuration = Math.max(maxFgcDuration, avgDuration);
                    }
                }
            }
            
            fgcInfo.setFgcCount(totalFgcCount);
            fgcInfo.setFgcTime(totalFgcTime);
            fgcInfo.setAvgFgcDuration(totalFgcCount > 0 ? (double) totalFgcTime / totalFgcCount : 0);
            fgcInfo.setMaxFgcDuration(maxFgcDuration);
            
            // 获取内存信息
            MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
            MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
            
            fgcInfo.setHeapUsedBefore(heapUsage.getUsed());
            fgcInfo.setHeapUsedAfter(heapUsage.getUsed());
            fgcInfo.setHeapSize(heapUsage.getMax());
            fgcInfo.setHeapUsage((double) heapUsage.getUsed() / heapUsage.getMax() * 100);
            
            // 获取各代内存信息
            List<MemoryPoolMXBean> memoryPools = ManagementFactory.getMemoryPoolMXBeans();
            for (MemoryPoolMXBean pool : memoryPools) {
                String poolName = pool.getName();
                MemoryUsage usage = pool.getUsage();
                
                if (poolName.contains("Old") || poolName.contains("Tenured")) {
                    fgcInfo.setOldGenUsed(usage.getUsed());
                    fgcInfo.setOldGenSize(usage.getMax());
                    fgcInfo.setOldGenUsage((double) usage.getUsed() / usage.getMax() * 100);
                } else if (poolName.contains("Eden") || poolName.contains("Survivor")) {
                    fgcInfo.setYoungGenUsed(usage.getUsed());
                    fgcInfo.setYoungGenSize(usage.getMax());
                    fgcInfo.setYoungGenUsage((double) usage.getUsed() / usage.getMax() * 100);
                }
            }
            
            // 获取GC原因
            fgcInfo.setGcCause(getGcCause());
            
        } catch (Exception e) {
            log.error("获取FGC信息失败: {}", e.getMessage(), e);
        }
        
        return fgcInfo;
    }

    /**
     * 判断是否为Full GC
     */
    private boolean isFullGc(String gcName) {
        // 常见的Full GC名称
        return gcName.contains("MarkSweepCompact") || 
               gcName.contains("ConcurrentMarkSweep") ||
               gcName.contains("G1 Old Generation") ||
               gcName.contains("PS MarkSweep") ||
               gcName.contains("ParOldGen");
    }

    /**
     * 获取GC原因
     */
    private String getGcCause() {
        try {
            // 通过JVM参数获取GC原因
            RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
            List<String> jvmArgs = runtimeBean.getInputArguments();
            
            for (String arg : jvmArgs) {
                if (arg.contains("GC")) {
                    return arg;
                }
            }
            
        } catch (Exception e) {
            log.error("获取GC原因失败: {}", e.getMessage(), e);
        }
        
        return "Unknown";
    }

    /**
     * 创建FGC监控数据
     */
    private FgcMonitorData createFgcMonitorData(FgcInfo fgcInfo) {
        FgcMonitorData monitorData = new FgcMonitorData();
        
        // 设置基本信息
        monitorData.setHostname(getHostname());
        monitorData.setIp(getLocalIpAddress());
        monitorData.setJvmName(getJvmName());
        monitorData.setCollectTime(new Date());
        monitorData.setCreateTime(new Date());
        
        // 设置FGC信息
        monitorData.setFgcCount(fgcInfo.getFgcCount());
        monitorData.setFgcTime(fgcInfo.getFgcTime());
        monitorData.setAvgFgcDuration(fgcInfo.getAvgFgcDuration());
        monitorData.setMaxFgcDuration(fgcInfo.getMaxFgcDuration());
        
        // 设置内存信息
        monitorData.setHeapUsedBefore(fgcInfo.getHeapUsedBefore());
        monitorData.setHeapUsedAfter(fgcInfo.getHeapUsedAfter());
        monitorData.setHeapSize(fgcInfo.getHeapSize());
        monitorData.setHeapUsage(fgcInfo.getHeapUsage());
        monitorData.setOldGenUsed(fgcInfo.getOldGenUsed());
        monitorData.setOldGenSize(fgcInfo.getOldGenSize());
        monitorData.setOldGenUsage(fgcInfo.getOldGenUsage());
        monitorData.setYoungGenUsed(fgcInfo.getYoungGenUsed());
        monitorData.setYoungGenSize(fgcInfo.getYoungGenSize());
        monitorData.setYoungGenUsage(fgcInfo.getYoungGenUsage());
        
        // 设置GC信息
        monitorData.setGcCause(fgcInfo.getGcCause());
        
        return monitorData;
    }

    /**
     * 更新FGC缓存
     */
    private void updateFgcCache(FgcMonitorData monitorData) {
        try {
            String cacheKey = "fgc:realtime:" + monitorData.getHostname();
            redisTemplate.opsForValue().set(cacheKey, monitorData, Duration.ofMinutes(5));
            
            // 更新历史数据缓存
            String historyKey = "fgc:history:" + monitorData.getHostname();
            redisTemplate.opsForList().leftPush(historyKey, monitorData);
            redisTemplate.opsForList().trim(historyKey, 0, 99);
            redisTemplate.expire(historyKey, Duration.ofHours(1));
            
        } catch (Exception e) {
            log.warn("更新FGC缓存失败: {}", e.getMessage());
        }
    }

    /**
     * 检查FGC告警
     */
    private void checkFgcAlert(FgcMonitorData monitorData) {
        try {
            String alertType = null;
            String alertLevel = null;
            String alertMessage = null;
            
            // 检查FGC频率告警
            if (monitorData.getFgcCount() > 10) {
                alertType = "FGC_FREQUENCY_HIGH";
                alertLevel = "WARNING";
                alertMessage = String.format("FGC频率过高: %d次", monitorData.getFgcCount());
            }
            
            // 检查FGC耗时告警
            if (monitorData.getAvgFgcDuration() > 1000) {
                alertType = "FGC_DURATION_HIGH";
                alertLevel = "CRITICAL";
                alertMessage = String.format("FGC耗时过长: %.2fms", monitorData.getAvgFgcDuration());
            }
            
            // 检查堆内存使用率告警
            if (monitorData.getHeapUsage() > 90) {
                alertType = "HEAP_USAGE_HIGH";
                alertLevel = "CRITICAL";
                alertMessage = String.format("堆内存使用率过高: %.2f%%", monitorData.getHeapUsage());
            }
            
            // 检查老年代使用率告警
            if (monitorData.getOldGenUsage() > 85) {
                alertType = "OLD_GEN_USAGE_HIGH";
                alertLevel = "WARNING";
                alertMessage = String.format("老年代使用率过高: %.2f%%", monitorData.getOldGenUsage());
            }
            
            // 发送告警
            if (alertType != null) {
                sendFgcAlert(monitorData, alertType, alertLevel, alertMessage);
            }
            
        } catch (Exception e) {
            log.error("检查FGC告警失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 发送FGC告警
     */
    private void sendFgcAlert(FgcMonitorData monitorData, String alertType, String alertLevel, String alertMessage) {
        try {
            // 检查是否已经发送过相同告警
            String alertKey = "fgc:alert:" + monitorData.getHostname() + ":" + alertType;
            Boolean hasAlert = redisTemplate.hasKey(alertKey);
            
            if (hasAlert == null || !hasAlert) {
                // 创建告警记录
                FgcAlertRecord alertRecord = new FgcAlertRecord();
                alertRecord.setHostname(monitorData.getHostname());
                alertRecord.setJvmName(monitorData.getJvmName());
                alertRecord.setAlertType(alertType);
                alertRecord.setAlertLevel(alertLevel);
                alertRecord.setAlertMessage(alertMessage);
                alertRecord.setAlertStatus("ACTIVE");
                alertRecord.setAlertTime(new Date());
                
                // 发送告警通知
                alertService.sendFgcAlert(alertRecord);
                
                // 设置告警缓存
                redisTemplate.opsForValue().set(alertKey, "1", Duration.ofMinutes(5));
                
                log.warn("发送FGC告警: hostname={}, type={}, level={}", 
                        monitorData.getHostname(), alertType, alertLevel);
            }
            
        } catch (Exception e) {
            log.error("发送FGC告警失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 获取实时FGC数据
     */
    public FgcMonitorData getRealTimeFgcData(String hostname) {
        String cacheKey = "fgc:realtime:" + hostname;
        return (FgcMonitorData) redisTemplate.opsForValue().get(cacheKey);
    }

    /**
     * 获取FGC历史数据
     */
    public List<FgcMonitorData> getFgcHistoryData(String hostname, Date startTime, Date endTime) {
        return fgcMonitorDataMapper.selectByHostnameAndTimeRange(hostname, startTime, endTime);
    }

    /**
     * 获取主机名
     */
    private String getHostname() {
        try {
            return InetAddress.getLocalHost().getHostName();
        } catch (UnknownHostException e) {
            return "unknown";
        }
    }

    /**
     * 获取本地IP地址
     */
    private String getLocalIpAddress() {
        try {
            return InetAddress.getLocalHost().getHostAddress();
        } catch (UnknownHostException e) {
            return "127.0.0.1";
        }
    }

    /**
     * 获取JVM名称
     */
    private String getJvmName() {
        try {
            RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
            return runtimeBean.getName();
        } catch (Exception e) {
            return "unknown";
        }
    }
}

/**
 * FGC信息实体类
 */
@Data
public class FgcInfo {
    private Long fgcCount;               // Full GC次数
    private Long fgcTime;                // Full GC总时间
    private Double avgFgcDuration;       // 平均FGC耗时
    private Double maxFgcDuration;        // 最大FGC耗时
    private Long heapUsedBefore;         // GC前堆内存使用
    private Long heapUsedAfter;          // GC后堆内存使用
    private Long heapSize;                // 堆内存大小
    private Double heapUsage;             // 堆内存使用率
    private Long oldGenUsed;              // 老年代使用量
    private Long oldGenSize;              // 老年代大小
    private Double oldGenUsage;           // 老年代使用率
    private Long youngGenUsed;            // 新生代使用量
    private Long youngGenSize;            // 新生代大小
    private Double youngGenUsage;         // 新生代使用率
    private String gcCause;               // GC原因
}

4. FGC优化服务

4.1 FGC优化服务

/**
 * FGC优化服务
 * 提供FGC性能优化和调优功能
 */
@Service
public class FgcOptimizationService {

    @Autowired
    private FgcMonitorService fgcMonitorService;
    
    @Autowired
    private AlertService alertService;

    /**
     * 自动FGC优化
     * 根据FGC性能情况自动进行优化
     */
    @Scheduled(fixedRate = 300000) // 每5分钟执行一次
    public void autoOptimizeFgc() {
        try {
            String hostname = getHostname();
            FgcMonitorData fgcData = fgcMonitorService.getRealTimeFgcData(hostname);
            
            if (fgcData == null) {
                return;
            }
            
            // 检查FGC频率
            if (fgcData.getFgcCount() > 20) {
                // FGC频率过高，执行紧急优化
                executeEmergencyOptimization(fgcData);
            } else if (fgcData.getFgcCount() > 10) {
                // FGC频率较高，执行预防性优化
                executePreventiveOptimization(fgcData);
            }
            
            // 检查FGC耗时
            if (fgcData.getAvgFgcDuration() > 2000) {
                // FGC耗时过长，执行耗时优化
                executeDurationOptimization(fgcData);
            }
            
        } catch (Exception e) {
            log.error("自动FGC优化失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 执行紧急优化
     */
    private void executeEmergencyOptimization(FgcMonitorData fgcData) {
        log.warn("执行紧急FGC优化: fgcCount={}, avgDuration={}ms", 
                fgcData.getFgcCount(), fgcData.getAvgFgcDuration());
        
        try {
            // 1. 强制GC
            forceGarbageCollection();
            
            // 2. 调整GC参数
            adjustGcParameters();
            
            // 3. 优化内存分配
            optimizeMemoryAllocation();
            
            // 4. 生成堆转储
            generateHeapDump();
            
            // 5. 发送优化通知
            sendOptimizationNotification(fgcData, "EMERGENCY_OPTIMIZATION");
            
        } catch (Exception e) {
            log.error("执行紧急优化失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 执行预防性优化
     */
    private void executePreventiveOptimization(FgcMonitorData fgcData) {
        log.info("执行预防性FGC优化: fgcCount={}, avgDuration={}ms", 
                fgcData.getFgcCount(), fgcData.getAvgFgcDuration());
        
        try {
            // 1. 优化GC策略
            optimizeGcStrategy();
            
            // 2. 调整内存参数
            adjustMemoryParameters();
            
            // 3. 优化对象生命周期
            optimizeObjectLifecycle();
            
            // 4. 发送优化通知
            sendOptimizationNotification(fgcData, "PREVENTIVE_OPTIMIZATION");
            
        } catch (Exception e) {
            log.error("执行预防性优化失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 执行耗时优化
     */
    private void executeDurationOptimization(FgcMonitorData fgcData) {
        log.info("执行FGC耗时优化: avgDuration={}ms", fgcData.getAvgFgcDuration());
        
        try {
            // 1. 优化GC算法
            optimizeGcAlgorithm();
            
            // 2. 调整GC线程数
            adjustGcThreads();
            
            // 3. 优化GC并发度
            optimizeGcConcurrency();
            
            // 4. 发送优化通知
            sendOptimizationNotification(fgcData, "DURATION_OPTIMIZATION");
            
        } catch (Exception e) {
            log.error("执行耗时优化失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 强制垃圾回收
     */
    private void forceGarbageCollection() {
        log.info("执行强制垃圾回收");
        
        try {
            // 建议JVM执行GC
            System.gc();
            
            // 等待GC完成
            Thread.sleep(1000);
            
            log.info("强制垃圾回收完成");
            
        } catch (Exception e) {
            log.error("强制垃圾回收失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 调整GC参数
     */
    private void adjustGcParameters() {
        log.info("调整GC参数");
        
        try {
            // 获取当前JVM参数
            RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
            List<String> jvmArgs = runtimeBean.getInputArguments();
            
            // 分析GC参数
            analyzeGcParameters(jvmArgs);
            
            // 提供优化建议
            provideGcOptimizationSuggestions();
            
        } catch (Exception e) {
            log.error("调整GC参数失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 分析GC参数
     */
    private void analyzeGcParameters(List<String> jvmArgs) {
        Map<String, String> gcParams = new HashMap<>();
        
        for (String arg : jvmArgs) {
            if (arg.startsWith("-XX:") && arg.contains("GC")) {
                String[] parts = arg.split("=", 2);
                if (parts.length == 2) {
                    gcParams.put(parts[0], parts[1]);
                } else {
                    gcParams.put(parts[0], "true");
                }
            }
        }
        
        log.info("当前GC参数: {}", gcParams);
    }

    /**
     * 提供GC优化建议
     */
    private void provideGcOptimizationSuggestions() {
        List<String> suggestions = new ArrayList<>();
        
        // GC算法建议
        suggestions.add("建议使用G1垃圾收集器: -XX:+UseG1GC");
        suggestions.add("建议设置最大GC暂停时间: -XX:MaxGCPauseMillis=200");
        suggestions.add("建议启用并行GC: -XX:+UseParallelGC");
        
        // 内存参数建议
        suggestions.add("建议设置合适的堆内存: -Xms2g -Xmx2g");
        suggestions.add("建议设置新生代大小: -XX:NewRatio=1");
        suggestions.add("建议设置Survivor比例: -XX:SurvivorRatio=8");
        
        log.info("GC优化建议: {}", suggestions);
    }

    /**
     * 优化内存分配
     */
    private void optimizeMemoryAllocation() {
        log.info("优化内存分配");
        
        try {
            // 优化对象分配
            optimizeObjectAllocation();
            
            // 优化内存对齐
            optimizeMemoryAlignment();
            
        } catch (Exception e) {
            log.error("优化内存分配失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 优化对象分配
     */
    private void optimizeObjectAllocation() {
        // 实现对象分配优化逻辑
        log.info("优化对象分配");
    }

    /**
     * 优化内存对齐
     */
    private void optimizeMemoryAlignment() {
        // 实现内存对齐优化逻辑
        log.info("优化内存对齐");
    }

    /**
     * 生成堆转储
     */
    private void generateHeapDump() {
        log.warn("生成堆转储文件");
        
        try {
            // 获取堆转储信息
            MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
            MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
            
            // 生成堆转储文件路径
            String dumpPath = "/tmp/heapdump_" + System.currentTimeMillis() + ".hprof";
            
            // 生成堆转储
            MBeanServer server = ManagementFactory.getPlatformMBeanServer();
            HotSpotDiagnosticMXBean hotspotMBean = ManagementFactory.newPlatformMXBeanProxy(
                    server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class);
            
            hotspotMBean.dumpHeap(dumpPath, true);
            
            log.warn("堆转储文件生成完成: {}", dumpPath);
            
            // 发送堆转储通知
            sendHeapDumpNotification(dumpPath);
            
        } catch (Exception e) {
            log.error("生成堆转储失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 发送堆转储通知
     */
    private void sendHeapDumpNotification(String dumpPath) {
        try {
            AlertMessage alert = new AlertMessage();
            alert.setType("HEAP_DUMP");
            alert.setLevel("WARNING");
            alert.setMessage("FGC频率过高，已生成堆转储文件: " + dumpPath);
            alert.setTimestamp(new Date());
            
            alertService.sendAlert(alert);
            
        } catch (Exception e) {
            log.error("发送堆转储通知失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 优化GC策略
     */
    private void optimizeGcStrategy() {
        log.info("优化GC策略");
        
        try {
            // 分析当前GC策略
            analyzeCurrentGcStrategy();
            
            // 提供GC策略建议
            provideGcStrategySuggestions();
            
        } catch (Exception e) {
            log.error("优化GC策略失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 分析当前GC策略
     */
    private void analyzeCurrentGcStrategy() {
        try {
            List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
            
            for (GarbageCollectorMXBean gcBean : gcBeans) {
                log.info("GC策略分析: name={}, count={}, time={}ms", 
                        gcBean.getName(), gcBean.getCollectionCount(), gcBean.getCollectionTime());
            }
            
        } catch (Exception e) {
            log.error("分析当前GC策略失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 提供GC策略建议
     */
    private void provideGcStrategySuggestions() {
        List<String> suggestions = new ArrayList<>();
        
        suggestions.add("建议使用G1GC替代CMS: -XX:+UseG1GC");
        suggestions.add("建议启用增量GC: -XX:+UseIncrementalGC");
        suggestions.add("建议优化GC触发条件: -XX:G1HeapRegionSize=16m");
        
        log.info("GC策略建议: {}", suggestions);
    }

    /**
     * 调整内存参数
     */
    private void adjustMemoryParameters() {
        log.info("调整内存参数");
        
        try {
            // 调整堆内存参数
            adjustHeapMemoryParameters();
            
            // 调整非堆内存参数
            adjustNonHeapMemoryParameters();
            
        } catch (Exception e) {
            log.error("调整内存参数失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 调整堆内存参数
     */
    private void adjustHeapMemoryParameters() {
        // 实现堆内存参数调整逻辑
        log.info("调整堆内存参数");
    }

    /**
     * 调整非堆内存参数
     */
    private void adjustNonHeapMemoryParameters() {
        // 实现非堆内存参数调整逻辑
        log.info("调整非堆内存参数");
    }

    /**
     * 优化对象生命周期
     */
    private void optimizeObjectLifecycle() {
        log.info("优化对象生命周期");
        
        try {
            // 优化对象创建
            optimizeObjectCreation();
            
            // 优化对象回收
            optimizeObjectReclamation();
            
        } catch (Exception e) {
            log.error("优化对象生命周期失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 优化对象创建
     */
    private void optimizeObjectCreation() {
        // 实现对象创建优化逻辑
        log.info("优化对象创建");
    }

    /**
     * 优化对象回收
     */
    private void optimizeObjectReclamation() {
        // 实现对象回收优化逻辑
        log.info("优化对象回收");
    }

    /**
     * 优化GC算法
     */
    private void optimizeGcAlgorithm() {
        log.info("优化GC算法");
        
        try {
            // 分析GC算法性能
            analyzeGcAlgorithmPerformance();
            
            // 提供GC算法建议
            provideGcAlgorithmSuggestions();
            
        } catch (Exception e) {
            log.error("优化GC算法失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 分析GC算法性能
     */
    private void analyzeGcAlgorithmPerformance() {
        try {
            List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
            
            for (GarbageCollectorMXBean gcBean : gcBeans) {
                long count = gcBean.getCollectionCount();
                long time = gcBean.getCollectionTime();
                double avgTime = count > 0 ? (double) time / count : 0;
                
                log.info("GC算法性能: name={}, avgTime={}ms", gcBean.getName(), avgTime);
            }
            
        } catch (Exception e) {
            log.error("分析GC算法性能失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 提供GC算法建议
     */
    private void provideGcAlgorithmSuggestions() {
        List<String> suggestions = new ArrayList<>();
        
        suggestions.add("建议使用G1GC: -XX:+UseG1GC -XX:MaxGCPauseMillis=200");
        suggestions.add("建议使用ZGC: -XX:+UnlockExperimentalVMOptions -XX:+UseZGC");
        suggestions.add("建议使用ShenandoahGC: -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC");
        
        log.info("GC算法建议: {}", suggestions);
    }

    /**
     * 调整GC线程数
     */
    private void adjustGcThreads() {
        log.info("调整GC线程数");
        
        try {
            // 获取CPU核心数
            int cpuCores = Runtime.getRuntime().availableProcessors();
            
            // 计算最优GC线程数
            int optimalGcThreads = calculateOptimalGcThreads(cpuCores);
            
            log.info("建议GC线程数: {}", optimalGcThreads);
            
        } catch (Exception e) {
            log.error("调整GC线程数失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 计算最优GC线程数
     */
    private int calculateOptimalGcThreads(int cpuCores) {
        // 根据CPU核心数计算最优GC线程数
        if (cpuCores <= 4) {
            return cpuCores;
        } else if (cpuCores <= 8) {
            return cpuCores - 1;
        } else {
            return cpuCores - 2;
        }
    }

    /**
     * 优化GC并发度
     */
    private void optimizeGcConcurrency() {
        log.info("优化GC并发度");
        
        try {
            // 优化GC并发参数
            optimizeGcConcurrencyParameters();
            
        } catch (Exception e) {
            log.error("优化GC并发度失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 优化GC并发参数
     */
    private void optimizeGcConcurrencyParameters() {
        List<String> suggestions = new ArrayList<>();
        
        suggestions.add("建议设置并发GC线程数: -XX:ConcGCThreads=4");
        suggestions.add("建议设置并行GC线程数: -XX:ParallelGCThreads=8");
        suggestions.add("建议启用并发标记: -XX:+CMSParallelRemarkEnabled");
        
        log.info("GC并发参数建议: {}", suggestions);
    }

    /**
     * 发送优化通知
     */
    private void sendOptimizationNotification(FgcMonitorData fgcData, String optimizationType) {
        try {
            AlertMessage alert = new AlertMessage();
            alert.setType("FGC_OPTIMIZATION");
            alert.setLevel("INFO");
            alert.setMessage(String.format("FGC优化完成: 类型=%s, FGC次数=%d, 平均耗时=%.2fms", 
                    optimizationType, fgcData.getFgcCount(), fgcData.getAvgFgcDuration()));
            alert.setTimestamp(new Date());
            
            alertService.sendAlert(alert);
            
        } catch (Exception e) {
            log.error("发送优化通知失败: {}", e.getMessage(), e);
        }
    }

    /**
     * 获取主机名
     */
    private String getHostname() {
        try {
            return InetAddress.getLocalHost().getHostName();
        } catch (UnknownHostException e) {
            return "unknown";
        }
    }
}

5. 总结

本文详细介绍了JVM Full GC运维监控与优化的完整解决方案，包括：

5.1 核心技术点

FGC监控: 实时监控Full GC频率、耗时、内存使用等指标
性能分析: 分析FGC性能趋势、内存使用模式
自动优化: 自动GC参数调整、内存优化、堆转储生成
故障诊断: 快速定位FGC性能问题和内存泄漏
告警通知: 多级告警、智能通知

5.2 架构优势

实时监控: 5秒间隔的实时FGC数据采集
智能告警: 基于阈值的智能告警机制
自动优化: 自动化的FGC性能优化
多维度分析: FGC频率、耗时、内存使用等多维度分析

5.3 最佳实践

监控策略: 设置合理的FGC监控阈值
优化策略: 根据FGC情况执行针对性优化
GC调优: 选择合适的GC算法和参数
预防措施: 提前预防FGC性能问题

通过以上架构设计，可以构建完善的FGC运维监控系统，实现JVM性能的有效管理和优化。