前言

分库分表作为现代互联网应用中的重要数据库架构设计,广泛应用于高并发、大数据量的业务场景。通过合理的分库分表策略,能够有效解决单库单表的性能瓶颈,提高系统的可扩展性和可用性。本文从分库分表策略到选择方案,从基础实现到企业级方案,系统梳理分库分表的完整解决方案。

一、分库分表架构设计

1.1 分库分表整体架构

1.2 分库分表策略架构

二、分库策略实现

2.1 垂直分库

2.1.1 垂直分库设计

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
/**
* 垂直分库服务
*/
@Service
public class VerticalDatabaseService {

@Autowired
private UserDatabaseService userDatabaseService;

@Autowired
private OrderDatabaseService orderDatabaseService;

@Autowired
private ProductDatabaseService productDatabaseService;

@Autowired
private PaymentDatabaseService paymentDatabaseService;

/**
* 根据业务类型选择数据库
*/
public Object executeByBusinessType(String businessType, String sql, Object... params) {
switch (businessType) {
case "USER":
return userDatabaseService.execute(sql, params);
case "ORDER":
return orderDatabaseService.execute(sql, params);
case "PRODUCT":
return productDatabaseService.execute(sql, params);
case "PAYMENT":
return paymentDatabaseService.execute(sql, params);
default:
throw new IllegalArgumentException("不支持的业务类型: " + businessType);
}
}

/**
* 跨库事务处理
*/
@Transactional
public void executeCrossDatabaseTransaction(List<DatabaseOperation> operations) {
try {
// 按数据库分组操作
Map<String, List<DatabaseOperation>> groupedOperations =
operations.stream().collect(Collectors.groupingBy(DatabaseOperation::getDatabaseType));

// 执行各数据库操作
for (Map.Entry<String, List<DatabaseOperation>> entry : groupedOperations.entrySet()) {
String databaseType = entry.getKey();
List<DatabaseOperation> dbOperations = entry.getValue();

executeDatabaseOperations(databaseType, dbOperations);
}

} catch (Exception e) {
log.error("跨库事务执行失败", e);
throw new DatabaseException("跨库事务执行失败", e);
}
}

/**
* 执行数据库操作
*/
private void executeDatabaseOperations(String databaseType, List<DatabaseOperation> operations) {
switch (databaseType) {
case "USER":
userDatabaseService.executeOperations(operations);
break;
case "ORDER":
orderDatabaseService.executeOperations(operations);
break;
case "PRODUCT":
productDatabaseService.executeOperations(operations);
break;
case "PAYMENT":
paymentDatabaseService.executeOperations(operations);
break;
default:
throw new IllegalArgumentException("不支持的数据库类型: " + databaseType);
}
}
}

2.1.2 垂直分库配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# application.yml
spring:
datasource:
user:
url: jdbc:mysql://localhost:3306/user_db?useUnicode=true&characterEncoding=utf8
username: user_db_user
password: user_db_pass
driver-class-name: com.mysql.cj.jdbc.Driver

order:
url: jdbc:mysql://localhost:3306/order_db?useUnicode=true&characterEncoding=utf8
username: order_db_user
password: order_db_pass
driver-class-name: com.mysql.cj.jdbc.Driver

product:
url: jdbc:mysql://localhost:3306/product_db?useUnicode=true&characterEncoding=utf8
username: product_db_user
password: product_db_pass
driver-class-name: com.mysql.cj.jdbc.Driver

payment:
url: jdbc:mysql://localhost:3306/payment_db?useUnicode=true&characterEncoding=utf8
username: payment_db_user
password: payment_db_pass
driver-class-name: com.mysql.cj.jdbc.Driver

2.2 水平分库

2.2.1 水平分库服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
/**
* 水平分库服务
*/
@Service
public class HorizontalDatabaseService {

@Autowired
private DatabaseRouter databaseRouter;

@Autowired
private DatabaseConnectionManager connectionManager;

private final int DATABASE_COUNT = 4; // 分库数量

/**
* 根据分片键选择数据库
*/
public Object executeByShardKey(String shardKey, String sql, Object... params) {
try {
// 计算分片
int shardIndex = calculateShardIndex(shardKey);

// 获取数据库连接
DataSource dataSource = connectionManager.getDataSource(shardIndex);

// 执行SQL
return executeSQL(dataSource, sql, params);

} catch (Exception e) {
log.error("水平分库执行失败", e);
throw new DatabaseException("水平分库执行失败", e);
}
}

/**
* 计算分片索引
*/
private int calculateShardIndex(String shardKey) {
// 使用一致性哈希算法
return Math.abs(shardKey.hashCode()) % DATABASE_COUNT;
}

/**
* 批量执行SQL
*/
public List<Object> batchExecuteByShardKey(Map<String, String> shardKeySqlMap) {
// 按分片分组
Map<Integer, List<ShardOperation>> groupedOperations = new HashMap<>();

for (Map.Entry<String, String> entry : shardKeySqlMap.entrySet()) {
String shardKey = entry.getKey();
String sql = entry.getValue();

int shardIndex = calculateShardIndex(shardKey);
groupedOperations.computeIfAbsent(shardIndex, k -> new ArrayList<>())
.add(new ShardOperation(shardKey, sql));
}

// 并行执行各分片操作
List<Object> results = new ArrayList<>();
groupedOperations.entrySet().parallelStream().forEach(entry -> {
int shardIndex = entry.getKey();
List<ShardOperation> operations = entry.getValue();

try {
DataSource dataSource = connectionManager.getDataSource(shardIndex);
for (ShardOperation operation : operations) {
Object result = executeSQL(dataSource, operation.getSql());
results.add(result);
}
} catch (Exception e) {
log.error("分片{}执行失败", shardIndex, e);
}
});

return results;
}

/**
* 执行SQL
*/
private Object executeSQL(DataSource dataSource, String sql, Object... params) {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql)) {

// 设置参数
for (int i = 0; i < params.length; i++) {
statement.setObject(i + 1, params[i]);
}

// 执行查询
if (sql.trim().toUpperCase().startsWith("SELECT")) {
ResultSet resultSet = statement.executeQuery();
return convertResultSetToList(resultSet);
} else {
return statement.executeUpdate();
}

} catch (SQLException e) {
log.error("SQL执行失败", e);
throw new DatabaseException("SQL执行失败", e);
}
}

/**
* 转换ResultSet为List
*/
private List<Map<String, Object>> convertResultSetToList(ResultSet resultSet) throws SQLException {
List<Map<String, Object>> results = new ArrayList<>();
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();

while (resultSet.next()) {
Map<String, Object> row = new HashMap<>();
for (int i = 1; i <= columnCount; i++) {
String columnName = metaData.getColumnName(i);
Object value = resultSet.getObject(i);
row.put(columnName, value);
}
results.add(row);
}

return results;
}
}

2.2.2 数据库连接管理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
/**
* 数据库连接管理器
*/
@Component
public class DatabaseConnectionManager {

@Autowired
private DatabaseConfig databaseConfig;

private final Map<Integer, DataSource> dataSourceMap = new ConcurrentHashMap<>();

@PostConstruct
public void initDataSources() {
// 初始化各分片数据源
for (int i = 0; i < databaseConfig.getShardCount(); i++) {
DataSource dataSource = createDataSource(i);
dataSourceMap.put(i, dataSource);
}
}

/**
* 获取数据源
*/
public DataSource getDataSource(int shardIndex) {
DataSource dataSource = dataSourceMap.get(shardIndex);
if (dataSource == null) {
throw new DatabaseException("数据源不存在: " + shardIndex);
}
return dataSource;
}

/**
* 创建数据源
*/
private DataSource createDataSource(int shardIndex) {
HikariConfig config = new HikariConfig();

// 设置数据库连接信息
String url = String.format("jdbc:mysql://localhost:3306/shard_%d?useUnicode=true&characterEncoding=utf8", shardIndex);
config.setJdbcUrl(url);
config.setUsername("shard_user");
config.setPassword("shard_pass");
config.setDriverClassName("com.mysql.cj.jdbc.Driver");

// 设置连接池参数
config.setMaximumPoolSize(20);
config.setMinimumIdle(5);
config.setConnectionTimeout(30000);
config.setIdleTimeout(600000);
config.setMaxLifetime(1800000);
config.setLeakDetectionThreshold(60000);

return new HikariDataSource(config);
}

/**
* 关闭所有数据源
*/
@PreDestroy
public void closeAllDataSources() {
dataSourceMap.values().forEach(dataSource -> {
if (dataSource instanceof HikariDataSource) {
((HikariDataSource) dataSource).close();
}
});
}
}

三、分表策略实现

3.1 垂直分表

3.1.1 垂直分表服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
/**
* 垂直分表服务
*/
@Service
public class VerticalTableService {

@Autowired
private UserBasicTableService userBasicTableService;

@Autowired
private UserDetailTableService userDetailTableService;

@Autowired
private UserProfileTableService userProfileTableService;

/**
* 根据字段类型选择表
*/
public Object executeByFieldType(String fieldType, String sql, Object... params) {
switch (fieldType) {
case "BASIC":
return userBasicTableService.execute(sql, params);
case "DETAIL":
return userDetailTableService.execute(sql, params);
case "PROFILE":
return userProfileTableService.execute(sql, params);
default:
throw new IllegalArgumentException("不支持的字段类型: " + fieldType);
}
}

/**
* 跨表查询
*/
public List<Map<String, Object>> crossTableQuery(String userId) {
List<Map<String, Object>> results = new ArrayList<>();

try {
// 查询基础信息
List<Map<String, Object>> basicInfo = userBasicTableService.queryByUserId(userId);
if (!basicInfo.isEmpty()) {
results.addAll(basicInfo);
}

// 查询详细信息
List<Map<String, Object>> detailInfo = userDetailTableService.queryByUserId(userId);
if (!detailInfo.isEmpty()) {
results.addAll(detailInfo);
}

// 查询档案信息
List<Map<String, Object>> profileInfo = userProfileTableService.queryByUserId(userId);
if (!profileInfo.isEmpty()) {
results.addAll(profileInfo);
}

// 合并结果
return mergeResults(results);

} catch (Exception e) {
log.error("跨表查询失败", e);
throw new DatabaseException("跨表查询失败", e);
}
}

/**
* 合并查询结果
*/
private List<Map<String, Object>> mergeResults(List<Map<String, Object>> results) {
Map<String, Map<String, Object>> mergedMap = new HashMap<>();

for (Map<String, Object> result : results) {
String userId = result.get("user_id").toString();
mergedMap.computeIfAbsent(userId, k -> new HashMap<>()).putAll(result);
}

return new ArrayList<>(mergedMap.values());
}

/**
* 跨表事务处理
*/
@Transactional
public void executeCrossTableTransaction(String userId, Map<String, Object> basicData,
Map<String, Object> detailData, Map<String, Object> profileData) {
try {
// 更新基础信息表
if (basicData != null && !basicData.isEmpty()) {
userBasicTableService.updateByUserId(userId, basicData);
}

// 更新详细信息表
if (detailData != null && !detailData.isEmpty()) {
userDetailTableService.updateByUserId(userId, detailData);
}

// 更新档案信息表
if (profileData != null && !profileData.isEmpty()) {
userProfileTableService.updateByUserId(userId, profileData);
}

} catch (Exception e) {
log.error("跨表事务执行失败", e);
throw new DatabaseException("跨表事务执行失败", e);
}
}
}

3.2 水平分表

3.2.1 水平分表服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/**
* 水平分表服务
*/
@Service
public class HorizontalTableService {

@Autowired
private TableRouter tableRouter;

@Autowired
private DatabaseConnectionManager connectionManager;

private final int TABLE_COUNT = 8; // 分表数量

/**
* 根据分片键选择表
*/
public Object executeByShardKey(String shardKey, String sql, Object... params) {
try {
// 计算分片
int shardIndex = calculateShardIndex(shardKey);

// 获取表名
String tableName = getTableName(shardIndex);

// 替换SQL中的表名
String shardedSQL = replaceTableName(sql, tableName);

// 获取数据库连接
DataSource dataSource = connectionManager.getDataSource(0); // 假设单库多表

// 执行SQL
return executeSQL(dataSource, shardedSQL, params);

} catch (Exception e) {
log.error("水平分表执行失败", e);
throw new DatabaseException("水平分表执行失败", e);
}
}

/**
* 计算分片索引
*/
private int calculateShardIndex(String shardKey) {
// 使用取模算法
return Math.abs(shardKey.hashCode()) % TABLE_COUNT;
}

/**
* 获取表名
*/
private String getTableName(int shardIndex) {
return String.format("user_%d", shardIndex);
}

/**
* 替换SQL中的表名
*/
private String replaceTableName(String sql, String tableName) {
// 简单的表名替换,实际应用中需要更复杂的SQL解析
return sql.replaceAll("user", tableName);
}

/**
* 跨表查询
*/
public List<Map<String, Object>> crossTableQuery(String sql, Object... params) {
List<Map<String, Object>> allResults = new ArrayList<>();

// 并行查询所有分表
IntStream.range(0, TABLE_COUNT).parallel().forEach(shardIndex -> {
try {
String tableName = getTableName(shardIndex);
String shardedSQL = replaceTableName(sql, tableName);

DataSource dataSource = connectionManager.getDataSource(0);
List<Map<String, Object>> results = executeQuery(dataSource, shardedSQL, params);

synchronized (allResults) {
allResults.addAll(results);
}

} catch (Exception e) {
log.error("分表{}查询失败", shardIndex, e);
}
});

return allResults;
}

/**
* 执行查询
*/
private List<Map<String, Object>> executeQuery(DataSource dataSource, String sql, Object... params) {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql)) {

// 设置参数
for (int i = 0; i < params.length; i++) {
statement.setObject(i + 1, params[i]);
}

// 执行查询
ResultSet resultSet = statement.executeQuery();
return convertResultSetToList(resultSet);

} catch (SQLException e) {
log.error("查询执行失败", e);
throw new DatabaseException("查询执行失败", e);
}
}

/**
* 执行SQL
*/
private Object executeSQL(DataSource dataSource, String sql, Object... params) {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql)) {

// 设置参数
for (int i = 0; i < params.length; i++) {
statement.setObject(i + 1, params[i]);
}

// 执行SQL
if (sql.trim().toUpperCase().startsWith("SELECT")) {
ResultSet resultSet = statement.executeQuery();
return convertResultSetToList(resultSet);
} else {
return statement.executeUpdate();
}

} catch (SQLException e) {
log.error("SQL执行失败", e);
throw new DatabaseException("SQL执行失败", e);
}
}

/**
* 转换ResultSet为List
*/
private List<Map<String, Object>> convertResultSetToList(ResultSet resultSet) throws SQLException {
List<Map<String, Object>> results = new ArrayList<>();
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();

while (resultSet.next()) {
Map<String, Object> row = new HashMap<>();
for (int i = 1; i <= columnCount; i++) {
String columnName = metaData.getColumnName(i);
Object value = resultSet.getObject(i);
row.put(columnName, value);
}
results.add(row);
}

return results;
}
}

四、分库分表选择方案

4.1 选择决策树

4.1.1 分库分表选择决策

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
/**
* 分库分表选择决策服务
*/
@Service
public class ShardingSelectionService {

@Autowired
private DatabaseMetricsService metricsService;

@Autowired
private BusinessAnalysisService businessAnalysisService;

/**
* 分库分表选择决策
*/
public ShardingStrategy selectShardingStrategy(ShardingRequirement requirement) {
// 1. 分析业务特征
BusinessFeature feature = analyzeBusinessFeature(requirement);

// 2. 分析数据特征
DataFeature dataFeature = analyzeDataFeature(requirement);

// 3. 分析性能需求
PerformanceRequirement perfRequirement = analyzePerformanceRequirement(requirement);

// 4. 分析扩展需求
ScalabilityRequirement scalabilityRequirement = analyzeScalabilityRequirement(requirement);

// 5. 综合决策
return makeDecision(feature, dataFeature, perfRequirement, scalabilityRequirement);
}

/**
* 分析业务特征
*/
private BusinessFeature analyzeBusinessFeature(ShardingRequirement requirement) {
BusinessFeature feature = new BusinessFeature();

// 分析业务类型
feature.setBusinessType(requirement.getBusinessType());

// 分析访问模式
feature.setAccessPattern(analyzeAccessPattern(requirement));

// 分析事务特征
feature.setTransactionFeature(analyzeTransactionFeature(requirement));

// 分析查询特征
feature.setQueryFeature(analyzeQueryFeature(requirement));

return feature;
}

/**
* 分析数据特征
*/
private DataFeature analyzeDataFeature(ShardingRequirement requirement) {
DataFeature feature = new DataFeature();

// 分析数据量
feature.setDataVolume(requirement.getDataVolume());

// 分析数据增长
feature.setDataGrowth(requirement.getDataGrowth());

// 分析数据分布
feature.setDataDistribution(analyzeDataDistribution(requirement));

// 分析数据关联
feature.setDataRelation(analyzeDataRelation(requirement));

return feature;
}

/**
* 分析性能需求
*/
private PerformanceRequirement analyzePerformanceRequirement(ShardingRequirement requirement) {
PerformanceRequirement perfRequirement = new PerformanceRequirement();

// 分析QPS需求
perfRequirement.setQpsRequirement(requirement.getQpsRequirement());

// 分析响应时间需求
perfRequirement.setResponseTimeRequirement(requirement.getResponseTimeRequirement());

// 分析并发需求
perfRequirement.setConcurrencyRequirement(requirement.getConcurrencyRequirement());

return perfRequirement;
}

/**
* 分析扩展需求
*/
private ScalabilityRequirement analyzeScalabilityRequirement(ShardingRequirement requirement) {
ScalabilityRequirement scalabilityRequirement = new ScalabilityRequirement();

// 分析水平扩展需求
scalabilityRequirement.setHorizontalScaling(requirement.isHorizontalScaling());

// 分析垂直扩展需求
scalabilityRequirement.setVerticalScaling(requirement.isVerticalScaling());

// 分析未来扩展需求
scalabilityRequirement.setFutureScaling(requirement.getFutureScaling());

return scalabilityRequirement;
}

/**
* 综合决策
*/
private ShardingStrategy makeDecision(BusinessFeature feature, DataFeature dataFeature,
PerformanceRequirement perfRequirement, ScalabilityRequirement scalabilityRequirement) {

ShardingStrategy strategy = new ShardingStrategy();

// 决策逻辑
if (shouldUseVerticalSharding(feature, dataFeature)) {
strategy.setShardingType(ShardingType.VERTICAL);
strategy.setShardingLevel(ShardingLevel.DATABASE);
} else if (shouldUseHorizontalSharding(feature, dataFeature, perfRequirement)) {
strategy.setShardingType(ShardingType.HORIZONTAL);
strategy.setShardingLevel(determineShardingLevel(dataFeature, perfRequirement));
} else if (shouldUseHybridSharding(feature, dataFeature, perfRequirement, scalabilityRequirement)) {
strategy.setShardingType(ShardingType.HYBRID);
strategy.setShardingLevel(ShardingLevel.BOTH);
} else {
strategy.setShardingType(ShardingType.NONE);
strategy.setShardingLevel(ShardingLevel.NONE);
}

// 设置分片策略
strategy.setShardingStrategy(determineShardingStrategy(strategy, dataFeature));

// 设置分片数量
strategy.setShardCount(calculateShardCount(dataFeature, perfRequirement));

return strategy;
}

/**
* 判断是否使用垂直分片
*/
private boolean shouldUseVerticalSharding(BusinessFeature feature, DataFeature dataFeature) {
// 业务模块独立性强
if (feature.getBusinessType() == BusinessType.MODULAR) {
return true;
}

// 数据关联度低
if (dataFeature.getDataRelation() == DataRelation.LOW) {
return true;
}

// 表结构差异大
if (dataFeature.getTableStructureDifference() > 0.7) {
return true;
}

return false;
}

/**
* 判断是否使用水平分片
*/
private boolean shouldUseHorizontalSharding(BusinessFeature feature, DataFeature dataFeature,
PerformanceRequirement perfRequirement) {

// 数据量大
if (dataFeature.getDataVolume() > 1000000) { // 100万条记录
return true;
}

// QPS高
if (perfRequirement.getQpsRequirement() > 10000) { // 1万QPS
return true;
}

// 单表查询多
if (feature.getQueryFeature().getSingleTableQueryRatio() > 0.8) {
return true;
}

return false;
}

/**
* 判断是否使用混合分片
*/
private boolean shouldUseHybridSharding(BusinessFeature feature, DataFeature dataFeature,
PerformanceRequirement perfRequirement, ScalabilityRequirement scalabilityRequirement) {

// 需要水平扩展
if (scalabilityRequirement.isHorizontalScaling()) {
return true;
}

// 数据量大且业务复杂
if (dataFeature.getDataVolume() > 10000000 && feature.getBusinessType() == BusinessType.COMPLEX) {
return true;
}

return false;
}

/**
* 确定分片级别
*/
private ShardingLevel determineShardingLevel(DataFeature dataFeature, PerformanceRequirement perfRequirement) {
// 数据量大,使用分库
if (dataFeature.getDataVolume() > 10000000) {
return ShardingLevel.DATABASE;
}

// QPS高,使用分库
if (perfRequirement.getQpsRequirement() > 50000) {
return ShardingLevel.DATABASE;
}

// 否则使用分表
return ShardingLevel.TABLE;
}

/**
* 确定分片策略
*/
private ShardingStrategyType determineShardingStrategy(ShardingStrategy strategy, DataFeature dataFeature) {
// 数据分布均匀,使用取模
if (dataFeature.getDataDistribution() == DataDistribution.UNIFORM) {
return ShardingStrategyType.MODULO;
}

// 数据有时间特征,使用范围
if (dataFeature.hasTimeFeature()) {
return ShardingStrategyType.RANGE;
}

// 数据有地理特征,使用哈希
if (dataFeature.hasGeoFeature()) {
return ShardingStrategyType.HASH;
}

// 默认使用一致性哈希
return ShardingStrategyType.CONSISTENT_HASH;
}

/**
* 计算分片数量
*/
private int calculateShardCount(DataFeature dataFeature, PerformanceRequirement perfRequirement) {
// 基于数据量计算
int dataBasedCount = (int) Math.ceil(dataFeature.getDataVolume() / 1000000.0);

// 基于QPS计算
int qpsBasedCount = (int) Math.ceil(perfRequirement.getQpsRequirement() / 10000.0);

// 取较大值,但不超过16
return Math.min(Math.max(dataBasedCount, qpsBasedCount), 16);
}
}

4.2 分片策略选择

4.2.1 分片策略实现

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
/**
* 分片策略服务
*/
@Service
public class ShardingStrategyService {

/**
* 取模分片策略
*/
public int moduloSharding(String shardKey, int shardCount) {
return Math.abs(shardKey.hashCode()) % shardCount;
}

/**
* 范围分片策略
*/
public int rangeSharding(String shardKey, List<Range> ranges) {
for (int i = 0; i < ranges.size(); i++) {
Range range = ranges.get(i);
if (range.contains(shardKey)) {
return i;
}
}
throw new ShardingException("分片键不在任何范围内: " + shardKey);
}

/**
* 哈希分片策略
*/
public int hashSharding(String shardKey, int shardCount) {
return Math.abs(shardKey.hashCode()) % shardCount;
}

/**
* 一致性哈希分片策略
*/
public int consistentHashSharding(String shardKey, List<String> nodes) {
ConsistentHash<String> consistentHash = new ConsistentHash<>(nodes);
return consistentHash.get(shardKey);
}

/**
* 根据策略类型选择分片
*/
public int shard(String shardKey, ShardingStrategyType strategyType, Object strategyConfig) {
switch (strategyType) {
case MODULO:
return moduloSharding(shardKey, (Integer) strategyConfig);
case RANGE:
return rangeSharding(shardKey, (List<Range>) strategyConfig);
case HASH:
return hashSharding(shardKey, (Integer) strategyConfig);
case CONSISTENT_HASH:
return consistentHashSharding(shardKey, (List<String>) strategyConfig);
default:
throw new IllegalArgumentException("不支持的分片策略: " + strategyType);
}
}
}

五、企业级分库分表应用

5.1 电商分库分表方案

5.1.1 电商分库分表设计

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
/**
* 电商分库分表服务
*/
@Service
public class EcommerceShardingService {

@Autowired
private UserShardingService userShardingService;

@Autowired
private OrderShardingService orderShardingService;

@Autowired
private ProductShardingService productShardingService;

/**
* 用户分库分表
*/
public void shardUserData() {
// 用户数据按用户ID分片
// 分库:按用户ID取模,4个库
// 分表:按用户ID取模,8个表

ShardingConfig userConfig = new ShardingConfig();
userConfig.setShardingType(ShardingType.HORIZONTAL);
userConfig.setShardingLevel(ShardingLevel.BOTH);
userConfig.setDatabaseCount(4);
userConfig.setTableCount(8);
userConfig.setShardingKey("user_id");
userConfig.setShardingStrategy(ShardingStrategyType.MODULO);

userShardingService.configureSharding(userConfig);
}

/**
* 订单分库分表
*/
public void shardOrderData() {
// 订单数据按订单ID分片
// 分库:按订单ID取模,4个库
// 分表:按订单ID取模,16个表

ShardingConfig orderConfig = new ShardingConfig();
orderConfig.setShardingType(ShardingType.HORIZONTAL);
orderConfig.setShardingLevel(ShardingLevel.BOTH);
orderConfig.setDatabaseCount(4);
orderConfig.setTableCount(16);
orderConfig.setShardingKey("order_id");
orderConfig.setShardingStrategy(ShardingStrategyType.MODULO);

orderShardingService.configureSharding(orderConfig);
}

/**
* 商品分库分表
*/
public void shardProductData() {
// 商品数据按商品ID分片
// 分库:按商品ID取模,2个库
// 分表:按商品ID取模,8个表

ShardingConfig productConfig = new ShardingConfig();
productConfig.setShardingType(ShardingType.HORIZONTAL);
productConfig.setShardingLevel(ShardingLevel.BOTH);
productConfig.setDatabaseCount(2);
productConfig.setTableCount(8);
productConfig.setShardingKey("product_id");
productConfig.setShardingStrategy(ShardingStrategyType.MODULO);

productShardingService.configureSharding(productConfig);
}

/**
* 跨库事务处理
*/
@Transactional
public void processOrderTransaction(OrderData orderData) {
try {
// 1. 创建订单
orderShardingService.createOrder(orderData);

// 2. 更新用户信息
userShardingService.updateUserOrderCount(orderData.getUserId());

// 3. 更新商品库存
productShardingService.updateProductStock(orderData.getProductId(), orderData.getQuantity());

} catch (Exception e) {
log.error("订单事务处理失败", e);
throw new EcommerceException("订单事务处理失败", e);
}
}

/**
* 跨库查询
*/
public OrderDetailInfo getOrderDetailInfo(String orderId) {
try {
// 1. 查询订单信息
OrderData orderData = orderShardingService.getOrderById(orderId);

// 2. 查询用户信息
UserData userData = userShardingService.getUserById(orderData.getUserId());

// 3. 查询商品信息
ProductData productData = productShardingService.getProductById(orderData.getProductId());

// 4. 组装结果
OrderDetailInfo detailInfo = new OrderDetailInfo();
detailInfo.setOrderData(orderData);
detailInfo.setUserData(userData);
detailInfo.setProductData(productData);

return detailInfo;

} catch (Exception e) {
log.error("查询订单详情失败", e);
throw new EcommerceException("查询订单详情失败", e);
}
}
}

5.2 社交分库分表方案

5.2.1 社交分库分表设计

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
/**
* 社交分库分表服务
*/
@Service
public class SocialShardingService {

@Autowired
private UserShardingService userShardingService;

@Autowired
private PostShardingService postShardingService;

@Autowired
private CommentShardingService commentShardingService;

/**
* 用户分库分表
*/
public void shardUserData() {
// 用户数据按用户ID分片
// 分库:按用户ID取模,8个库
// 分表:按用户ID取模,16个表

ShardingConfig userConfig = new ShardingConfig();
userConfig.setShardingType(ShardingType.HORIZONTAL);
userConfig.setShardingLevel(ShardingLevel.BOTH);
userConfig.setDatabaseCount(8);
userConfig.setTableCount(16);
userConfig.setShardingKey("user_id");
userConfig.setShardingStrategy(ShardingStrategyType.MODULO);

userShardingService.configureSharding(userConfig);
}

/**
* 帖子分库分表
*/
public void shardPostData() {
// 帖子数据按用户ID分片
// 分库:按用户ID取模,8个库
// 分表:按时间范围分表,每月一张表

ShardingConfig postConfig = new ShardingConfig();
postConfig.setShardingType(ShardingType.HORIZONTAL);
postConfig.setShardingLevel(ShardingLevel.BOTH);
postConfig.setDatabaseCount(8);
postConfig.setTableCount(12); // 12个月
postConfig.setShardingKey("user_id");
postConfig.setTimeShardingKey("create_time");
postConfig.setShardingStrategy(ShardingStrategyType.MODULO);
postConfig.setTimeShardingStrategy(ShardingStrategyType.RANGE);

postShardingService.configureSharding(postConfig);
}

/**
* 评论分库分表
*/
public void shardCommentData() {
// 评论数据按帖子ID分片
// 分库:按帖子ID取模,4个库
// 分表:按帖子ID取模,8个表

ShardingConfig commentConfig = new ShardingConfig();
commentConfig.setShardingType(ShardingType.HORIZONTAL);
commentConfig.setShardingLevel(ShardingLevel.BOTH);
commentConfig.setDatabaseCount(4);
commentConfig.setTableCount(8);
commentConfig.setShardingKey("post_id");
commentConfig.setShardingStrategy(ShardingStrategyType.MODULO);

commentShardingService.configureSharding(commentConfig);
}

/**
* 获取用户动态
*/
public List<PostData> getUserPosts(String userId, int page, int size) {
try {
// 查询用户帖子
return postShardingService.getPostsByUserId(userId, page, size);

} catch (Exception e) {
log.error("查询用户动态失败", e);
throw new SocialException("查询用户动态失败", e);
}
}

/**
* 获取帖子评论
*/
public List<CommentData> getPostComments(String postId, int page, int size) {
try {
// 查询帖子评论
return commentShardingService.getCommentsByPostId(postId, page, size);

} catch (Exception e) {
log.error("查询帖子评论失败", e);
throw new SocialException("查询帖子评论失败", e);
}
}

/**
* 跨库查询用户信息
*/
public UserSocialInfo getUserSocialInfo(String userId) {
try {
// 1. 查询用户基础信息
UserData userData = userShardingService.getUserById(userId);

// 2. 查询用户帖子数量
int postCount = postShardingService.getPostCountByUserId(userId);

// 3. 查询用户评论数量
int commentCount = commentShardingService.getCommentCountByUserId(userId);

// 4. 组装结果
UserSocialInfo socialInfo = new UserSocialInfo();
socialInfo.setUserData(userData);
socialInfo.setPostCount(postCount);
socialInfo.setCommentCount(commentCount);

return socialInfo;

} catch (Exception e) {
log.error("查询用户社交信息失败", e);
throw new SocialException("查询用户社交信息失败", e);
}
}
}

六、性能优化与监控

6.1 性能优化

6.1.1 分库分表性能优化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
/**
* 分库分表性能优化服务
*/
@Service
public class ShardingPerformanceOptimizationService {

@Autowired
private RedisTemplate<String, Object> redisTemplate;

@Autowired
private CaffeineCache localCache;

private final String SHARDING_CACHE_PREFIX = "sharding_cache:";

/**
* 缓存分片结果
*/
public int getShardIndexWithCache(String shardKey, ShardingStrategyType strategyType, Object strategyConfig) {
String cacheKey = SHARDING_CACHE_PREFIX + strategyType + ":" + DigestUtil.md5Hex(shardKey + strategyConfig.toString());

// 1. 尝试从本地缓存获取
Integer cachedResult = (Integer) localCache.getIfPresent(cacheKey);
if (cachedResult != null) {
return cachedResult;
}

// 2. 从Redis获取
String redisCacheKey = "redis_cache:" + cacheKey;
Integer redisResult = (Integer) redisTemplate.opsForValue().get(redisCacheKey);
if (redisResult != null) {
localCache.put(cacheKey, redisResult);
return redisResult;
}

// 3. 计算分片
int shardIndex = calculateShardIndex(shardKey, strategyType, strategyConfig);

// 4. 写入缓存
localCache.put(cacheKey, shardIndex);
redisTemplate.opsForValue().set(redisCacheKey, shardIndex, Duration.ofHours(1));

return shardIndex;
}

/**
* 批量分片优化
*/
public Map<String, Integer> batchSharding(Map<String, String> shardKeyMap, ShardingStrategyType strategyType, Object strategyConfig) {
Map<String, Integer> shardResults = new HashMap<>();

// 使用并行流处理
shardKeyMap.entrySet().parallelStream().forEach(entry -> {
String key = entry.getKey();
String shardKey = entry.getValue();

try {
int shardIndex = getShardIndexWithCache(shardKey, strategyType, strategyConfig);
shardResults.put(key, shardIndex);
} catch (Exception e) {
log.error("批量分片失败: {}", key, e);
shardResults.put(key, -1);
}
});

return shardResults;
}

/**
* 预热分片缓存
*/
@PostConstruct
public void warmupShardingCache() {
// 预热常用分片键
List<String> commonShardKeys = getCommonShardKeys();
ShardingStrategyType defaultStrategy = ShardingStrategyType.MODULO;
Object defaultConfig = 8; // 8个分片

for (String shardKey : commonShardKeys) {
try {
getShardIndexWithCache(shardKey, defaultStrategy, defaultConfig);
} catch (Exception e) {
log.error("预热分片缓存失败", e);
}
}
}

/**
* 清理过期缓存
*/
@Scheduled(fixedRate = 300000) // 5分钟
public void cleanupExpiredCache() {
// 清理本地缓存
localCache.cleanUp();

// 清理Redis过期缓存
cleanupRedisExpiredCache();
}

/**
* 计算分片索引
*/
private int calculateShardIndex(String shardKey, ShardingStrategyType strategyType, Object strategyConfig) {
switch (strategyType) {
case MODULO:
return Math.abs(shardKey.hashCode()) % (Integer) strategyConfig;
case HASH:
return Math.abs(shardKey.hashCode()) % (Integer) strategyConfig;
case CONSISTENT_HASH:
return Math.abs(shardKey.hashCode()) % (Integer) strategyConfig;
default:
return Math.abs(shardKey.hashCode()) % (Integer) strategyConfig;
}
}

/**
* 获取常用分片键
*/
private List<String> getCommonShardKeys() {
return Arrays.asList("user_1", "user_2", "user_3", "order_1", "order_2", "product_1");
}

/**
* 清理Redis过期缓存
*/
private void cleanupRedisExpiredCache() {
try {
Set<String> cacheKeys = redisTemplate.keys("redis_cache:" + SHARDING_CACHE_PREFIX + "*");

for (String key : cacheKeys) {
Long ttl = redisTemplate.getExpire(key);
if (ttl != null && ttl <= 0) {
redisTemplate.delete(key);
}
}
} catch (Exception e) {
log.error("清理Redis过期缓存失败", e);
}
}
}

6.2 监控告警

6.2.1 监控指标

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
/**
* 分库分表监控指标
*/
@Component
public class ShardingMetrics {

private final MeterRegistry meterRegistry;

public ShardingMetrics(MeterRegistry meterRegistry) {
this.meterRegistry = meterRegistry;
}

/**
* 记录分片查询
*/
public void recordShardQuery(String shardType, String tableName) {
Counter.builder("sharding.query.count")
.description("分片查询次数")
.tag("shard_type", shardType)
.tag("table_name", tableName)
.register(meterRegistry)
.increment();
}

/**
* 记录分片更新
*/
public void recordShardUpdate(String shardType, String tableName) {
Counter.builder("sharding.update.count")
.description("分片更新次数")
.tag("shard_type", shardType)
.tag("table_name", tableName)
.register(meterRegistry)
.increment();
}

/**
* 记录分片查询响应时间
*/
public void recordShardQueryResponseTime(String shardType, String tableName, long duration) {
Timer.builder("sharding.query.response.time")
.description("分片查询响应时间")
.tag("shard_type", shardType)
.tag("table_name", tableName)
.register(meterRegistry)
.record(duration, TimeUnit.MILLISECONDS);
}

/**
* 记录分片更新响应时间
*/
public void recordShardUpdateResponseTime(String shardType, String tableName, long duration) {
Timer.builder("sharding.update.response.time")
.description("分片更新响应时间")
.tag("shard_type", shardType)
.tag("table_name", tableName)
.register(meterRegistry)
.record(duration, TimeUnit.MILLISECONDS);
}

/**
* 记录分片数据量
*/
public void recordShardDataSize(String shardType, String tableName, long size) {
Gauge.builder("sharding.data.size")
.description("分片数据量")
.tag("shard_type", shardType)
.tag("table_name", tableName)
.register(meterRegistry, size);
}

/**
* 记录跨库查询
*/
public void recordCrossDatabaseQuery(String queryType) {
Counter.builder("sharding.cross.database.query.count")
.description("跨库查询次数")
.tag("query_type", queryType)
.register(meterRegistry)
.increment();
}

/**
* 记录跨表查询
*/
public void recordCrossTableQuery(String queryType) {
Counter.builder("sharding.cross.table.query.count")
.description("跨表查询次数")
.tag("query_type", queryType)
.register(meterRegistry)
.increment();
}
}

6.2.2 告警规则

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
# prometheus-rules.yml
groups:
- name: sharding_alerts
rules:
- alert: HighShardingQueryResponseTime
expr: sharding_query_response_time{quantile="0.95"} > 2000
for: 2m
labels:
severity: warning
annotations:
summary: "分片查询响应时间过长"
description: "分片查询响应时间P95超过2秒,当前值: {{ $value }}ms"

- alert: HighShardingUpdateResponseTime
expr: sharding_update_response_time{quantile="0.95"} > 1000
for: 2m
labels:
severity: warning
annotations:
summary: "分片更新响应时间过长"
description: "分片更新响应时间P95超过1秒,当前值: {{ $value }}ms"

- alert: HighCrossDatabaseQueryRate
expr: rate(sharding_cross_database_query_count[5m]) > 100
for: 2m
labels:
severity: warning
annotations:
summary: "跨库查询频率过高"
description: "跨库查询频率超过100次/分钟,当前值: {{ $value }}"

- alert: HighCrossTableQueryRate
expr: rate(sharding_cross_table_query_count[5m]) > 500
for: 2m
labels:
severity: warning
annotations:
summary: "跨表查询频率过高"
description: "跨表查询频率超过500次/分钟,当前值: {{ $value }}"

- alert: ShardDataSizeTooLarge
expr: sharding_data_size > 10000000
for: 5m
labels:
severity: warning
annotations:
summary: "分片数据量过大"
description: "分片数据量超过1000万,当前值: {{ $value }}"

七、总结

分库分表作为现代互联网应用中的重要数据库架构设计,通过合理的分库分表策略,能够有效解决单库单表的性能瓶颈。本文从分库分表策略到选择方案,从基础实现到企业级应用,系统梳理了分库分表的完整解决方案。

7.1 关键要点

  1. 策略选择:垂直分库、水平分库、垂直分表、水平分表各有特点,需要根据业务场景选择合适的策略
  2. 分片策略:取模、范围、哈希、一致性哈希等分片策略需要根据数据特征选择
  3. 性能优化:通过缓存、批量处理等手段提高分库分表性能
  4. 监控告警:建立完善的监控体系,及时发现和处理问题
  5. 扩展性:分库分表方案需要考虑未来的扩展需求

7.2 最佳实践

  1. 策略选择:根据业务特征、数据特征、性能需求选择合适的分库分表策略
  2. 分片设计:合理设计分片键和分片策略,避免数据倾斜
  3. 性能优化:使用缓存和批量处理提高分库分表性能
  4. 监控告警:建立完善的监控体系,确保系统稳定运行
  5. 扩展规划:提前规划扩展方案,支持业务增长

通过以上措施,可以构建一个高效、稳定、可扩展的分库分表系统,为企业的各种业务场景提供数据存储支持。