Redis 大 Key 分析利器：支持 TOP N、批量分析与从节点优先 - ToB企服应用市场:ToB评测及商务社交产业平台

# ./redis-find-big-key -addr 10.0.1.76:6379 -cluster-mode
Log file not specified, using default: /tmp/10.0.1.76:6379_20250222_043832.txt
Scanning keys from node: 10.0.1.76:6380 (slave)
Node: 10.0.1.76:6380
-------- Summary --------
Sampled 8 keys in the keyspace!
Total key length in bytes is 2.96 MB (avg len 379.43 KB)
Top biggest keys:
+------------------------------+--------+-----------+---------------------+
| Key | Type | Size | Number of elements |
+------------------------------+--------+-----------+---------------------+
| mysortedset_20250222043729:1 | zset | 739.6 KB | 8027 members |
| myhash_20250222043741:2 | hash | 648.12 KB | 9490 fields |
| mysortedset_20250222043741:1 | zset | 536.44 KB | 5608 members |
| myset_20250222043729:1 | set | 399.66 KB | 8027 members |
| myset_20250222043741:1 | set | 328.36 KB | 5608 members |
| myhash_20250222043729:2 | hash | 222.65 KB | 3917 fields |
| mylist_20250222043729:1 | list | 160.54 KB | 8027 items |
| mykey_20250222043729:2 | string | 73 bytes | 7 bytes (value len) |
+------------------------------+--------+-----------+---------------------+
Scanning keys from node: 10.0.1.202:6380 (slave)
Node: 10.0.1.202:6380
-------- Summary --------
Sampled 8 keys in the keyspace!
Total key length in bytes is 3.11 MB (avg len 398.23 KB)
Top biggest keys:
+------------------------------+--------+------------+---------------------+
| Key | Type | Size | Number of elements |
+------------------------------+--------+------------+---------------------+
| mysortedset_20250222043741:2 | zset | 1020.13 KB | 9490 members |
| myset_20250222043741:2 | set | 588.81 KB | 9490 members |
| myhash_20250222043729:1 | hash | 456.1 KB | 8027 fields |
| mysortedset_20250222043729:2 | zset | 404.5 KB | 3917 members |
| myhash_20250222043741:1 | hash | 335.79 KB | 5608 fields |
| myset_20250222043729:2 | set | 195.87 KB | 3917 members |
| mylist_20250222043741:2 | list | 184.55 KB | 9490 items |
| mykey_20250222043741:1 | string | 73 bytes | 7 bytes (value len) |
+------------------------------+--------+------------+---------------------+
Scanning keys from node: 10.0.1.147:6380 (slave)
Node: 10.0.1.147:6380
-------- Summary --------
Sampled 4 keys in the keyspace!
Total key length in bytes is 192.9 KB (avg len 48.22 KB)
Top biggest keys:
+-------------------------+--------+-----------+---------------------+
| Key | Type | Size | Number of elements |
+-------------------------+--------+-----------+---------------------+
| mylist_20250222043741:1 | list | 112.45 KB | 5608 items |
| mylist_20250222043729:2 | list | 80.31 KB | 3917 items |
| mykey_20250222043729:1 | string | 73 bytes | 7 bytes (value len) |
| mykey_20250222043741:2 | string | 73 bytes | 7 bytes (value len) |
+-------------------------+--------+-----------+---------------------+

复制代码

# wget https://github.com/slowtech/redis-find-big-key/releases/download/v1.0.0/redis-find-big-key-linux-amd64.tar.gz
# tar xvf redis-find-big-key-linux-amd64.tar.gz

复制代码

# wget https://github.com/slowtech/redis-find-big-key/archive/refs/tags/v1.0.0.tar.gz
# tar xvf v1.0.0.tar.gz
# cd redis-find-big-key-1.0.0
# go build

复制代码

# ./redis-find-big-key --help
Usage of ./redis-find-big-key:
-addr string
Redis server address in the format <hostname>:<port>
-cluster-mode
Enable cluster mode to get keys from all shards in the Redis cluster
-concurrency int
Maximum number of nodes to process concurrently (default 1)
-direct
Perform operation on the specified node. If not specified, the operation will default to executing on the slave node
-log-file string
Log file for saving progress and intermediate result
-master-yes
Execute even if the Redis role is master
-password string
Redis password
-samples uint
Samples for memory usage (default 5)
-skip-lazyfree-check
Skip check lazyfree-lazy-expire
-sleep float
Sleep duration (in seconds) after processing each batch
-tls
Enable TLS for Redis connection
-top int
Maximum number of biggest keys to display (default 100)

复制代码

./redis-find-big-key -addr 10.0.1.76:6379
Scanning keys from node: 10.0.1.202:6380 (slave)

复制代码

./redis-find-big-key -addr 10.0.1.76:6379 -cluster-mode

复制代码

./redis-find-big-key -addr 10.0.1.76:6379,10.0.1.202:6379,10.0.1.147:6379

复制代码

./redis-find-big-key -addr 10.0.1.76:6379 -direct -master-yes

复制代码

# ./redis-find-big-key -addr 10.0.1.76:6379 -top 1
Scanning keys from node: 10.0.1.202:6380 (slave)
...
Top biggest keys:
+------------------------------+------+------------+--------------------+
| Key | Type | Size | Number of elements |
+------------------------------+------+------------+--------------------+
| mysortedset_20250222043741:2 | zset | 1020.13 KB | 9490 members |
+------------------------------+------+------------+--------------------+
# redis-cli -h 10.0.1.76 -p 6379 -c MEMORY USAGE mysortedset_20250222043741:2
(integer) 1014242
# echo "scale=2; 1014242 / 1024" | bc
990.47

复制代码

# ./redis-find-big-key -addr 10.0.1.76:6379 -direct --master-yes -top 1 --skip-lazyfree-check
Scanning keys from node: 10.0.1.76:6379 (master)
...
Top biggest keys:
+------------------------------+------+-----------+--------------------+
| Key | Type | Size | Number of elements |
+------------------------------+------+-----------+--------------------+
| mysortedset_20250222043741:2 | zset | 990.47 KB | 9490 members |
+------------------------------+------+-----------+--------------------+

复制代码

/* Find big keys */
if (config.bigkeys) {
if (cliConnect(0) == REDIS_ERR) exit(1);
findBigKeys(0, 0);
}
/* Find large keys */
if (config.memkeys) {
if (cliConnect(0) == REDIS_ERR) exit(1);
findBigKeys(1, config.memkeys_samples);
}

复制代码

static void findBigKeys(int memkeys, unsigned memkeys_samples) {
...
// 通过 DBSIZE 命令获取 key 的总数量
total_keys = getDbSize();
/* Status message */
printf("\n# Scanning the entire keyspace to find biggest keys as well as\n");
printf("# average sizes per key type. You can use -i 0.1 to sleep 0.1 sec\n");
printf("# per 100 SCAN commands (not usually needed).\n\n");
/* SCAN loop */
do {
/* Calculate approximate percentage completion */
pct = 100 * (double)sampled/total_keys;
// 通过 SCAN 命令扫描 key
reply = sendScan(&it);
scan_loops++;
// 获取当前批次的 key 名称。
keys = reply->element[1];
...
// 使用 pipeline 技术批量发送 TYPE 命令，获取每个 key 的类型
getKeyTypes(types_dict, keys, types);
// 使用 pipeline 技术批量发送相应命令获取每个 key 的大小
getKeySizes(keys, types, sizes, memkeys, memkeys_samples);
// 逐个处理 key，更新统计信息
for(i=0;i<keys->elements;i++) {
typeinfo *type = types[i];
/* Skip keys that disappeared between SCAN and TYPE */
if(!type)
continue;
type->totalsize += sizes[i]; // 累计每个类型 key 的总大小
type->count++; // 累计每个类型 key 的数量
totlen += keys->element[i]->len; // 累计 key 的长度
sampled++; // 累计扫描的 key 的数量
// 如果当前 key 的大小超过该类型的最大值，则会更新该类型的最大键大小，并打印统计信息。
if(type->biggest<sizes[i]) {
if (type->biggest_key)
sdsfree(type->biggest_key);
type->biggest_key = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
...
printf(
"[%05.2f%%] Biggest %-6s found so far '%s' with %llu %s\n",
pct, type->name, type->biggest_key, sizes[i],
!memkeys? type->sizeunit: "bytes");
type->biggest = sizes[i];
}
// 每扫描 100 万个 key，还会输出当前进度和扫描的 key 数量。
if(sampled % 1000000 == 0) {
printf("[%05.2f%%] Sampled %llu keys so far\n", pct, sampled);
}
}
// 如果设置了 interval，则每执行 100 次 SCAN 命令，都会 sleep 一段时间。
if (config.interval && (scan_loops % 100) == 0) {
usleep(config.interval);
}
freeReplyObject(reply);
} while(force_cancel_loop == 0 && it != 0);
..
// 输出总的统计信息
printf("\n-------- summary -------\n\n");
if (force_cancel_loop) printf("[%05.2f%%] ", pct); // 如果循环被取消，则显示进度百分比
printf("Sampled %llu keys in the keyspace!\n", sampled); // 打印已经扫描的 key 的数量
printf("Total key length in bytes is %llu (avg len %.2f)\n\n",
totlen, totlen ? (double)totlen/sampled : 0); // 打印 key 名的总长度及平均长度
// 输出每种类型最大键的信息
di = dictGetIterator(types_dict);
while ((de = dictNext(di))) {
typeinfo *type = dictGetVal(de);
if(type->biggest_key) {
printf("Biggest %6s found '%s' has %llu %s\n", type->name, type->biggest_key,
type->biggest, !memkeys? type->sizeunit: "bytes");
} // type->name 是 key 的类型名称，type->biggest_key 是最大键的名称
} // type->biggest 是最大键的大小，!memkeys? type->sizeunit: "bytes" 是大小单位。
..
// 输出每种类型的统计信息
di = dictGetIterator(types_dict);
while ((de = dictNext(di))) {
typeinfo *type = dictGetVal(de);
printf("%llu %ss with %llu %s (%05.2f%% of keys, avg size %.2f)\n",
type->count, type->name, type->totalsize, !memkeys? type->sizeunit: "bytes",
sampled ? 100 * (double)type->count/sampled : 0,
type->count ? (double)type->totalsize/type->count : 0);
} // sampled ? 100 * (double)type->count/sampled : 0 是当前类型的 key 的数量在总扫描的 key 数量中的百分比。
..
exit(0);
}

复制代码