11.2-STABLE kernel wired memory leak
Karl Denninger
karl at denninger.net
Wed Feb 13 13:28:35 UTC 2019
On 2/13/2019 03:59, Andriy Gapon wrote:
> On 12/02/2019 20:17, Eugene Grosbein wrote:
>> 13.02.2019 1:14, Eugene Grosbein wrote:
>>
>>> Use following command to see how much memory is wasted in your case:
>>>
>>> vmstat -z | awk -F, '{printf "%10s %s\n", $2*$5/1024/1024, $1}' | sort -k1,1 -rn | head
>> Oops, small correction:
>>
>> vmstat -z | sed 's/:/,/' | awk -F, '{printf "%10s %s\n", $2*$5/1024/1024, $1}' | sort -k1,1 -rn | head
> I have a much uglier but somewhat more informative "one-liner" for
> post-processing vmstat -z output:
>
> vmstat -z | tail +3 | awk -F '[:,] *' 'BEGIN { total=0; cache=0; used=0 } {u =
> $2 * $4; c = $2 * $5; t = u + c; cache += c; used += u; total += t; name=$1;
> gsub(" ", "_", name); print t, name, u, c} END { print total, "TOTAL", used,
> cache } ' | sort -n | perl -a -p -e 'while (($j, $_) = each(@F)) { 1 while
> s/^(-?\d+)(\d{3})/$1,$2/; print $_, " "} print "\n"' | column -t
>
> This would be much nicer as a small python script.
> Or, even, we could add a sort option for vmstat -z / -m.
On 12-STABLE, r343809, /without /my patch set (and I've yet to see the
sort of stall problems I used to have regularly with 11.1)
root at NewFS:/disk/karl # sh vmstat.script
0 0 0
0 AIO 0 0
0 AIOCB 0 0
0 AIOLIO 0 0
0 AIOP 0 0
0 DMAR_MAP_ENTRY 0 0
0 FFS1_dinode 0 0
0 FPU_save_area 0 0
0 IPFW_IPv6_dynamic_states 0 0
0 IPFW_parent_dynamic_states 0 0
0 IPsec_SA_lft_c 0 0
0 LTS_VFS_Cache 0 0
0 MAC_labels 0 0
0 NCLNODE 0 0
0 STS_VFS_Cache 0 0
0 arc_buf_hdr_t_l2only 0 0
0 audit_record 0 0
0 cryptodesc 0 0
0 cryptop 0 0
0 domainset 0 0
0 itimer 0 0
0 mbuf_jumbo_16k 0 0
0 mbuf_jumbo_9k 0 0
0 nvme_request 0 0
0 procdesc 0 0
0 rentr 0 0
0 sctp_asconf 0 0
0 sctp_asconf_ack 0 0
0 sctp_asoc 0 0
0 sctp_chunk 0 0
0 sctp_ep 0 0
0 sctp_raddr 0 0
0 sctp_readq 0 0
0 sctp_stream_msg_out 0 0
0 sio_cache 0 0
0 tcp_log 0 0
0 tcp_log_bucket 0 0
0 tcp_log_node 0 0
0 tfo 0 0
0 tfo_ccache_entries 0 0
0 udplite_inpcb 0 0
0 umtx_pi 0 0
0 vtnet_tx_hdr 0 0
0 zio_buf_10485760 0 0
0 zio_buf_12582912 0 0
0 zio_buf_14680064 0 0
0 zio_buf_1572864 0 0
0 zio_buf_16777216 0 0
0 zio_buf_1835008 0 0
0 zio_buf_2097152 0 0
0 zio_buf_2621440 0 0
0 zio_buf_3145728 0 0
0 zio_buf_3670016 0 0
0 zio_buf_4194304 0 0
0 zio_buf_5242880 0 0
0 zio_buf_6291456 0 0
0 zio_buf_7340032 0 0
0 zio_buf_8388608 0 0
0 zio_data_buf_1048576 0 0
0 zio_data_buf_10485760 0 0
0 zio_data_buf_12582912 0 0
0 zio_data_buf_1310720 0 0
0 zio_data_buf_14680064 0 0
0 zio_data_buf_1572864 0 0
0 zio_data_buf_163840 0 0
0 zio_data_buf_16777216 0 0
0 zio_data_buf_1835008 0 0
0 zio_data_buf_196608 0 0
0 zio_data_buf_2097152 0 0
0 zio_data_buf_229376 0 0
0 zio_data_buf_262144 0 0
0 zio_data_buf_2621440 0 0
0 zio_data_buf_3145728 0 0
0 zio_data_buf_327680 0 0
0 zio_data_buf_3670016 0 0
0 zio_data_buf_393216 0 0
0 zio_data_buf_4194304 0 0
0 zio_data_buf_458752 0 0
0 zio_data_buf_524288 0 0
0 zio_data_buf_5242880 0 0
0 zio_data_buf_6291456 0 0
0 zio_data_buf_655360 0 0
0 zio_data_buf_7340032 0 0
0 zio_data_buf_786432 0 0
0 zio_data_buf_8388608 0 0
0 zio_data_buf_917504 0 0
7,168 vmem 5,376 1,792
15,776 MAP 696 15,080
16,384 epoch_record_pcpu 1,024 15,360
20,000 sackhole 0 20,000
20,040 mt_zone 10,512 9,528
26,880 UMA_Hash 13,312 13,568
27,888 tcpreass 0 27,888
31,808 ipq 0 31,808
40,960 64_pcpu 26,040 14,920
49,152 IPFW_counters 992 48,160
49,152 mt_stats_zone 28,032 21,120
51,792 sctp_laddr 0 51,792
52,104 crypto_session 1,344 50,760
60,928 UMA_Kegs 57,568 3,360
60,928 zio_buf_3584 0 60,928
74,784 hostcache 1,056 73,728
75,240 KMAP_ENTRY 1,440 73,800
81,920 zio_buf_5120 10,240 71,680
94,848 fakepg 0 94,848
108,864 swpctrie 32,112 76,752
110,592 zio_buf_6144 12,288 98,304
111,360 zil_lwb_cache 1,920 109,440
119,808 zio_buf_3072 15,360 104,448
121,856 zio_buf_7168 0 121,856
123,648 syncache 0 123,648
124,000 rl_entry 28,000 96,000
130,416 rtentry 3,536 126,880
132,000 IPFW_IPv4_dynamic_states 0 132,000
132,000 udpcb 800 131,200
132,000 zil_zcw_cache 0 132,000
138,600 metaslab_alloc_trace_cache 0 138,600
154,440 IPFW_dynamic_states_data 0 154,440
156,000 ttyinq 30,080 125,920
157,976 cpuset 2,808 155,168
171,776 ripcb 976 170,800
183,264 taskq_zone 48 183,216
189,440 zio_data_buf_5120 112,640 76,800
194,560 zio_buf_10240 51,200 143,360
198,000 tcptw 0 198,000
200,704 zio_data_buf_3584 71,680 129,024
204,800 NAMEI 0 204,800
215,040 zio_buf_14336 0 215,040
215,040 zio_buf_2560 112,640 102,400
225,720 VNODEPOLL 2,640 223,080
227,840 zio_data_buf_2560 40,960 186,880
230,336 udp_inpcb 12,200 218,136
230,640 SLEEPQUEUE 171,440 59,200
234,240 ttyoutq 26,368 207,872
242,048 tcp_inpcb 27,328 214,720
242,048 tcpcb 54,656 187,392
248,000 KNOTE 69,600 178,400
251,904 12_Bucket 35,328 216,576
254,800 ksiginfo 40,432 214,368
258,048 zio_data_buf_3072 52,224 205,824
258,400 pipe 128,440 129,960
265,216 zio_data_buf_7168 100,352 164,864
272,640 unpcb 45,824 226,816
277,760 selfd 51,008 226,752
278,528 DIRHASH 0 278,528
280,000 Files 81,840 198,160
282,624 zio_buf_1536 53,760 228,864
288,768 zio_data_buf_6144 153,600 135,168
342,720 TURNSTILE 291,448 51,272
357,696 filedesc0 209,760 147,936
366,792 swblk 303,144 63,648
368,640 zio_data_buf_1536 44,544 324,096
370,440 Mountpoints 293,608 76,832
387,072 zio_data_buf_14336 200,704 186,368
413,696 zio_buf_1024 48,128 365,568
448,384 16_Bucket 33,280 415,104
491,520 zio_data_buf_10240 266,240 225,280
492,032 8_Bucket 19,456 472,576
494,016 6_Bucket 36,576 457,440
513,936 PROC 250,992 262,944
516,096 zio_data_buf_2048 61,440 454,656
523,200 socket 231,952 291,248
537,680 g_bio 6,768 530,912
540,960 UMA_Zones 539,000 1,960
557,056 mbuf_jumbo_page 0 557,056
565,248 zio_buf_12288 12,288 552,960
581,632 zio_buf_2048 276,480 305,152
598,016 zio_data_buf_1024 33,792 564,224
605,392 VMSPACE 251,200 354,192
659,456 zio_buf_28672 28,672 630,784
761,856 64_Bucket 283,136 478,720
819,200 zio_buf_20480 20,480 798,720
844,000 4_Bucket 466,496 377,504
906,752 mbuf_packet 512 906,240
917,504 lz4_ctx 0 917,504
937,984 128_Bucket 719,872 218,112
972,096 zio_link_cache 14,592 957,504
1,128,960 32_Bucket 610,304 518,656
1,130,496 8,192 737,280 393,216
1,376,256 zio_buf_57344 0 1,376,256
1,376,256 zio_data_buf_512 21,504 1,354,752
1,400,832 zio_buf_24576 49,152 1,351,680
1,409,024 zio_buf_32768 0 1,409,024
1,474,560 zio_buf_81920 0 1,474,560
1,580,000 FFS_inode 0 1,580,000
1,604,608 mbuf 535,808 1,068,800
1,622,016 16,384 1,048,576 573,440
1,802,240 zio_data_buf_16384 327,680 1,474,560
1,870,080 FFS2_dinode 0 1,870,080
2,064,384 zio_data_buf_28672 516,096 1,548,288
2,097,152 zio_buf_8192 8,192 2,088,960
2,129,920 zio_buf_40960 122,880 2,007,040
2,260,992 zio_buf_49152 0 2,260,992
2,727,936 2,048 1,646,592 1,081,344
2,894,760 umtx_shm 2,758,800 135,960
2,949,120 zio_buf_163840 0 2,949,120
2,998,800 THREAD 2,795,800 203,000
3,145,728 zio_buf_196608 0 3,145,728
3,244,032 zio_data_buf_24576 688,128 2,555,904
3,702,784 zio_data_buf_32768 491,520 3,211,264
3,899,392 zio_buf_229376 0 3,899,392
3,932,160 zio_buf_98304 0 3,932,160
4,128,768 65,536 2,686,976 1,441,792
4,472,832 zio_buf_114688 0 4,472,832
4,747,264 zio_buf_4096 2,469,888 2,277,376
4,891,488 16 4,252,368 639,120
5,144,576 32,768 3,506,176 1,638,400
5,148,672 zio_data_buf_12288 270,336 4,878,336
5,242,880 zio_buf_262144 0 5,242,880
5,242,880 zio_buf_327680 0 5,242,880
5,304,320 zio_data_buf_20480 368,640 4,935,680
5,406,720 zio_data_buf_40960 1,024,000 4,382,720
5,817,088 arc_buf_t 2,932,160 2,884,928
5,853,184 512 1,529,344 4,323,840
5,935,104 256_Bucket 5,836,800 98,304
6,039,544 vmem_btag 4,980,528 1,059,016
6,502,272 L_VFS_Cache 2,106,088 4,396,184
7,471,104 zio_buf_393216 0 7,471,104
8,519,680 zio_buf_65536 0 8,519,680
8,988,000 32 8,279,456 708,544
9,175,040 zio_buf_458752 0 9,175,040
9,535,488 1,024 9,174,016 361,472
11,376,288 BUF_TRIE 0 11,376,288
11,640,832 zio_data_buf_57344 860,160 10,780,672
11,657,216 mbuf_cluster 11,446,272 210,944
11,796,480 zio_buf_655360 0 11,796,480
13,271,040 zio_data_buf_81920 737,280 12,533,760
14,024,704 zio_data_buf_65536 917,504 13,107,200
17,039,360 zio_buf_1310720 0 17,039,360
17,301,504 zio_buf_524288 0 17,301,504
18,087,936 zio_data_buf_98304 1,277,952 16,809,984
18,153,456 zio_cache 388,808 17,764,648
24,144,120 MAP_ENTRY 16,120,200 8,023,920
26,214,400 zio_buf_1048576 2,097,152 24,117,248
29,379,240 range_seg_cache 21,302,856 8,076,384
29,782,080 RADIX_NODE 17,761,104 12,020,976
34,511,400 S_VFS_Cache 31,512,672 2,998,728
38,535,168 zio_buf_786432 0 38,535,168
40,680,144 sa_cache 40,548,816 131,328
41,517,056 zio_data_buf_114688 1,490,944 40,026,112
42,205,184 zio_buf_917504 0 42,205,184
50,147,328 zio_data_buf_4096 98,304 50,049,024
50,675,712 zio_data_buf_49152 983,040 49,692,672
53,972,736 64 29,877,888 24,094,848
61,341,696 zio_buf_131072 42,205,184 19,136,512
72,019,200 VM_OBJECT 71,597,056 422,144
76,731,200 zfs_znode_cache 76,592,208 138,992
88,972,800 256 82,925,568 6,047,232
90,390,528 4,096 89,911,296 479,232
94,036,000 UMA_Slabs 94,033,280 2,720
135,456,000 VNODE 135,273,600 182,400
171,928,320 arc_buf_hdr_t_full 119,737,344 52,190,976
221,970,432 zio_data_buf_8192 166,076,416 55,894,016
277,923,840 dmu_buf_impl_t 130,788,240 147,135,600
376,586,240 zio_buf_16384 372,719,616 3,866,624
397,680,896 128 195,944,448 201,736,448
443,023,360 zio_data_buf_131072 158,072,832 284,950,528
535,584,768 zio_buf_512 255,641,088 279,943,680
713,552,840 dnode_t 373,756,656 339,796,184
3,849,744,384 abd_chunk 3,848,310,784 1,433,600
8,418,769,848 TOTAL 6,459,507,936 1,959,261,912
So far running 12-STABLE "neat" is behaving well for me....
--
Karl Denninger
karl at denninger.net <mailto:karl at denninger.net>
/The Market Ticker/
/[S/MIME encrypted email preferred]/
-------------- next part --------------
A non-text attachment was scrubbed...
Name: smime.p7s
Type: application/pkcs7-signature
Size: 4897 bytes
Desc: S/MIME Cryptographic Signature
URL: <http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20190213/32bc001b/attachment.bin>
More information about the freebsd-stable
mailing list