IPsec performace - netisr hits %100
Özkan KIRIK
ozkan.kirik at gmail.com
Sat May 1 12:39:44 UTC 2021
I've pulled the latest stable/13 branch and make buildworld.
Same configuration and same usage, ping works between two sides, iperf can
connect to server but no data transferred. throughput is around 10Kbps.
I tried both with and without netipsec/ipsec_input.c patch, but no change.
There is something wrong with ipsec (or ipsec in jail) in stable/13.
iperf between host and jail works without ipsec.
I saw some errors in truss output show as a brief:
write(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,131072) ERR#35 'Resource
temporarily unavailable'
recvfrom(3,0x8018048c0,131072,0,0x0,0x0) ERR#35 'Resource temporarily
unavailable'
_umtx_op(0x80026e008,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfffdec8)
ERR#60 'Operation timed out'
Full output of truss output is below:
root at host # jexec client bash
[root at client /]# truss iperf -B 172.16.70.1 -c 172.16.68.1 2>&1 | grep -v
clock_nanosleep
mmap(0x0,135168,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =
34362150912 (0x80024d000)
mprotect(0x80024a000,4096,PROT_READ) = 0 (0x0)
issetugid() = 0 (0x0)
sigfastblock(0x1,0x80024c510) = 0 (0x0)
open("/etc/libmap.conf",O_RDONLY|O_CLOEXEC,011136710) = 3 (0x3)
fstat(3,{ mode=-rw-r--r-- ,inode=482,size=47,blksize=4096 }) = 0 (0x0)
read(3,"# $FreeBSD$\nincludedir /usr/loc"...,47) = 47 (0x2f)
close(3) = 0 (0x0)
open("/usr/local/etc/libmap.d",O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC,0165)
ERR#2 'No such file or directory'
open("/var/run/ld-elf.so.hints",O_RDONLY|O_CLOEXEC,010015464) = 3 (0x3)
read(3,"Ehnt\^A\0\0\0\M^@\0\0\0\M-U\0\0"...,128) = 128 (0x80)
fstat(3,{ mode=-r--r--r-- ,inode=187,size=341,blksize=4096 }) = 0 (0x0)
pread(3,"/lib:/usr/lib:/usr/local/lib:/us"...,213,0x80) = 213 (0xd5)
close(3) = 0 (0x0)
open("/lib/libthr.so.3",O_RDONLY|O_CLOEXEC|O_VERIFY,00) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=186701,size=125952,blksize=32768 }) = 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) = 34362286080
(0x80026e000)
mmap(0x0,184320,PROT_NONE,MAP_GUARD,-1,0x0) = 34362290176 (0x80026f000)
mmap(0x80026f000,53248,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x0)
= 34362290176 (0x80026f000)
mmap(0x80027c000,73728,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0xc000)
= 34362343424 (0x80027c000)
mmap(0x80028e000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x1d000)
= 34362417152 (0x80028e000)
mmap(0x80028f000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x1d000)
= 34362421248 (0x80028f000)
mmap(0x800291000,45056,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,-1,0x0)
= 34362429440 (0x800291000)
munmap(0x80026e000,4096) = 0 (0x0)
close(3) = 0 (0x0)
open("/lib/librt.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,066000) ERR#2 'No such
file or directory'
open("/usr/lib/librt.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,066000) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=411606,size=22824,blksize=32768 }) = 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) = 34362286080
(0x80026e000)
mmap(0x0,36864,PROT_NONE,MAP_GUARD,-1,0x0) = 34362474496 (0x80029c000)
mmap(0x80029c000,12288,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x0)
= 34362474496 (0x80029c000)
mmap(0x80029f000,12288,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x2000)
= 34362486784 (0x80029f000)
mmap(0x8002a2000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x4000)
= 34362499072 (0x8002a2000)
mmap(0x8002a3000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x4000)
= 34362503168 (0x8002a3000)
mmap(0x8002a4000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,-1,0x0)
= 34362507264 (0x8002a4000)
munmap(0x80026e000,4096) = 0 (0x0)
close(3) = 0 (0x0)
open("/lib/libc++.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,00) ERR#2 'No such file
or directory'
open("/usr/lib/libc++.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,00) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=411469,size=824056,blksize=32768 }) = 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) = 34362286080
(0x80026e000)
mmap(0x0,860160,PROT_NONE,MAP_GUARD,-1,0x0) = 34362511360 (0x8002a5000)
mmap(0x8002a5000,376832,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x0)
= 34362511360 (0x8002a5000)
mmap(0x800301000,425984,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x5b000)
= 34362888192 (0x800301000)
mmap(0x800369000,24576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0xc2000)
= 34363314176 (0x800369000)
mmap(0x80036f000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0xc7000)
= 34363338752 (0x80036f000)
mmap(0x800371000,24576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,-1,0x0)
= 34363346944 (0x800371000)
munmap(0x80026e000,4096) = 0 (0x0)
close(3) = 0 (0x0)
open("/lib/libcxxrt.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,041400) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=186668,size=113688,blksize=32768 }) = 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) = 34362286080
(0x80026e000)
mmap(0x0,143360,PROT_NONE,MAP_GUARD,-1,0x0) = 34363371520 (0x800377000)
mmap(0x800377000,53248,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x0)
= 34363371520 (0x800377000)
mmap(0x800384000,61440,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0xc000)
= 34363424768 (0x800384000)
mmap(0x800393000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x1a000)
= 34363486208 (0x800393000)
mmap(0x800394000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x1a000)
= 34363490304 (0x800394000)
mmap(0x800396000,16384,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,-1,0x0)
= 34363498496 (0x800396000)
munmap(0x80026e000,4096) = 0 (0x0)
close(3) = 0 (0x0)
open("/lib/libm.so.5",O_RDONLY|O_CLOEXEC|O_VERIFY,00) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=186671,size=214192,blksize=32768 }) = 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) = 34362286080
(0x80026e000)
mmap(0x0,225280,PROT_NONE,MAP_GUARD,-1,0x0) = 34363514880 (0x80039a000)
mmap(0x80039a000,73728,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x0)
= 34363514880 (0x80039a000)
mmap(0x8003ac000,143360,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x11000)
= 34363588608 (0x8003ac000)
mmap(0x8003cf000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x33000)
= 34363731968 (0x8003cf000)
mmap(0x8003d0000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x33000)
= 34363736064 (0x8003d0000)
munmap(0x80026e000,4096) = 0 (0x0)
close(3) = 0 (0x0)
open("/lib/libgcc_s.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,020643400) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=186687,size=93448,blksize=32768 }) = 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) = 34362286080
(0x80026e000)
mmap(0x0,106496,PROT_NONE,MAP_GUARD,-1,0x0) = 34363740160 (0x8003d1000)
mmap(0x8003d1000,36864,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x0)
= 34363740160 (0x8003d1000)
mmap(0x8003da000,57344,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x8000)
= 34363777024 (0x8003da000)
mmap(0x8003e8000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x15000)
= 34363834368 (0x8003e8000)
mmap(0x8003ea000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x16000)
= 34363842560 (0x8003ea000)
munmap(0x80026e000,4096) = 0 (0x0)
close(3) = 0 (0x0)
open("/lib/libc.so.7",O_RDONLY|O_CLOEXEC|O_VERIFY,00) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=186667,size=1904424,blksize=32768 }) = 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) = 34362286080
(0x80026e000)
mmap(0x0,4182016,PROT_NONE,MAP_GUARD,-1,0x0) = 34363846656 (0x8003eb000)
mmap(0x8003eb000,520192,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x0)
= 34363846656 (0x8003eb000)
mmap(0x80046a000,1331200,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFAULT_READ,3,0x7e000)
= 34364366848 (0x80046a000)
mmap(0x8005af000,36864,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x1c2000)
= 34365698048 (0x8005af000)
mmap(0x8005b8000,28672,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAULT_READ,3,0x1ca000)
= 34365734912 (0x8005b8000)
mmap(0x8005bf000,2265088,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,-1,0x0)
= 34365763584 (0x8005bf000)
munmap(0x80026e000,4096) = 0 (0x0)
close(3) = 0 (0x0)
mprotect(0x80028e000,4096,PROT_READ) = 0 (0x0)
mprotect(0x8002a2000,4096,PROT_READ) = 0 (0x0)
mprotect(0x800369000,20480,PROT_READ) = 0 (0x0)
mprotect(0x800393000,4096,PROT_READ) = 0 (0x0)
mprotect(0x8003cf000,4096,PROT_READ) = 0 (0x0)
mprotect(0x8003e8000,4096,PROT_READ) = 0 (0x0)
mprotect(0x8005af000,36864,PROT_READ) = 0 (0x0)
mprotect(0x8005af000,36864,PROT_READ|PROT_WRITE) = 0 (0x0)
mprotect(0x8005af000,36864,PROT_READ) = 0 (0x0)
readlink("/etc/malloc.conf",0x7fffffffd5c0,1024) ERR#2 'No such file or
directory'
issetugid() = 0 (0x0)
__sysctl("vm.overcommit",2,0x7fffffffbb54,0x7fffffffbb48,0x0,0) = 0 (0x0)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),-1,0x0)
= 34368126976 (0x800800000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),-1,0x0)
= 34370224128 (0x800a00000)
mmap(0x0,4194304,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),-1,0x0)
= 34372321280 (0x800c00000)
__sysctl("kern.usrstack",2,0x8002903c8,0x7fffffffd878,0x0,0) = 0 (0x0)
getrlimit(RLIMIT_STACK,{ cur=536870912,max=536870912 }) = 0 (0x0)
thr_self(0x800a12000) = 0 (0x0)
mmap(0x7fffdfffe000,4096,PROT_NONE,MAP_ANON,-1,0x0) = 140736951476224
(0x7fffdfffe000)
rtprio_thread(RTP_LOOKUP,102215,0x7fffffffd838) = 0 (0x0)
sigaction(SIGTHR,{ 0x8002884d0 SA_SIGINFO ss_t },0x0) = 0 (0x0)
sigprocmask(SIG_UNBLOCK,{ },0x0) = 0 (0x0)
_umtx_op(0x7fffffffd830,UMTX_OP_WAKE,0x1,0x0,0x0) = 0 (0x0)
mprotect(0x0,0,PROT_NONE) = 0 (0x0)
getpid() = 7030 (0x1b76)
getpid() = 7030 (0x1b76)
sigprocmask(SIG_BLOCK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|SIGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2
},{ }) = 0 (0x0)
sigfastblock(0x3,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,{ },0x0) = 0 (0x0)
sigfastblock(0x1,0x800a12038) = 0 (0x0)
getcontext(0x7fffffffcbd0) = 0 (0x0)
sysarch(AMD64_GET_XFPUSTATE,0x7fffffffcb98) = 0 (0x0)
mmap(0x0,135168,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =
34376515584 (0x801000000)
mprotect(0x22c000,4096,PROT_READ) = 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|SIGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2
},{ }) = 0 (0x0)
sigaction(SIGTERM,{ 0x800288380 SA_RESTART|SA_SIGINFO ss_t },{ SIG_DFL 0x0
ss_t }) = 0 (0x0)
sigprocmask(SIG_SETMASK,{ },0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|SIGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2
},{ }) = 0 (0x0)
sigaction(SIGINT,{ 0x800288380 SA_RESTART|SA_SIGINFO ss_t },{ SIG_DFL 0x0
ss_t }) = 0 (0x0)
sigprocmask(SIG_SETMASK,{ },0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|SIGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2
},{ }) = 0 (0x0)
sigaction(SIGALRM,{ 0x800288380 SA_SIGINFO ss_t },{ SIG_DFL 0x0 ss_t }) = 0
(0x0)
sigprocmask(SIG_SETMASK,{ },0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|SIGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2
},{ }) = 0 (0x0)
sigaction(SIGPIPE,{ SIG_IGN SA_RESTART ss_t },{ SIG_DFL 0x0 ss_t }) = 0
(0x0)
sigprocmask(SIG_SETMASK,{ },0x0) = 0 (0x0)
mmap(0x0,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =
34362286080 (0x80026e000)
mmap(0x0,147456,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =
34376650752 (0x801021000)
mmap(0x7fffdfdfd000,2101248,PROT_READ|PROT_WRITE,MAP_STACK,-1,0x0) =
140736949374976 (0x7fffdfdfd000)
mprotect(0x7fffdfdfd000,4096,PROT_NONE) = 0 (0x0)
thr_new(0x7fffffffe880,0x68) = 0 (0x0)
<new thread 102901>
sigfastblock(0x1,0x800a12738) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) = 0 (0x0)
mmap(0x7fffdfbfc000,2101248,PROT_READ|PROT_WRITE,MAP_STACK,-1,0x0) =
140736947273728 (0x7fffdfbfc000)
mprotect(0x7fffdfbfc000,4096,PROT_NONE) = 0 (0x0)
thr_new(0x7fffffffe880,0x68) = 0 (0x0)
<new thread 102902>
sigfastblock(0x1,0x800a12e38) = 0 (0x0)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),-1,0x0)
= 34378612736 (0x801200000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),-1,0x0)
= 34380709888 (0x801400000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),-1,0x0)
= 34382807040 (0x801600000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),-1,0x0)
= 34384904192 (0x801800000)
socket(PF_INET,SOCK_STREAM,0) = 3 (0x3)
bind(3,{ AF_INET 172.16.70.1:0 },16) = 0 (0x0)
connect(3,{ AF_INET 172.16.68.1:5001 },16) = 0 (0x0)
setsockopt(3,SOL_SOCKET,SO_SNDTIMEO,0x7fffdfdfced0,16) = 0 (0x0)
getsockname(3,{ AF_INET 172.16.70.1:34901 },0x800a1b200) = 0 (0x0)
getpeername(3,{ AF_INET 172.16.68.1:5001 },0x800a1b178) = 0 (0x0)
getsockname(3,{ AF_INET 172.16.70.1:34901 },0x7fffdfdfce3c) = 0 (0x0)
__sysctl("net.routetable.0.0.5.0",6,0x0,0x7fffdfdfcdc8,0x0,0) = 0 (0x0)
__sysctl("net.routetable.0.0.5.0",6,0x801826000,0x7fffdfdfcdc8,0x0,0) = 0
(0x0)
_umtx_op(0x80026e008,UMTX_OP_WAKE_PRIVATE,0x7fffffff,0x0,0x0) = 0 (0x0)
_umtx_op(0x80026e008,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfffdec8) = 0
(0x0)
getpid() = 7030 (0x1b76)
getsockopt(3,IPPROTO_TCP,TCP_MAXSEG,0x7fffdfdfceb4,0x7fffdfdfceb0) = 0 (0x0)
getsockopt(3,SOL_SOCKET,SO_SNDBUF,0x7fffdfdfceb4,0x7fffdfdfceb0) = 0 (0x0)
fstat(1,{ mode=p--------- ,inode=12766,size=0,blksize=4096 }) = 0 (0x0)
mmap(0x0,2621440,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),-1,0x0)
= 34387001344 (0x801a00000)
sendto(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,60,MSG_DONTWAIT,NULL,0) = 60
(0x3c)
getsockopt(3,SOL_SOCKET,SO_SNDBUF,0x7fffdfffde64,0x7fffdfffde60) = 0 (0x0)
setsockopt(3,SOL_SOCKET,SO_SNDTIMEO,0x7fffdfdfced0,16) = 0 (0x0)
------------------------------------------------------------
Client connecting to 172.16.68.1, TCP port 5001
TCP window size: 33.2 KByte (default)
------------------------------------------------------------
write(1,"--------------------------------"...,208) = 208 (0xd0)
[ 1] local 172.16.70.1 port 34901 connected with 172.16.68.1 port 5001
write(1,"[ 1] local 172.16.70.1 port 349"...,72) = 72 (0x48)
write(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,131072) = 34000 (0x84d0)
write(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,131072) ERR#35 'Resource
temporarily unavailable'
setitimer(0,{ 0.000000, 0.000000 },0x0) = 0 (0x0)
shutdown(3,SHUT_WR) = 0 (0x0)
setsockopt(3,SOL_SOCKET,SO_RCVTIMEO,0x7fffdfdfce60,16) = 0 (0x0)
recvfrom(3,"\0\0\0\^B\0\0\0\^\\0\0\0\0\0\^B"...,131072,0,NULL,0x0) = 28
(0x1c)
recvfrom(3,0x8018048c0,131072,0,0x0,0x0) ERR#35 'Resource temporarily
unavailable'
_umtx_op(0x80026e020,UMTX_OP_WAKE_PRIVATE,0x7fffffff,0x0,0x0) = 0 (0x0)
_umtx_op(0x80026e020,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfdfcd38) = 0
(0x0)
[ ID] Interval Transfer Bandwidth
[ 1] 0.00-20.05 sec 33.3 KBytes 13.6 Kbits/sec
write(1,"[ ID] Interval Transfer "...,94) = 94 (0x5e)
_umtx_op(0x80026e020,UMTX_OP_WAKE_PRIVATE,0x7fffffff,0x0,0x0) = 0 (0x0)
_umtx_op(0x80026e020,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfdfcd38) = 0
(0x0)
close(3) = 0 (0x0)
_umtx_op(0x800a130c0,UMTX_OP_NWAKE_PRIVATE,0x1,0x0,0x0) = 0 (0x0)
_umtx_op(0x8002908a8,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x0,0x0) = 0 (0x0)
__sysctl("kern.ostype",2,0x80024be6a,0x7fffdfdfb720,0x0,0) = 0 (0x0)
__sysctl("kern.hostname",2,0x80024bf6a,0x7fffdfdfb720,0x0,0) = 0 (0x0)
__sysctl("kern.osrelease",2,0x80024c06a,0x7fffdfdfb720,0x0,0) = 0 (0x0)
__sysctl("kern.version",2,0x80024c16a,0x7fffdfdfb720,0x0,0) = 0 (0x0)
__sysctl("hw.machine",2,0x80024c26a,0x7fffdfdfb720,0x0,0) = 0 (0x0)
open("/lib/libgcc_s.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,00) = 3 (0x3)
fstat(3,{ mode=-r--r--r-- ,inode=186687,size=93448,blksize=32768 }) = 0
(0x0)
close(3) = 0 (0x0)
madvise(0x801a00000,1839104,MADV_FREE) = 0 (0x0)
madvise(0x80184f000,4096,MADV_FREE) = 0 (0x0)
madvise(0x801851000,36864,MADV_FREE) = 0 (0x0)
madvise(0x801801000,274432,MADV_FREE) = 0 (0x0)
<thread 102902 exited>
_umtx_op(0x80026e008,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfffdec8)
ERR#60 'Operation timed out'
_umtx_op(0x800a129c0,UMTX_OP_NWAKE_PRIVATE,0x1,0x0,0x0) = 0 (0x0)
_umtx_op(0x8002908a8,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) = 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) = 0 (0x0)
<thread 102901 exited>
exit(0x0)
process exit, rval = 0
On Sat, May 1, 2021 at 1:26 AM Özkan KIRIK <ozkan.kirik at gmail.com> wrote:
> Here is fresh results;
> a bit performance gain achieved. But strangely QAT is better now.
>
> - with CCR - 2.14 Gbps
> PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU COMMAND
> 15 root -16 - 0B 16K CPU7 7 0:11 100.00% [crypto
> returns 9]
> 11 root -92 - 0B 1120K CPU4 4 0:10 98.74%
> [intr{irq295: t6nex0:0a0}]
> 14 root -16 - 0B 16K CPU11 11 0:09 95.32% [crypto
> returns 8]
> 11 root -92 - 0B 1120K CPU8 8 0:07 63.49%
> [intr{irq297: t6nex0:0a2}]
> 11 root -92 - 0B 1120K WAIT 12 0:05 42.73%
> [intr{irq307: t6nex0:1a2}]
> 16 root -16 - 0B 16K CPU15 15 0:04 33.82% [crypto
> returns 10]
> 13 root -16 - 0B 16K RUN 3 0:02 21.78% [crypto
> returns 7]
> 4317 root 22 0 23M 4348K sbwait 14 0:01 7.24% iperf
> -B 172.16.70.6 -c 172.16.68.1 -P 2 -t 20{iperf}
> 4329 root 23 0 23M 4348K sbwait 14 0:01 7.18% iperf
> -B 172.16.70.10 -c 172.16.68.1 -P 2 -t 20{iperf}
> ...
> 11 root -92 - 0B 1120K WAIT 3 0:00 0.31%
> [intr{irq294: t6nex0:evt}]
> 0 root -92 - 0B 2144K - 14 0:00 0.30%
> [kernel{t6nex0 tq0}]
> ...
>
>
> - with QAT - 2.63 Gbps
> PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU COMMAND
> 8 root -16 - 0B 16K CPU7 7 0:31 92.05% [crypto
> returns 5]
> 7 root -16 - 0B 16K CPU0 0 0:29 86.03% [crypto
> returns 4]
> 9 root -16 - 0B 16K RUN 15 0:15 43.32% [crypto
> returns 6]
> 11 root -92 - 0B 1664K RUN 4 0:15 43.03%
> [intr{irq297: t6nex0:0a2}]
> 6 root -16 - 0B 16K crypto 4 0:09 26.01% [crypto
> returns 3]
> 11 root -92 - 0B 1664K WAIT 12 0:07 19.88%
> [intr{irq307: t6nex0:1a2}]
> 11 root -92 - 0B 1664K WAIT 8 0:02 7.22%
> [intr{irq368: qat1}]
> 4313 root 22 0 23M 4348K sbwait 2 0:01 6.51% iperf
> -B 172.16.70.8 -c 172.16.68.1 -P 2 -t 20{iperf}
> 4316 root 22 0 23M 4348K sbwait 3 0:01 6.18% iperf
> -B 172.16.70.9 -c 172.16.68.1 -P 2 -t 20{iperf}
> ...
> 11 root -92 - 0B 1664K WAIT 12 0:02 4.52%
> [intr{irq355: qat0}]
> ...
> 11 root -92 - 0B 1664K WAIT 12 0:01 1.89%
> [intr{irq372: qat1}]
> 11 root -92 - 0B 1664K WAIT 0 0:00 1.12%
> [intr{irq360: qat1}]
> 11 root -92 - 0B 1664K WAIT 2 0:00 1.11%
> [intr{irq362: qat1}]
> 11 root -92 - 0B 1664K WAIT 14 0:00 1.10%
> [intr{irq374: qat1}]
> 11 root -92 - 0B 1664K WAIT 4 0:00 1.10%
> [intr{irq364: qat1}]
> 11 root -92 - 0B 1664K WAIT 10 0:00 1.10%
> [intr{irq370: qat1}]
> 11 root -92 - 0B 1664K WAIT 7 0:00 1.09%
> [intr{irq367: qat1}]
> 11 root -92 - 0B 1664K WAIT 11 0:00 1.07%
> [intr{irq371: qat1}]
> 11 root -92 - 0B 1664K WAIT 5 0:00 1.04%
> [intr{irq365: qat1}]
> 11 root -92 - 0B 1664K WAIT 15 0:00 1.04%
> [intr{irq375: qat1}]
> 11 root -92 - 0B 1664K WAIT 1 0:00 1.04%
> [intr{irq361: qat1}]
> 11 root -92 - 0B 1664K WAIT 6 0:00 1.03%
> [intr{irq366: qat1}]
>
> On Sat, May 1, 2021 at 12:45 AM Mark Johnston <markj at freebsd.org> wrote:
>
>> On Sat, May 01, 2021 at 12:31:57AM +0300, Özkan KIRIK wrote:
>> > Hello again,
>> >
>> > patch is applied, now netisr is not eating CPU. but performance drops
>> > around 0.2Gbps according to previous kernel.
>> >
>> > I tried also both net.isr.maxthreads=1 and net.isr.maxthreads=4 .
>> results
>> > are same
>> >
>> > Results are:
>> >
>> > - with CCR - 1.8Gbps
>> > top:
>> > PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU
>> COMMAND
>> > 14 root -16 - 0B 16K CPU5 5 1:38 100.00%
>> [crypto
>> > returns 8]
>> > 3 root -16 - 0B 16K CPU1 1 0:58 77.83%
>> [crypto
>> > returns 0]
>>
>> Could you also try this patch? It won't help with aesni.
>>
>> diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
>> index dfd22662e87f..bc23056e86ad 100644
>> --- a/sys/opencrypto/crypto.c
>> +++ b/sys/opencrypto/crypto.c
>> @@ -94,6 +94,7 @@ struct crypto_session {
>> void *softc;
>> uint32_t hid;
>> uint32_t capabilities;
>> + uint64_t id;
>> };
>>
>> SDT_PROVIDER_DEFINE(opencrypto);
>> @@ -572,6 +573,7 @@ crypto_select_driver(const struct cryptoini *cri, int
>> flags)
>> int
>> crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int
>> crid)
>> {
>> + static uint64_t sessid = 0;
>> crypto_session_t res;
>> void *softc_mem;
>> struct cryptocap *cap;
>> @@ -616,6 +618,7 @@ crypto_newsession(crypto_session_t *cses, struct
>> cryptoini *cri, int crid)
>> softc_mem = malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO);
>> res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO);
>> res->softc = softc_mem;
>> + res->id = atomic_fetchadd_64(&sessid, 1);
>>
>> CRYPTO_DRIVER_LOCK();
>> cap = crypto_checkdriver(hid);
>> @@ -1016,7 +1019,7 @@ crypto_dispatch(struct cryptop *crp)
>> binuptime(&crp->crp_tstamp);
>> #endif
>>
>> - crp->crp_retw_id = ((uintptr_t)crp->crp_session) %
>> crypto_workers_num;
>> + crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
>>
>> if (CRYPTOP_ASYNC(crp)) {
>> if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
>>
>
More information about the freebsd-net
mailing list