zfs locking

[Slawa Olhovchenkov]

I have some kernel traps inside ZFS.

Jan 14 00:13:05 srv3 kernel: Fatal trap 12: page fault while in kernel mode
Jan 14 00:13:05 srv3 kernel: cpuid = 15; apic id = 2e
Jan 14 00:13:05 srv3 kernel: fault virtual address      = 0x14
Jan 14 00:13:05 srv3 kernel: fault code         = supervisor read data, page not present
Jan 14 00:13:05 srv3 kernel: instruction pointer        = 0x20:0xffffffff80de2dd1
Jan 14 00:13:05 srv3 kernel: stack pointer              = 0x28:0xfffffe104ac45460
Jan 14 00:16:04 srv3 syslogd: kernel boot file is /boot/kernel/kernel
Jan 14 00:16:04 srv3 kernel: current process            = 13233 (aiod22)
Jan 14 00:16:04 srv3 kernel: trap number                = 12
Jan 14 00:16:04 srv3 kernel: panic: page fault
Jan 14 00:16:04 srv3 kernel: cpuid = 3
Jan 14 00:16:04 srv3 kernel: KDB: stack backtrace:
Jan 14 00:16:04 srv3 kernel: #0 0xffffffff80523b50 at kdb_backtrace+0x60
Jan 14 00:16:04 srv3 kernel: #1 0xffffffff804edfa5 at panic+0x155
Jan 14 00:16:04 srv3 kernel: #2 0xffffffff806e9a42 at trap_fatal+0x3a2
Jan 14 00:16:04 srv3 kernel: #3 0xffffffff806e9d19 at trap_pfault+0x2c9
Jan 14 00:16:04 srv3 kernel: #4 0xffffffff806e94a6 at trap+0x5e6
Jan 14 00:16:04 srv3 kernel: #5 0xffffffff806d07d2 at calltrap+0x8
Jan 14 00:16:04 srv3 kernel: #6 0xffffffff80dea2f6 at dbuf_read+0x656
Jan 14 00:16:04 srv3 kernel: #7 0xffffffff80df12ff at dmu_buf_hold_array_by_dnode+0x1cf
Jan 14 00:16:04 srv3 kernel: #8 0xffffffff80df21d6 at dmu_read_uio+0x66
Jan 14 00:16:04 srv3 kernel: #9 0xffffffff80e79107 at zfs_freebsd_read+0x357
Jan 14 00:16:04 srv3 kernel: #10 0xffffffff80784a12 at VOP_READ_APV+0x92
Jan 14 00:16:04 srv3 kernel: #11 0xffffffff80596266 at vn_read+0x166
Jan 14 00:16:04 srv3 kernel: #12 0xffffffff80592beb at vn_io_fault+0x15b
Jan 14 00:16:04 srv3 kernel: #13 0xffffffff8104f387 at aio_daemon+0x387
Jan 14 00:16:04 srv3 kernel: #14 0xffffffff804c01ea at fork_exit+0x9a
Jan 14 00:16:04 srv3 kernel: #15 0xffffffff806d0d0e at fork_trampoline+0xe
Jan 14 00:16:04 srv3 kernel: Uptime: 2d9h6m7s

0x20:0xffffffff80de2dd1 is inside arc_read:

(kgdb) x/30i 0xffffffff80de2dc0
0xffffffff80de2dc0 <arc_read+2128>:     add    %ch,%al
0xffffffff80de2dc2 <arc_read+2130>:     pop    %rdx
0xffffffff80de2dc3 <arc_read+2131>:     (bad)  
0xffffffff80de2dc5 <arc_read+2133>:     add    %cl,-0x77(%rax)
0xffffffff80de2dc8 <arc_read+2136>:     retq   
0xffffffff80de2dc9 <arc_read+2137>:     mov    0xc0(%r12),%rax
0xffffffff80de2dd1 <arc_read+2145>:     movslq 0x14(%rax),%rsi
0xffffffff80de2dd5 <arc_read+2149>:     mov    $0xffffffff80ee7c80,%rdi
0xffffffff80de2ddc <arc_read+2156>:     callq  0xffffffff806cfa60 <atomic_add_long>
0xffffffff80de2de1 <arc_read+2161>:     mov    0x18(%rbp),%rax
0xffffffff80de2de5 <arc_read+2165>:     testb  $0x4,(%rax)
0xffffffff80de2de8 <arc_read+2168>:     je     0xffffffff80de2e07 <arc_read+2199>
0xffffffff80de2dea <arc_read+2170>:     mov    %rbx,%rdi
0xffffffff80de2ded <arc_read+2173>:     callq  0xffffffff80e520e0 <zio_nowait>
0xffffffff80de2df2 <arc_read+2178>:     xor    %r15d,%r15d
0xffffffff80de2df5 <arc_read+2181>:     mov    %r15d,%eax
0xffffffff80de2df8 <arc_read+2184>:     add    $0x78,%rsp
0xffffffff80de2dfc <arc_read+2188>:     pop    %rbx
0xffffffff80de2dfd <arc_read+2189>:     pop    %r12
0xffffffff80de2dff <arc_read+2191>:     pop    %r13
0xffffffff80de2e01 <arc_read+2193>:     pop    %r14
0xffffffff80de2e03 <arc_read+2195>:     pop    %r15
0xffffffff80de2e05 <arc_read+2197>:     pop    %rbp
0xffffffff80de2e06 <arc_read+2198>:     retq   

                                DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
                                    zio_t *, rzio);
                                ARCSTAT_INCR(arcstat_l2_read_bytes, // arc_read+2137
                                    hdr->b_l2hdr->b_asize);

                                if (*arc_flags & ARC_NOWAIT) {
                                        zio_nowait(rzio);
                                        return (0);
                                }

                                ASSERT(*arc_flags & ARC_WAIT);
                                if (zio_wait(rzio) == 0)
                                        return (0);

                                /* l2arc read error; goto zio_read() */

Is this locking issue?