[panic] Race in IEEE802.11 layer towards device drivers

PseudoCylon moonlightakkiy at yahoo.ca
Tue Jul 20 10:03:23 UTC 2010


----- Original Message ----
> From: Hans Petter Selasky <hselasky at c2i.net>
> To: freebsd-current at freebsd.org
> Cc: PseudoCylon <moonlightakkiy at yahoo.ca>; Sam Leffler <sam at freebsd.org>; 
>freebsd-usb at freebsd.org
> Sent: Mon, July 19, 2010 1:17:04 PM
> Subject: Re: [panic] Race in IEEE802.11 layer towards device drivers
> 
> Hi AK,
> 
> I've committed your patches to USB P4. I've made some additional  patches.
> 
> Can you check and verify everything?
> 
> http://p4web.freebsd.org/@@181189?ac=10
> 

Hi

If we change sc->cmdq_run = RUN_CMDQ_ABORT,

-- begin excerpt --


@@ -4890,7 +4877,10 @@ run_stop(void *arg)
 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 
 sc->ratectl_run = RUN_RATECTL_OFF;
-sc->cmdq_run = RUN_CMDQ_ABORT;
+
+RUN_CMDQ_LOCK(sc);
+sc->cmdq_run = sc->cmdq_key_set = RUN_CMDQ_ABORT;
+RUN_CMDQ_UNLOCK(sc);

-- end excerpt --


we also need to change this, otherwise key will be cleared.


-- begin patch --

diff --git a/dev/usb/wlan/if_run.c b/dev/usb/wlan/if_run.c
index 017e4b0..f7abe17 100644
--- a/dev/usb/wlan/if_run.c
+++ b/dev/usb/wlan/if_run.c
@@ -4670,8 +4670,6 @@ run_init_locked(struct run_softc *sc)
 if(ic->ic_nrunning > 1)
 return;
 
-run_stop(sc);
-
 for (ntries = 0; ntries < 100; ntries++) {
 if (run_read(sc, RT2860_ASIC_VER_ID, &tmp) != 0)
 goto fail;

-- end patch --


> Also please compile a kernel  with WITNESS enabled to catch any LOR's, hence we 
>
> introduced another  mutex.
> 


The 2nd mutex did solve a deadlock, but doesn't solve the LOR.


-- begin message --

lock order reversal:
 1st 0xffffff8000a257d0 run0_node_lock (run0_node_lock) @ 
/usr/src/sys/net80211/ieee80211_node.c:1736
 2nd 0xffffff8000a19348 run0 (network driver) @ 
/mnt/share/home/AK/FreeBSD/modules/usb/run/../../../../mnt/dev/usb/wlan/if_run.c:2212

KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_mtx_lock_flags() at _mtx_lock_flags+0x78
run_key_delete() at run_key_delete+0x45
_ieee80211_crypto_delkey() at _ieee80211_crypto_delkey+0x9e
ieee80211_crypto_delkey() at ieee80211_crypto_delkey+0x28
ieee80211_node_delucastkey() at ieee80211_node_delucastkey+0x78
ieee80211_sta_leave() at ieee80211_sta_leave+0x16
ieee80211_node_leave() at ieee80211_node_leave+0x11d
hostap_recv_mgmt() at hostap_recv_mgmt+0x33f
hostap_input() at hostap_input+0xc09
run_rx_frame() at run_rx_frame+0x13f
run_bulk_rx_callback() at run_bulk_rx_callback+0x3b7
usbd_callback_wrapper() at usbd_callback_wrapper+0x12b
usb_command_wrapper() at usb_command_wrapper+0x76
usb_callback_proc() at usb_callback_proc+0x76
usb_process() at usb_process+0xbb
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8029b4ed30, rbp = 0 ---

-- end message --

or

-- begin message --

lock order reversal:
 1st 0xffffff8000a257d0 run0_node_lock (run0_node_lock) @ 
/usr/src/sys/net80211/ieee80211_node.c:1736
 2nd 0xffffff8000a19348 run0 (network driver) @ 
/mnt/share/home/AK/FreeBSD/modules/usb/run/../../../../mnt/dev/usb/wlan/if_run.c:2212

KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_mtx_lock_flags() at _mtx_lock_flags+0x78
run_key_delete() at run_key_delete+0x47
_ieee80211_crypto_delkey() at _ieee80211_crypto_delkey+0x9e
ieee80211_crypto_delkey() at ieee80211_crypto_delkey+0x28
ieee80211_node_delucastkey() at ieee80211_node_delucastkey+0x78
ieee80211_sta_leave() at ieee80211_sta_leave+0x16
ieee80211_node_leave() at ieee80211_node_leave+0x11d
ieee80211_node_timeout() at ieee80211_node_timeout+0x1d5
softclock() at softclock+0x2a0
intr_event_execute_handlers() at intr_event_execute_handlers+0x66
ithread_loop() at ithread_loop+0xb2
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000052d30, rbp = 0 ---

-- end message --


There are new warning, "acquiring duplicate lock." For example,


-- begin message --

acquiring duplicate lock of same type: "network driver"
 1st run0 @ /usr/src/sys/dev/usb/usb_request.c:691
 2nd run0 @ 
/mnt/share/home/AK/FreeBSD/modules/usb/run/../../../../mnt/dev/usb/wlan/if_run.c:4831

KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x8ef
_mtx_lock_flags() at _mtx_lock_flags+0x78
run_init_locked() at run_init_locked+0x753
run_ioctl() at run_ioctl+0xad
taskqueue_run() at taskqueue_run+0x91
taskqueue_thread_loop() at taskqueue_thread_loop+0x3f
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff803e5b2d30, rbp = 0 ---

-- end message --


I don't know if it's worth patching or safe to patch (specially lock/unlock in 
run_bulk_tx_callbackN()), but here is one


-- begin patch --

diff --git a/dev/usb/wlan/if_run.c b/dev/usb/wlan/if_run.c
index 017e4b0..2a0b5b6 100644
--- a/dev/usb/wlan/if_run.c
+++ b/dev/usb/wlan/if_run.c
@@ -712,14 +712,14 @@ run_detach(device_t self)
 /* stop all USB transfers */
 usbd_transfer_unsetup(sc->sc_xfer, RUN_N_XFER);
 
-RUN_LOCK(sc);
-
-sc->ratectl_run = RUN_RATECTL_OFF;
-
 RUN_CMDQ_LOCK(sc);
 sc->cmdq_run = sc->cmdq_key_set = RUN_CMDQ_ABORT;
 RUN_CMDQ_UNLOCK(sc);
 
+RUN_LOCK(sc);
+
+sc->ratectl_run = RUN_RATECTL_OFF;
+
 /* free TX list, if any */
 for (i = 0; i != RUN_EP_QUEUES; i++)
 run_unsetup_tx_list(sc, &sc->sc_epq[i]);
@@ -2865,6 +2865,9 @@ tr_setup:
 if ((error == USB_ERR_TIMEOUT) && (vap != NULL)) {
 uint8_t i;
 device_printf(sc->sc_dev, "device timeout\n");
+
+RUN_UNLOCK(sc);
+
 RUN_CMDQ_LOCK(sc);
 i = run_cmdq_append(sc);
 if (i < RUN_CMDQ_MAX) {
@@ -2874,6 +2877,8 @@ tr_setup:
 RUN_CMDQ_UNLOCK(sc);
 if (i < RUN_CMDQ_MAX)
 ieee80211_runtask(ic, &sc->cmdq_task);
+
+RUN_LOCK(sc);
 }
 
 /*
@@ -3134,6 +3139,8 @@ run_tx(struct run_softc *sc, struct mbuf *m, struct 
ieee80211_node *ni)
  */
 uint8_t i;
 
+RUN_UNLOCK(sc);
+
 RUN_CMDQ_LOCK(sc);
 i = run_cmdq_append(sc);
 if (i < RUN_CMDQ_MAX) {
@@ -3144,6 +3151,7 @@ run_tx(struct run_softc *sc, struct mbuf *m, struct 
ieee80211_node *ni)
 if (i < RUN_CMDQ_MAX)
 ieee80211_runtask(ic, &sc->cmdq_task);
 
+RUN_LOCK(sc);
 }
 }
 
@@ -4670,8 +4678,6 @@ run_init_locked(struct run_softc *sc)
 if(ic->ic_nrunning > 1)
 return;
 
-run_stop(sc);
-
 for (ntries = 0; ntries < 100; ntries++) {
 if (run_read(sc, RT2860_ASIC_VER_ID, &tmp) != 0)
 goto fail;
@@ -4827,9 +4833,11 @@ run_init_locked(struct run_softc *sc)
 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 ifp->if_drv_flags |= IFF_DRV_RUNNING;
 
+RUN_UNLOCK(sc);
 RUN_CMDQ_LOCK(sc);
 sc->cmdq_run = RUN_CMDQ_GO;
 RUN_CMDQ_UNLOCK(sc);
+RUN_LOCK(sc);
 
 for(i = 0; i != RUN_N_XFER; i++)
 usbd_xfer_set_stall(sc->sc_xfer[i]);
@@ -4878,12 +4886,12 @@ run_stop(void *arg)
 
 sc->ratectl_run = RUN_RATECTL_OFF;
 
+RUN_UNLOCK(sc);
+
 RUN_CMDQ_LOCK(sc);
 sc->cmdq_run = sc->cmdq_key_set = RUN_CMDQ_ABORT;
 RUN_CMDQ_UNLOCK(sc);
 
-RUN_UNLOCK(sc);
-
 for(i = 0; i < RUN_N_XFER; i++)
 usbd_transfer_drain(sc->sc_xfer[i]);

-- end patch --


There is a LOR between node lock and run lock in run_raw_xmit(), but I haven't 
been able to reproduce with the latest driver. This LOR has been around since 
addition of hostap mode. I didn't fix it because everyone would have objected if 
I had deferred run_raw_xmit().


Following LORs are also around from the beginning and not related to this 
change, but just for info


-- begin message --

lock order reversal:
 1st 0xffffff8000a257d0 run0_node_lock (run0_node_lock) @ 
/usr/src/sys/net80211/ieee80211_ioctl.c:1326
 2nd 0xffffff8000a24018 run0_com_lock (run0_com_lock) @ 
/usr/src/sys/net80211/ieee80211_node.c:2486
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_mtx_lock_flags() at _mtx_lock_flags+0x78
ieee80211_node_leave() at ieee80211_node_leave+0x80
setmlme_common() at setmlme_common+0x27b
ieee80211_ioctl_setmlme() at ieee80211_ioctl_setmlme+0x7e
ieee80211_ioctl_set80211() at ieee80211_ioctl_set80211+0xaba
in_control() at in_control+0x1ff
ifioctl() at ifioctl+0x1100
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xf0
syscallenter() at syscallenter+0x1b5
syscall() at syscall+0x4c
Xfast_syscall() at Xfast_syscall+0xe2
--- syscall (54, FreeBSD ELF64, ioctl), rip = 0x8008a8d4c, rsp = 0x7fffffffe898, 
rbp = 0x800ca6200 ---

-- end message --

-- begin message --

lock order reversal:
 1st 0xffffff8000a24018 run0_com_lock (run0_com_lock) @ 
/usr/src/sys/net80211/ieee80211_scan.c:683
 2nd 0xffffff8000a25928 run0_scan_lock (run0_scan_lock) @ 
/usr/src/sys/net80211/ieee80211_node.c:2135
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_mtx_lock_flags() at _mtx_lock_flags+0x78
ieee80211_iterate_nodes() at ieee80211_iterate_nodes+0x3d
hostap_newstate() at hostap_newstate+0x3a1
run_newstate() at run_newstate+0x1ef
ieee80211_newstate_cb() at ieee80211_newstate_cb+0x71
taskqueue_run() at taskqueue_run+0x91
taskqueue_thread_loop() at taskqueue_thread_loop+0x3f
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff803e5b7d30, rbp = 0 ---

-- end message --


AK





More information about the freebsd-current mailing list