Driver and 802.11 stack Locking issues

# lock order reversal:                                                          
 1st 0xc6b116dc ath1_node_lock (ath1_node_lock) @ /usr/home/adrian/work/freebsd/
 svn/src/sys/net80211/ieee80211_node.c:1948                                      
  2nd 0xc6b10014 ath1_com_lock (ath1_com_lock) @ /usr/home/adrian/work/freebsd/sv
  n/src/sys/net80211/ieee80211_power.c:295                                        
  KDB: stack backtrace:
  db_trace_thread+30 (?,?,?,?) ra c037da9800000018 sp 0 sz 0
  db_trace_self+1c (?,?,?,?) ra c037dab000000018 sp 0 sz 0
  8007b888+34 (?,?,?,?) ra c037dac8000001a0 sp 0 sz 0
  kdb_backtrace+44 (?,?,?,?) ra c037dc6800000018 sp 0 sz 0
  80203104+34 (?,?,?,?) ra c037dc8000000020 sp 0 sz 0
  witness_checkorder+9cc (?,?,80440f58,127) ra c037dca000000050 sp 0 sz 1
  _mtx_lock_flags+d0 (?,?,?,?) ra c037dcf000000030 sp 0 sz 0
  802c87c8+88 (?,?,?,?) ra c037dd2000000028 sp 0 sz 0
  802c0e74+64 (?,?,?,?) ra c037dd4800000020 sp 0 sz 0
  ieee80211_node_timeout+1a4 (?,?,?,?) ra c037dd6800000040 sp 0 sz 0
  softclock+298 (?,?,?,?) ra c037dda800000058 sp 0 sz 0
  intr_event_execute_handlers+158 (?,?,?,?) ra c037de0000000028 sp 0 sz 0
  8018a7c8+10c (?,?,?,?) ra c037de2800000030 sp 0 sz 0
  fork_exit+a8 (?,?,?,?) ra c037de5800000028 sp 0 sz 0
  fork_trampoline+10 (?,?,?,?) ra c037de8000000000 sp 0 sz 0
  pid 11

# lock order reversal:                                                          
 1st 0xc6b11794 ath1_scan_lock (ath1_scan_lock) @ /usr/home/adrian/work/freebsd/
 svn/src/sys/net80211/ieee80211_node.c:1945                                      
  2nd 0xc6b10014 ath1_com_lock (ath1_com_lock) @ /usr/home/adrian/work/freebsd/sv
  n/src/sys/net80211/ieee80211_node.c:2611                                        
  KDB: stack backtrace:
  db_trace_thread+30 (?,?,?,?) ra c037dab000000018 sp 0 sz 0
  db_trace_self+1c (?,?,?,?) ra c037dac800000018 sp 0 sz 0
  8007b888+34 (?,?,?,?) ra c037dae0000001a0 sp 0 sz 0
  kdb_backtrace+44 (?,?,?,?) ra c037dc8000000018 sp 0 sz 0
  80203104+34 (?,?,?,?) ra c037dc9800000020 sp 0 sz 0
  witness_checkorder+9cc (?,?,804401b0,a33) ra c037dcb800000050 sp 0 sz 1
  _mtx_lock_flags+d0 (?,?,?,?) ra c037dd0800000030 sp 0 sz 0
  ieee80211_node_leave+b8 (?,?,?,?) ra c037dd3800000030 sp 0 sz 0
  ieee80211_node_timeout+2e4 (?,?,?,?) ra c037dd6800000040 sp 0 sz 0
  softclock+298 (?,?,?,?) ra c037dda800000058 sp 0 sz 0
  intr_event_execute_handlers+158 (?,?,?,?) ra c037de0000000028 sp 0 sz 0
  8018a7c8+10c (?,?,?,?) ra c037de2800000030 sp 0 sz 0
  fork_exit+a8 (?,?,?,?) ra c037de5800000028 sp 0 sz 0
  fork_trampoline+10 (?,?,?,?) ra c037de8000000000 sp 0 sz 0
  pid 11

# lock order reversal:
 1st 0xc6be76dc ath0_node_lock (ath0_node_lock) @ /usr/home/adrian/work/freebsd/svn/src/sys/modules/wlan/../../net80211/ieee80211_ioctl.c:1341
 2nd 0xc6be6014 ath0_com_lock (ath0_com_lock) @ /usr/home/adrian/work/freebsd/svn/src/sys/modules/wlan/../../net80211/ieee80211_node.c:2611
KDB: stack backtrace:
db_trace_thread+30 (?,?,?,?) ra c02a381000000018 sp 0 sz 0
db_trace_self+1c (?,?,?,?) ra c02a382800000018 sp 0 sz 0
80077bb8+34 (?,?,?,?) ra c02a3840000001a0 sp 0 sz 0
kdb_backtrace+44 (?,?,?,?) ra c02a39e000000018 sp 0 sz 0
8015d564+34 (?,?,?,?) ra c02a39f800000020 sp 0 sz 0
witness_checkorder+9cc (?,?,c032d8bc,a33) ra c02a3a1800000050 sp 0 sz 1
_mtx_lock_flags+d0 (?,?,?,?) ra c02a3a6800000030 sp 0 sz 0
ieee80211_node_leave+d0 (?,?,?,?) ra c02a3a9800000030 sp 0 sz 0
domlme+9c (?,?,?,?) ra c02a3ac800000020 sp 0 sz 0
setmlme_common+164 (?,?,?,?) ra c02a3ae800000040 sp 0 sz 0
ieee80211_ioctl_setmlme+b8 (?,?,?,?) ra c02a3b2800000048 sp 0 sz 0
ieee80211_ioctl_set80211+5d4 (?,?,?,?) ra c02a3b7000000080 sp 0 sz 0
ieee80211_ioctl+344 (?,?,?,?) ra c02a3bf000000030 sp 0 sz 0
in_control+21c (?,?,?,?) ra c02a3c2000000078 sp 0 sz 0
ifioctl+13e0 (?,?,81e13280,80757c40) ra c02a3c9800000090 sp 0 sz 1
soo_ioctl+3b0 (?,?,?,?) ra c02a3d2800000028 sp 0 sz 0
kern_ioctl+248 (?,?,?,?) ra c02a3d5000000040 sp 0 sz 0
sys_ioctl+130 (?,?,?,?) ra c02a3d9000000038 sp 0 sz 0
trap+7f4 (?,?,?,?) ra c02a3dc8000000b8 sp 0 sz 0
MipsUserGenException+10c (?,?,?,4086a660) ra c02a3e8000000000 sp 0 sz 0
pid 410

This is a recent introduction - due to putting a big lock across the entire TX path. Because ieee80211_node_free is being called as part of the TX path. Somewhere, however, the TX lock is being grabbed whilst the node lock is held - that's going to be fun to figure out.

lock order reversal:
 1st 0xc66016c4 ath1 TX lock (ath1 TX lock) @ /usr/home/adrian/work/freebsd/svn/src/sys/dev/ath/if_ath_misc.h:127
 2nd 0xc66086dc ath1_node_lock (ath1_node_lock) @ /usr/home/adrian/work/freebsd/svn/src/sys/net80211/ieee80211_node.c:1710
KDB: stack backtrace:
db_trace_thread+30 (?,?,?,?) ra c7ecb7b800000018 sp 0 sz 0
db_trace_self+1c (?,?,?,?) ra c7ecb7d000000018 sp 0 sz 0
8007bde8+34 (?,?,?,?) ra c7ecb7e8000001a0 sp 0 sz 0
kdb_backtrace+44 (?,?,?,?) ra c7ecb98800000018 sp 0 sz 0
80213f48+34 (?,?,?,?) ra c7ecb9a000000020 sp 0 sz 0
witness_checkorder+9e0 (?,?,8045fbfc,6ae) ra c7ecb9c000000050 sp 0 sz 1
__mtx_lock_flags+c8 (?,?,?,?) ra c7ecba1000000040 sp 0 sz 0
ieee80211_free_node+40 (?,?,?,?) ra c7ecba5000000030 sp 0 sz 0
ath_start+3a4 (?,?,?,?) ra c7ecba8000000060 sp 0 sz 0
80083c3c+84 (?,?,?,?) ra c7ecbae000000038 sp 0 sz 0
if_start+14 (?,?,?,?) ra c7ecbb1800000018 sp 0 sz 0
8027e978+14c (?,?,?,?) ra c7ecbb3000000028 sp 0 sz 0
ieee80211_start+7a4 (?,?,?,?) ra c7ecbb5800000048 sp 0 sz 0
if_start+14 (?,?,?,?) ra c7ecbba000000018 sp 0 sz 0
8027e978+14c (?,?,?,?) ra c7ecbbb800000028 sp 0 sz 0
80286b74+b0 (?,?,?,?) ra c7ecbbe000000030 sp 0 sz 0
802870dc+26c (?,?,821d5500,?) ra c7ecbc1000000040 sp 0 sz 1
8028959c+260 (?,?,821d5500,?) ra c7ecbc5000000038 sp 0 sz 1
80289908+228 (?,81c40300,?,?) ra c7ecbc8800000038 sp 1 sz 0
8028cb80+3bc (81c40300,?,?,?) ra c7ecbcc000000030 sp 100000000 sz 0
netisr_dispatch_src+134 (?,?,?,?) ra c7ecbcf000000040 sp 0 sz 0
netisr_dispatch+14 (?,?,?,?) ra c7ecbd3000000018 sp 0 sz 0
8028c748+34 (?,?,?,?) ra c7ecbd4800000018 sp 0 sz 0
803d4f90+28c (?,?,?,?) ra c7ecbd6000000048 sp 0 sz 0
intr_event_execute_handlers+1ac (?,?,?,?) ra c7ecbda800000048 sp 0 sz 0
801943b0+150 (?,?,?,?) ra c7ecbdf000000050 sp 0 sz 0
fork_exit+ec (?,?,?,?) ra c7ecbe4000000040 sp 0 sz 0
fork_trampoline+10 (?,?,?,?) ra c7ecbe8000000000 sp 0 sz 0
pid 11

known LOR

com lock / scan lock

witness output

 1st 0xffffff800251f018 run0_com_lock (run0_com_lock) @ /usr/src/sys/net80211/ieee80211_scan.c:686
 2nd 0xffffff8002520948 run0_scan_lock (run0_scan_lock) @ /usr/src/sys/net80211/ieee80211_node.c:2171
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2c
witness_checkorder() at witness_checkorder+0x853
_mtx_lock_flags() at _mtx_lock_flags+0x85
ieee80211_iterate_nt() at ieee80211_iterate_nt+0x3c
ieee80211_iterate_nodes() at ieee80211_iterate_nodes+0x5c
hostap_newstate() at hostap_newstate+0x3a1
run_newstate() at run_newstate+0x1b2
ieee80211_newstate_cb() at ieee80211_newstate_cb+0x71
taskqueue_run_locked() at taskqueue_run_locked+0x93
taskqueue_thread_loop() at taskqueue_thread_loop+0x3e
fork_exit() at fork_exit+0x135
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8090607cf0, rbp = 0 ---


node lock / driver lock
node lock / com lock
occur at the same time and could cause deadlock in 11n mode.

witness output with run(4)

1st 0xffffff80025207f0 run0_node_lock (run0_node_lock) @ /usr/src/sys/net80211/ieee80211_ioctl.c:1341
 2nd 0xffffff80025142a8 run0 (network driver) @ /usr/src/sys/modules/usb/run/../../../dev/usb/wlan/if_run.c:3368
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2c
witness_checkorder() at witness_checkorder+0x853
_mtx_lock_flags() at _mtx_lock_flags+0x85
run_raw_xmit() at run_raw_xmit+0x58
ieee80211_send_mgmt() at ieee80211_send_mgmt+0x4d5
domlme() at domlme+0x95
setmlme_common() at setmlme_common+0x2f0
ieee80211_ioctl_setmlme() at ieee80211_ioctl_setmlme+0x7e
ieee80211_ioctl_set80211() at ieee80211_ioctl_set80211+0x46f
in_control() at in_control+0xad
ifioctl() at ifioctl+0xece
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xf0
amd64_syscall() at amd64_syscall+0x380
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800de7aec, rsp = 0x7fffffffd848, rbp =
 0x2a ---

1st 0xffffff80025207f0 run0_node_lock (run0_node_lock) @ /usr/src/sys/net80211/ieee80211_ioctl.c:1341
 2nd 0xffffff800251f018 run0_com_lock (run0_com_lock) @ /usr/src/sys/net80211/ieee80211_node.c:2565
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2c
witness_checkorder() at witness_checkorder+0x853
_mtx_lock_flags() at _mtx_lock_flags+0x85
ieee80211_node_leave() at ieee80211_node_leave+0x80
setmlme_common() at setmlme_common+0x2f0
ieee80211_ioctl_setmlme() at ieee80211_ioctl_setmlme+0x7e
ieee80211_ioctl_set80211() at ieee80211_ioctl_set80211+0x46f
in_control() at in_control+0xad
ifioctl() at ifioctl+0xece
kern_ioctl() at kern_ioctl+0xcd
sys_ioctl() at sys_ioctl+0xf0
amd64_syscall() at amd64_syscall+0x380
Xfast_syscall() at Xfast_syscall+0xf7
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800de7aec, rsp = 0x7fffffffd848, rbp = 0x2a ---


node lock / driver lock

witness output with run(4)

1st 0xffffff800251c7f0 run0_node_lock (run0_node_lock) @ /usr/src/sys/net80211/ieee80211_node.c:1767
 2nd 0xffffff80025102a8 run0 (network driver) @ /usr/src/sys/modules/usb/run/../../../dev/usb/wlan/if_run.c:2170
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
kdb_backtrace() at kdb_backtrace+0x37
_witness_debugger() at _witness_debugger+0x2c
witness_checkorder() at witness_checkorder+0x853
_mtx_lock_flags() at _mtx_lock_flags+0x85
run_key_delete() at run_key_delete+0x45
_ieee80211_crypto_delkey() at _ieee80211_crypto_delkey+0x9e
ieee80211_crypto_delkey() at ieee80211_crypto_delkey+0x28
ieee80211_node_delucastkey() at ieee80211_node_delucastkey+0x78
ieee80211_sta_leave() at ieee80211_sta_leave+0x16
ieee80211_node_leave() at ieee80211_node_leave+0x11d
hostap_recv_mgmt() at hostap_recv_mgmt+0x1c9
hostap_input() at hostap_input+0xcd5
run_rx_frame() at run_rx_frame+0x13c
run_bulk_rx_callback() at run_bulk_rx_callback+0x2c8
usbd_callback_wrapper() at usbd_callback_wrapper+0x147
usb_command_wrapper() at usb_command_wrapper+0x76
usb_callback_proc() at usb_callback_proc+0x76
usb_process() at usb_process+0xc3
fork_exit() at fork_exit+0x135
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff808e053cf0, rbp = 0 ---

node lock / if_bridge lock
node lock / tcp lock
occur at the same time when running in 11n HOSTAP mode with bridge(4) and could cause deadlock.

witness output with run(4) (11n support has not yet committed to HEAD)

lock order reversal:
 1st 0xffffff8000a267d0 run0_node_lock (run0_node_lock) @ /usr/src/sys/net80211/ieee80211_node.c:1360
 2nd 0xffffff0001716818 if_bridge (if_bridge) @ /usr/src/sys/net/if_bridge.c:2184
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_mtx_lock_flags() at _mtx_lock_flags+0x78
bridge_input() at bridge_input+0x7e
ether_input() at ether_input+0x143
hostap_input() at hostap_input+0x4ea
ampdu_rx_flush() at ampdu_rx_flush+0x5e
ieee80211_ht_node_age() at ieee80211_ht_node_age+0x7b
ieee80211_node_timeout() at ieee80211_node_timeout+0x2dc
softclock() at softclock+0x2a0
intr_event_execute_handlers() at intr_event_execute_handlers+0x66
ithread_loop() at ithread_loop+0xb2
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000052d30, rbp = 0 ---lock order reversal:

lock order reversal:
 1st 0xffffff8000a267d0 run0_node_lock (run0_node_lock) @ /usr/src/sys/net80211/ieee80211_node.c:1360
 2nd 0xffffffff80a186c8 tcp (tcp) @ /usr/src/sys/netinet/tcp_input.c:498
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_rw_rlock() at _rw_rlock+0x5f
tcp_input() at tcp_input+0xa58
ip_input() at ip_input+0xbc
netisr_dispatch_src() at netisr_dispatch_src+0xb8
ether_demux() at ether_demux+0x17d
ether_input() at ether_input+0x175
hostap_input() at hostap_input+0x4ea
ampdu_rx_flush() at ampdu_rx_flush+0x5e
ieee80211_ht_node_age() at ieee80211_ht_node_age+0x7b
ieee80211_node_timeout() at ieee80211_node_timeout+0x2dc
softclock() at softclock+0x2a0
intr_event_execute_handlers() at intr_event_execute_handlers+0x66
ithread_loop() at ithread_loop+0xb2
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000052d30, rbp = 0 ---

WiFi/LockIssues (last edited 2018-04-05T23:19:46+0000 by MateuszPiotrowski)