BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 140s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=7 refcnt=8
    pending: 3*nsim_dev_hwstats_traffic_work, vmstat_shepherd, psi_avgs_work, 2*ovs_dp_masks_rebalance
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=16 refcnt=17
    pending: 7*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, 5*ovs_dp_masks_rebalance, delayed_vfree_work, psi_avgs_work
workqueue events_long: flags=0x100
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=8 refcnt=9
    pending: 8*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=12 refcnt=13
    in-flight: 20145:cfg80211_wiphy_work for 28s cfg80211_wiphy_work ,17073:nsim_dev_trap_report_work for 0s ,1743:cfg80211_wiphy_work for 6s cfg80211_wiphy_work ,26322:cfg80211_wiphy_work for 6s cfg80211_wiphy_work
    pending: 2*nsim_dev_trap_report_work, 2*cfg80211_wiphy_work
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=13 refcnt=14
    in-flight: 1747:nsim_dev_trap_report_work for 5s ,26319:cfg80211_wiphy_work for 28s cfg80211_wiphy_work ,20140:toggle_allocation_gate for 6s ,13971:cfg80211_wiphy_work for 123s cfg80211_wiphy_work ,20149:nsim_dev_trap_report_work for 6s ,1746:cfg80211_wiphy_work for 6s cfg80211_wiphy_work ,26325:cfg80211_wiphy_work for 6s cfg80211_wiphy_work
    pending: 2*cfg80211_wiphy_work, flush_memcg_stats_dwork
workqueue events_freezable: flags=0x104
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x180
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=12 refcnt=13
    in-flight: 5968:wg_ratelimiter_gc_entries for 0s
    pending: neigh_periodic_work, 2*neigh_managed_work, neigh_periodic_work, gc_worker, do_cache_clean, check_lifetime, reg_check_chans_work, 3*check_lifetime
workqueue rcu_gp: flags=0x108
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 32359:process_srcu for 150s
workqueue mm_percpu_wq: flags=0x108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=44
    in-flight: 26317:batadv_tt_purge for 0s
    inactive: 2*batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, 5*batadv_mcast_mla_update, 3*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, 3*batadv_iv_send_outstanding_bat_ogm_packet, batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 5*batadv_iv_send_outstanding_bat_ogm_packet
workqueue hci1: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5345:hci_cmd_sync_work for 120s
workqueue hci2: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5844:hci_cmd_sync_work for 120s
workqueue hci3: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5836:hci_cmd_sync_work for 121s
workqueue hci8: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 945:hci_cmd_sync_work for 122s
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue hci4: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5350:hci_cmd_sync_work for 73s
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue hci0: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 5346:hci_cmd_sync_work for 50s
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue hci5: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=6
    in-flight: 5347:hci_rx_work for 50s
    inactive: hci_rx_work, hci_cmd_work
workqueue hci6: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5
    in-flight: 5359:hci_rx_work for 48s
    inactive: hci_cmd_work
workqueue hci7: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5
    in-flight: 5379:hci_rx_work for 0s
    inactive: hci_cmd_work
workqueue hci9: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5
    in-flight: 5364:hci_rx_work for 0s
    inactive: hci_cmd_work
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=0s workers=8 idle: 5968 24 5898 3179 5828
pool 8: cpus=0-1 flags=0x4 nice=0 hung=0s workers=21 idle: 20140 1743 17073 26325 20149 26319 1746 20145 26322 26324 1747 26320 28629 6007 1756 27297 20151 1751 26321
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=13 idle: 5377 5365 5378
Showing backtraces of busy workers in stalled worker pools:
pool 6:
task:kworker/1:6     state:D stack:24680 pid:32359 tgid:32359 ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: rcu_gp process_srcu
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5298 [inline]
 __schedule+0x15dd/0x52d0 kernel/sched/core.c:6911
 __schedule_loop kernel/sched/core.c:6993 [inline]
 schedule+0x164/0x360 kernel/sched/core.c:7008
 exp_funnel_lock kernel/rcu/tree_exp.h:334 [inline]
 synchronize_rcu_expedited+0x619/0x770 kernel/rcu/tree_exp.h:961
 srcu_readers_active_idx_check kernel/rcu/srcutree.c:537 [inline]
 try_check_zero+0x2cd/0x4d0 kernel/rcu/srcutree.c:1156
 srcu_advance_state kernel/rcu/srcutree.c:1864 [inline]
 process_srcu+0x31d/0x1630 kernel/rcu/srcutree.c:1973
 process_one_work kernel/workqueue.c:3276 [inline]
 process_scheduled_works+0xb6e/0x18c0 kernel/workqueue.c:3359
 worker_thread+0xa53/0xfc0 kernel/workqueue.c:3440
 kthread+0x388/0x470 kernel/kthread.c:436
 ret_from_fork+0x51e/0xb90 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>
pool 6:
task:kworker/1:7     state:R  running task     stack:24168 pid:5948  tgid:5948  ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: events_power_efficient gc_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5298 [inline]
 __schedule+0x15dd/0x52d0 kernel/sched/core.c:6911
 preempt_schedule_irq+0x4d/0xa0 kernel/sched/core.c:7238
 irqentry_exit+0x599/0x620 kernel/entry/common.c:239
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:seqcount_lockdep_reader_access+0xed/0x100 include/linux/seqlock.h:76
Code: 00 75 11 e8 15 86 fa f7 4d 85 f6 75 16 e8 0b 86 fa f7 eb 15 e8 04 86 fa f7 e8 bf f3 e4 01 4d 85 f6 74 ea e8 f5 85 fa f7 fb 5b <41> 5e e9 bc d9 e7 01 cc 66 66 2e 0f 1f 84 00 00 00 00 00 90 90 90
RSP: 0018:ffffc900044c7970 EFLAGS: 00000293
RAX: ffffffff89cb2edb RBX: 0000000000000001 RCX: ffff888030993d00
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc900044c7ad0 R08: ffffffff9011d9b7 R09: 1ffffffff2023b36
R10: dffffc0000000000 R11: fffffbfff2023b37 R12: dffffc0000000000
R13: ffff888020783518 R14: 0000000000000200 R15: 000000000001085f
 nf_conntrack_get_ht include/net/netfilter/nf_conntrack.h:343 [inline]
 gc_worker+0x304/0x1290 net/netfilter/nf_conntrack_core.c:1548
 process_one_work kernel/workqueue.c:3276 [inline]
 process_scheduled_works+0xb6e/0x18c0 kernel/workqueue.c:3359
 worker_thread+0xa53/0xfc0 kernel/workqueue.c:3440
 kthread+0x388/0x470 kernel/kthread.c:436
 ret_from_fork+0x51e/0xb90 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>
pool 6:
task:kworker/1:5     state:D stack:23688 pid:14821 tgid:14821 ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: events_power_efficient reg_check_chans_work
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5298 [inline]
 __schedule+0x15dd/0x52d0 kernel/sched/core.c:6911
 __schedule_loop kernel/sched/core.c:6993 [inline]
 schedule+0x164/0x360 kernel/sched/core.c:7008
 schedule_preempt_disabled+0x13/0x30 kernel/sched/core.c:7065
 __mutex_lock_common kernel/locking/mutex.c:692 [inline]
 __mutex_lock+0x7fe/0x1300 kernel/locking/mutex.c:776
 class_wiphy_constructor include/net/cfg80211.h:6441 [inline]
 reg_leave_invalid_chans net/wireless/reg.c:2450 [inline]
 reg_check_chans_work+0x171/0xfe0 net/wireless/reg.c:2465
 process_one_work kernel/workqueue.c:3276 [inline]
 process_scheduled_works+0xb6e/0x18c0 kernel/workqueue.c:3359
 worker_thread+0xa53/0xfc0 kernel/workqueue.c:3440
 kthread+0x388/0x470 kernel/kthread.c:436
 ret_from_fork+0x51e/0xb90 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>
----------------
Code disassembly (best guess):
   0:	00 75 11             	add    %dh,0x11(%rbp)
   3:	e8 15 86 fa f7       	call   0xf7fa861d
   8:	4d 85 f6             	test   %r14,%r14
   b:	75 16                	jne    0x23
   d:	e8 0b 86 fa f7       	call   0xf7fa861d
  12:	eb 15                	jmp    0x29
  14:	e8 04 86 fa f7       	call   0xf7fa861d
  19:	e8 bf f3 e4 01       	call   0x1e4f3dd
  1e:	4d 85 f6             	test   %r14,%r14
  21:	74 ea                	je     0xd
  23:	e8 f5 85 fa f7       	call   0xf7fa861d
  28:	fb                   	sti
  29:	5b                   	pop    %rbx
* 2a:	41 5e                	pop    %r14 <-- trapping instruction
  2c:	e9 bc d9 e7 01       	jmp    0x1e7d9ed
  31:	cc                   	int3
  32:	66 66 2e 0f 1f 84 00 	data16 cs nopw 0x0(%rax,%rax,1)
  39:	00 00 00 00
  3d:	90                   	nop
  3e:	90                   	nop
  3f:	90                   	nop