BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 187s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=20 refcnt=21
    pending: 2*psi_avgs_work, vmstat_shepherd, xfrm_state_gc_task, rht_deferred_worker, psi_avgs_work, 6*nsim_dev_hwstats_traffic_work, 4*ovs_dp_masks_rebalance, psi_avgs_work, 3*rht_deferred_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=13 refcnt=14
    in-flight: 8761:rht_deferred_worker ,5941:switchdev_deferred_process_work
    pending: 2*ovs_dp_masks_rebalance, psi_avgs_work, destroy_super_work, rht_deferred_worker, psi_avgs_work, 4*nsim_dev_hwstats_traffic_work, delayed_vfree_work
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6
    pending: defense_work_handler, 3*br_multicast_gc_work, br_fdb_cleanup
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=9 refcnt=10
    pending: 2*defense_work_handler, br_multicast_gc_work, 2*defense_work_handler, 3*br_fdb_cleanup, defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: toggle_allocation_gate
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=8 refcnt=9
    in-flight: 8574:cfg80211_wiphy_work cfg80211_wiphy_work ,8579:cfg80211_wiphy_work cfg80211_wiphy_work ,1091:cfg80211_wiphy_work cfg80211_wiphy_work ,8584:cfg80211_wiphy_work cfg80211_wiphy_work
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=14 refcnt=15
    in-flight: 146:nsim_dev_trap_report_work ,36:nsim_dev_trap_report_work ,8578:nsim_dev_trap_report_work ,15872:cfg80211_wiphy_work cfg80211_wiphy_work ,8580:nsim_dev_trap_report_work ,8573:cfg80211_wiphy_work cfg80211_wiphy_work
    pending: 6*nsim_dev_trap_report_work
workqueue events_freezable: flags=0x104
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x180
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=11 refcnt=12
    in-flight: 5909:neigh_periodic_work
    pending: nf_flow_offload_work_gc, do_cache_clean, 2*neigh_managed_work, reg_check_chans_work, check_lifetime, neigh_periodic_work, 3*check_lifetime
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=4 refcnt=5
    pending: 2*check_lifetime, wg_ratelimiter_gc_entries, gc_worker
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=4 MAYDAY
    pending: 2*kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=8
    in-flight: 7246:addrconf_verify_work
    inactive: 4*addrconf_verify_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=35
    in-flight: 8582:batadv_tt_purge
    inactive: 3*batadv_tt_purge, batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_mcast_mla_update, 8*batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, 2*batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 23272:wg_packet_handshake_receive_worker wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 12:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 39:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 13:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 8585:wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 5997:wg_packet_handshake_receive_worker wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 4300:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    in-flight: 24:wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 5996:wg_packet_handshake_send_worker ,4881:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 6089:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 21194:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 4918:wg_packet_handshake_send_worker ,8571:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 5930:wg_packet_handshake_receive_worker wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 13:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 23721:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 8576:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5
    in-flight: 5963:wg_packet_tx_worker
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 15863:wg_packet_handshake_send_worker ,72:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 981:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=4
    in-flight: 5991:wg_packet_handshake_send_worker wg_packet_handshake_send_worker ,8577:wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue hci1: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4
    in-flight: 15401:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 5888:wg_packet_handshake_receive_worker wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 15864:wg_packet_handshake_send_worker
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 50:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 23283:wg_packet_handshake_receive_worker wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=4
    in-flight: 3480:wg_packet_handshake_send_worker ,5995:wg_packet_handshake_send_worker wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=189s workers=11 idle: 6094 5902 5881 14770 14768 14769 14765 9 14762
pool 6: cpus=1 node=0 flags=0x2 nice=0 hung=3s workers=15 manager: 8762 idle: 795
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=30 manager: 2085
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=6 idle: 23720 14582 23722 5828 5830
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/0:7     state:R  running task     stack:24328 pid:5909  tgid:5909  ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: events_power_efficient neigh_periodic_work
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5256 [inline]
 __schedule+0x149b/0x4fd0 kernel/sched/core.c:6863
 preempt_schedule_irq+0x4d/0xa0 kernel/sched/core.c:7190
 irqentry_exit+0x5e3/0x670 kernel/entry/common.c:216
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:should_resched arch/x86/include/asm/preempt.h:104 [inline]
RIP: 0010:__local_bh_enable_ip+0xd8/0x130 kernel/softirq.c:457
Code: 8b e8 6c 0c d5 09 65 66 8b 05 dc ef fa 10 66 85 c0 75 32 bf 01 00 00 00 e8 05 7d 0b 00 e8 50 91 43 00 fb 65 8b 05 b8 ef fa 10 <85> c0 75 05 e8 ef 46 ae ff 5b 41 5e e9 97 cf d7 09 cc 90 0f 0b 90
RSP: 0018:ffffc90004f2f9e0 EFLAGS: 00000206
RAX: 0000000000000000 RBX: 0000000000000201 RCX: 0000000000000046
RDX: 0000000000000006 RSI: ffffffff8d793871 RDI: ffffffff8bc08a60
RBP: ffff88807b67e840 R08: ffffffff8f822a77 R09: 1ffffffff1f0454e
R10: dffffc0000000000 R11: fffffbfff1f0454f R12: ffff88807b67eb00
R13: 0000000000000000 R14: ffffffff893d4ba8 R15: ffff88804fc7c2f8
 spin_unlock_bh include/linux/spinlock.h:396 [inline]
 neigh_periodic_work+0xc28/0xe50 net/core/neighbour.c:1039
 process_one_work kernel/workqueue.c:3257 [inline]
 process_scheduled_works+0xad1/0x1770 kernel/workqueue.c:3340
 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3421
 kthread+0x711/0x8a0 kernel/kthread.c:463
 ret_from_fork+0x510/0xa50 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>
pool 2:
task:kworker/0:9     state:R  running task     stack:24120 pid:5963  tgid:5963  ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: wg-crypt-wg0 wg_packet_tx_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5256 [inline]
 __schedule+0x149b/0x4fd0 kernel/sched/core.c:6863
 preempt_schedule_irq+0x4d/0xa0 kernel/sched/core.c:7190
 irqentry_exit+0x5e3/0x670 kernel/entry/common.c:216
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:should_resched arch/x86/include/asm/preempt.h:104 [inline]
RIP: 0010:__local_bh_enable_ip+0xd8/0x130 kernel/softirq.c:457
Code: 8b e8 6c 0c d5 09 65 66 8b 05 dc ef fa 10 66 85 c0 75 32 bf 01 00 00 00 e8 05 7d 0b 00 e8 50 91 43 00 fb 65 8b 05 b8 ef fa 10 <85> c0 75 05 e8 ef 46 ae ff 5b 41 5e e9 97 cf d7 09 cc 90 0f 0b 90
RSP: 0018:ffffc9000512f9f0 EFLAGS: 00000206
RAX: 0000000000000000 RBX: 0000000000000200 RCX: 0000000000000046
RDX: 0000000000000006 RSI: ffffffff8d793871 RDI: ffffffff8bc08a60
RBP: dffffc0000000000 R08: ffffffff8f822a77 R09: 1ffffffff1f0454e
R10: dffffc0000000000 R11: fffffbfff1f0454f R12: 000000e20e072de3
R13: 1000000000000000 R14: ffffffff863f374a R15: ffff888078a2d928
 local_bh_enable include/linux/bottom_half.h:33 [inline]
 rcu_read_unlock_bh include/linux/rcupdate.h:936 [inline]
 keep_key_fresh drivers/net/wireguard/send.c:135 [inline]
 wg_packet_create_data_done drivers/net/wireguard/send.c:259 [inline]
 wg_packet_tx_worker+0x70b/0x7c0 drivers/net/wireguard/send.c:276
 process_one_work kernel/workqueue.c:3257 [inline]
 process_scheduled_works+0xad1/0x1770 kernel/workqueue.c:3340
 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3421
 kthread+0x711/0x8a0 kernel/kthread.c:463
 ret_from_fork+0x510/0xa50 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>
----------------
Code disassembly (best guess):
   0:	8b e8                	mov    %eax,%ebp
   2:	6c                   	insb   (%dx),%es:(%rdi)
   3:	0c d5                	or     $0xd5,%al
   5:	09 65 66             	or     %esp,0x66(%rbp)
   8:	8b 05 dc ef fa 10    	mov    0x10faefdc(%rip),%eax        # 0x10faefea
   e:	66 85 c0             	test   %ax,%ax
  11:	75 32                	jne    0x45
  13:	bf 01 00 00 00       	mov    $0x1,%edi
  18:	e8 05 7d 0b 00       	call   0xb7d22
  1d:	e8 50 91 43 00       	call   0x439172
  22:	fb                   	sti
  23:	65 8b 05 b8 ef fa 10 	mov    %gs:0x10faefb8(%rip),%eax        # 0x10faefe2
* 2a:	85 c0                	test   %eax,%eax <-- trapping instruction
  2c:	75 05                	jne    0x33
  2e:	e8 ef 46 ae ff       	call   0xffae4722
  33:	5b                   	pop    %rbx
  34:	41 5e                	pop    %r14
  36:	e9 97 cf d7 09       	jmp    0x9d7cfd2
  3b:	cc                   	int3
  3c:	90                   	nop
  3d:	0f 0b                	ud2
  3f:	90                   	nop