BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 142s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=22 refcnt=23
    pending: 6*nsim_dev_hwstats_traffic_work, 5*psi_avgs_work, vmstat_shepherd, 4*ovs_dp_masks_rebalance, xfrm_state_gc_task, 2*bpf_prog_free_deferred, switchdev_deferred_process_work, 2*rht_deferred_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=7 refcnt=8
    pending: 4*nsim_dev_hwstats_traffic_work, psi_avgs_work, 2*ovs_dp_masks_rebalance
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=7 refcnt=8
    pending: 3*defense_work_handler, 2*br_multicast_gc_work, 2*br_fdb_cleanup
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5
    in-flight: 805:defense_work_handler for 3s
    pending: 3*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=10 refcnt=11
    in-flight: 14262:cfg80211_wiphy_work for 157s cfg80211_wiphy_work ,8618:cfg80211_wiphy_work for 126s cfg80211_wiphy_work ,8613:cfg80211_wiphy_work for 80s cfg80211_wiphy_work ,14267:cfg80211_wiphy_work for 156s cfg80211_wiphy_work ,14264:cfg80211_wiphy_work for 146s cfg80211_wiphy_work
    pending: idle_cull_fn
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=12 refcnt=13
    in-flight: 14255:nsim_dev_trap_report_work for 25s ,14260:nsim_dev_trap_report_work for 35s ,8610:nsim_dev_trap_report_work for 0s ,14258:nsim_dev_trap_report_work for 6s ,14266:nsim_dev_trap_report_work for 4s ,8612:nsim_dev_trap_report_work for 30s
    pending: nsim_dev_trap_report_work, toggle_allocation_gate, 3*nsim_dev_trap_report_work
workqueue events_freezable: flags=0x104
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x180
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=13 refcnt=14
    in-flight: 5884:neigh_periodic_work for 142s
    pending: gc_worker, wg_ratelimiter_gc_entries, 2*neigh_managed_work, neigh_periodic_work, do_cache_clean, hash_ipport6_gc, reg_check_chans_work, 4*check_lifetime
workqueue mm_percpu_wq: flags=0x108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=35
    in-flight: 14261:batadv_tt_purge for 87s
    inactive: 4*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_mcast_mla_update, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_dat_purge, batadv_bla_periodic_work, batadv_tt_purge, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 3*batadv_mcast_mla_update
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5
    in-flight: 10502:wg_packet_encrypt_worker for 143s
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    in-flight: 14259:wg_packet_handshake_send_worker for 2s
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    in-flight: 150:wg_packet_handshake_send_worker for 25s
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 29:wg_packet_handshake_receive_worker for 22s
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    in-flight: 14253:wg_packet_handshake_send_worker for 11s
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5
    in-flight: 5911:wg_packet_tx_worker for 144s
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    in-flight: 10732:wg_packet_handshake_receive_worker for 39s wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    in-flight: 14257:wg_packet_handshake_send_worker for 22s
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    in-flight: 14277:wg_packet_handshake_send_worker for 3s
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=143s workers=8 idle: 5883 989 10 9 5891
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=4s workers=9 idle: 6731 6730 10733 10734 5836 6732
pool 8: cpus=0-1 flags=0x4 nice=0 hung=0s workers=39 idle: 8605 36 8624 13 8602 14254 48 14256 168 8607 8620 8614 145 8627 8622 136 159 12 14269 14275 14274 14272
Showing backtraces of busy workers in stalled worker pools:
pool 2:
task:kworker/0:7     state:R  running task     stack:24072 pid:5911  tgid:5911  ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: wg-crypt-wg0 wg_packet_tx_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5387 [inline]
 __schedule+0x17b4/0x5680 kernel/sched/core.c:7188
 preempt_schedule_common+0x82/0xd0 kernel/sched/core.c:7369
 preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
 __local_bh_enable_ip+0xe1/0x130 kernel/softirq.c:457
 wg_packet_create_data_done drivers/net/wireguard/send.c:247 [inline]
 wg_packet_tx_worker+0x16b/0x7c0 drivers/net/wireguard/send.c:276
 process_one_work kernel/workqueue.c:3302 [inline]
 process_scheduled_works+0xb5d/0x1860 kernel/workqueue.c:3385
 worker_thread+0xa53/0xfc0 kernel/workqueue.c:3466
 kthread+0x388/0x470 kernel/kthread.c:436
 ret_from_fork+0x514/0xb70 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>
pool 2:
task:kworker/0:6     state:R  running task     stack:25696 pid:10502 tgid:10502 ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: wg-crypt-wg0 wg_packet_encrypt_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5387 [inline]
 __schedule+0x17b4/0x5680 kernel/sched/core.c:7188
 preempt_schedule_common+0x82/0xd0 kernel/sched/core.c:7369
 preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
 __raw_spin_unlock_irq include/linux/spinlock_api_smp.h:188 [inline]
 _raw_spin_unlock_irq+0x45/0x50 kernel/locking/spinlock.c:206
 process_one_work kernel/workqueue.c:3271 [inline]
 process_scheduled_works+0x953/0x1860 kernel/workqueue.c:3385
 worker_thread+0xa53/0xfc0 kernel/workqueue.c:3466
 kthread+0x388/0x470 kernel/kthread.c:436
 ret_from_fork+0x514/0xb70 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>
pool 2:
task:kworker/0:4     state:R  running task     stack:24376 pid:5884  tgid:5884  ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: events_power_efficient neigh_periodic_work
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5387 [inline]
 __schedule+0x17b4/0x5680 kernel/sched/core.c:7188
 preempt_schedule_common+0x82/0xd0 kernel/sched/core.c:7369
 preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
 __local_bh_enable_ip+0xe1/0x130 kernel/softirq.c:457
 spin_unlock_bh include/linux/spinlock.h:396 [inline]
 neigh_periodic_work+0xc28/0xe50 net/core/neighbour.c:1038
 process_one_work kernel/workqueue.c:3302 [inline]
 process_scheduled_works+0xb5d/0x1860 kernel/workqueue.c:3385
 worker_thread+0xa53/0xfc0 kernel/workqueue.c:3466
 kthread+0x388/0x470 kernel/kthread.c:436
 ret_from_fork+0x514/0xb70 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
 </TASK>