BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 160s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=16/256 refcnt=17
    in-flight: 23:nsim_fib_event_work nsim_fib_event_work
    pending: psi_avgs_work, 7*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, ovs_dp_masks_rebalance, rht_deferred_worker, kfree_rcu_monitor, rht_deferred_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=15/256 refcnt=16
    pending: ovs_dp_masks_rebalance, switchdev_deferred_process_work, vmstat_shepherd, 5*ovs_dp_masks_rebalance, kfree_rcu_monitor, fqdir_free_fn, 3*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work
workqueue events_long: flags=0x0
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=5/256 refcnt=6
    pending: 4*defense_work_handler, br_multicast_gc_work
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=6/256 refcnt=7
    pending: br_multicast_gc_work, br_fdb_cleanup, 4*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=17/512 refcnt=18
    in-flight: 11381:nsim_dev_trap_report_work, 10369:cfg80211_wiphy_work cfg80211_wiphy_work, 11386:nsim_dev_trap_report_work, 6192:nsim_dev_trap_report_work, 11389:cfg80211_wiphy_work cfg80211_wiphy_work, 8948:nsim_dev_trap_report_work, 10367:nsim_dev_trap_report_work, 10360:cfg80211_wiphy_work cfg80211_wiphy_work, 10371:toggle_allocation_gate
    pending: 5*nsim_dev_trap_report_work
workqueue events_freezable: flags=0x4
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
    in-flight: 5226:gc_worker
    pending: neigh_managed_work, neigh_periodic_work, do_cache_clean
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=5/256 refcnt=6
    in-flight: 8677:neigh_periodic_work, 8676:reg_check_chans_work
    pending: wg_ratelimiter_gc_entries, neigh_managed_work, check_lifetime
workqueue rcu_gp: flags=0x8
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    in-flight: 8679:wait_rcu_exp_gp
workqueue netns: flags=0xe000a
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=4
    in-flight: 11384:cleanup_net
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: vmstat_update
workqueue kblockd: flags=0x18
  pwq 3: cpus=1 node=0 flags=0x0 nice=-20 active=1/256 refcnt=2
    pending: blk_mq_requeue_work
workqueue dm_bufio_cache: flags=0x8
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: work_fn
workqueue ipv6_addrconf: flags=0xe000a
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=11
    in-flight: 10364:addrconf_dad_work
    inactive: 7*addrconf_verify_work
workqueue bat_events: flags=0xe000a
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=48
    in-flight: 11382:batadv_tt_purge
    inactive: 5*batadv_mcast_mla_update, batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_tt_purge, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 5*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_nc_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    in-flight: 11387:wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    in-flight: 11390:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=2/256 refcnt=3
    in-flight: 61:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    in-flight: 9785:wg_packet_tx_worker
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=2/256 refcnt=3
    in-flight: 8678:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    in-flight: 10356:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue hci0: flags=0xa0012
  pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
    in-flight: 51:hci_cmd_sync_work
workqueue hci1: flags=0xa0012
  pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
    in-flight: 5797:hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=3 MAYDAY
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
    in-flight: 5871:wg_packet_decrypt_worker
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=2/256 refcnt=3
    in-flight: 8674:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
    in-flight: 11383:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 0: cpus=0 node=0 flags=0x1 nice=0 active=1/256 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
pool 0: cpus=0 node=0 flags=0x1 nice=0 hung=5s workers=6 manager: 13395
pool 2: cpus=1 node=0 flags=0x0 nice=0 hung=161s workers=7 idle: 5839 5882 1148
pool 4: cpus=0-1 flags=0x4 nice=0 hung=3s workers=47 idle: 11385 11388 11391 3520 2916 42 6188 49 10363 6184 10366 11 6187 10370 10365 3427 10358 12 1134 6190 10359 10361 8951 6185 59 6189 8947 10357 6191 6186
pool 5: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=7 idle: 11284 5103 5804 5799 5805
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/1:1     state:R  running task     stack:26440 pid:9785  ppid:2      flags:0x00004000
Workqueue: wg-crypt-wg0 wg_packet_tx_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5380 [inline]
 __schedule+0x14d2/0x44d0 kernel/sched/core.c:6699
 preempt_schedule_common+0x82/0xc0 kernel/sched/core.c:6866
 preempt_schedule+0xab/0xc0 kernel/sched/core.c:6890
 preempt_schedule_thunk+0x1a/0x30 arch/x86/entry/thunk_64.S:45
 __local_bh_enable_ip+0x13f/0x1c0 kernel/softirq.c:413
 wg_socket_send_skb_to_peer+0x16b/0x1c0 drivers/net/wireguard/socket.c:184
 wg_packet_create_data_done drivers/net/wireguard/send.c:251 [inline]
 wg_packet_tx_worker+0x1c8/0x7c0 drivers/net/wireguard/send.c:276
 process_one_work kernel/workqueue.c:2634 [inline]
 process_scheduled_works+0xa45/0x15b0 kernel/workqueue.c:2711
 worker_thread+0xa55/0xfc0 kernel/workqueue.c:2792
 kthread+0x2fa/0x390 kernel/kthread.c:388
 ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
 ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
 </TASK>
pool 2:
task:kworker/1:3     state:R  running task     stack:24616 pid:5226  ppid:2      flags:0x00004000
Workqueue: events_power_efficient gc_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5380 [inline]
 __schedule+0x14d2/0x44d0 kernel/sched/core.c:6699
 preempt_schedule_irq+0xb5/0x140 kernel/sched/core.c:7009
 irqentry_exit+0x67/0x70 kernel/entry/common.c:438
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:687
RIP: 0010:seqcount_lockdep_reader_access+0x17f/0x1c0 include/linux/seqlock.h:105
Code: f8 4d 85 e4 75 16 e8 10 21 d9 f8 eb 15 e8 09 21 d9 f8 e8 34 be c2 01 4d 85 e4 74 ea e8 fa 20 d9 f8 fb 48 c7 04 24 0e 36 e0 45 <4b> c7 04 3e 00 00 00 00 66 43 c7 44 3e 09 00 00 43 c6 44 3e 0b 00
RSP: 0018:ffffc900035079a0 EFLAGS: 00000293
RAX: ffffffff88ac7f86 RBX: 0000000000000000 RCX: ffff888023935a00
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc90003507a50 R08: ffffffff90da9597 R09: 1ffffffff21b52b2
R10: dffffc0000000000 R11: fffffbfff21b52b3 R12: 0000000000000200
R13: dffffc0000000000 R14: 1ffff920006a0f34 R15: dffffc0000000000
 nf_conntrack_get_ht include/net/netfilter/nf_conntrack.h:350 [inline]
 gc_worker+0x317/0x14b0 net/netfilter/nf_conntrack_core.c:1504
 process_one_work kernel/workqueue.c:2634 [inline]
 process_scheduled_works+0xa45/0x15b0 kernel/workqueue.c:2711
 worker_thread+0xa55/0xfc0 kernel/workqueue.c:2792
 kthread+0x2fa/0x390 kernel/kthread.c:388
 ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
 ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
 </TASK>
pool 2:
task:kworker/1:0     state:R  running task     stack:24584 pid:23    ppid:2      flags:0x00004000
Workqueue: events nsim_fib_event_work
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5380 [inline]
 __schedule+0x14d2/0x44d0 kernel/sched/core.c:6699
 preempt_schedule_irq+0xb5/0x140 kernel/sched/core.c:7009
 irqentry_exit+0x67/0x70 kernel/entry/common.c:438
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:687
RIP: 0010:slab_update_freelist mm/slub.c:617 [inline]
RIP: 0010:__slab_free+0x233/0x410 mm/slub.c:3682
Code: 75 33 48 89 f7 4c 89 d6 4c 89 ea 48 89 d9 4d 89 f8 e8 51 df ff ff 41 89 c7 eb 55 4c 89 f9 4c 89 d0 4c 89 ea f0 48 0f c7 4e 20 <0f> 84 86 00 00 00 e9 9b fe ff ff 48 89 6c 24 08 48 89 f5 4c 89 64
RSP: 0018:ffffc900001d7708 EFLAGS: 00000242
RAX: ffff888067fea000 RBX: ffff888067fec000 RCX: 0000000080040001
RDX: 0000000080040002 RSI: ffffea00019ffa00 RDI: 0000000000000000
RBP: ffff888017842140 R08: 0000000000000001 R09: 0000000000000000
R10: ffff888067fea000 R11: fffffbfff21b52a6 R12: ffff888017842140
R13: 0000000080040002 R14: 0000000080040001 R15: 0000000080040001
 qlink_free mm/kasan/quarantine.c:166 [inline]
 qlist_free_all+0x75/0xe0 mm/kasan/quarantine.c:185
 kasan_quarantine_reduce+0x143/0x160 mm/kasan/quarantine.c:292
 ____kasan_kmalloc mm/kasan/common.c:340 [inline]
 __kasan_kmalloc+0x22/0xa0 mm/kasan/common.c:383
 kmalloc include/linux/slab.h:600 [inline]
 kzalloc include/linux/slab.h:721 [inline]
 nsim_fib4_rt_create drivers/net/netdevsim/fib.c:280 [inline]
 nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:426 [inline]
 nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline]
 nsim_fib_event drivers/net/netdevsim/fib.c:884 [inline]
 nsim_fib_event_work+0x836/0x3170 drivers/net/netdevsim/fib.c:1494
 process_one_work kernel/workqueue.c:2634 [inline]
 process_scheduled_works+0xa45/0x15b0 kernel/workqueue.c:2711
 worker_thread+0xa55/0xfc0 kernel/workqueue.c:2792
 kthread+0x2fa/0x390 kernel/kthread.c:388
 ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
 ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
 </TASK>
pool 2:
task:kworker/1:7     state:R  running task     stack:24392 pid:5871  ppid:2      flags:0x00004000
Workqueue: wg-crypt-wg1 wg_packet_decrypt_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5380 [inline]
 __schedule+0x14d2/0x44d0 kernel/sched/core.c:6699
 preempt_schedule_common+0x82/0xc0 kernel/sched/core.c:6866
 preempt_schedule+0xab/0xc0 kernel/sched/core.c:6890
 preempt_schedule_thunk+0x1a/0x30 arch/x86/entry/thunk_64.S:45
 __local_bh_enable_ip+0x13f/0x1c0 kernel/softirq.c:413
 spin_unlock_bh include/linux/spinlock.h:396 [inline]
 ptr_ring_consume_bh include/linux/ptr_ring.h:367 [inline]
 wg_packet_decrypt_worker+0xcf3/0xd90 drivers/net/wireguard/receive.c:499
 process_one_work kernel/workqueue.c:2634 [inline]
 process_scheduled_works+0xa45/0x15b0 kernel/workqueue.c:2711
 worker_thread+0xa55/0xfc0 kernel/workqueue.c:2792
 kthread+0x2fa/0x390 kernel/kthread.c:388
 ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
 ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
 </TASK>
----------------
Code disassembly (best guess):
   0:	f8                   	clc
   1:	4d 85 e4             	test   %r12,%r12
   4:	75 16                	jne    0x1c
   6:	e8 10 21 d9 f8       	call   0xf8d9211b
   b:	eb 15                	jmp    0x22
   d:	e8 09 21 d9 f8       	call   0xf8d9211b
  12:	e8 34 be c2 01       	call   0x1c2be4b
  17:	4d 85 e4             	test   %r12,%r12
  1a:	74 ea                	je     0x6
  1c:	e8 fa 20 d9 f8       	call   0xf8d9211b
  21:	fb                   	sti
  22:	48 c7 04 24 0e 36 e0 	movq   $0x45e0360e,(%rsp)
  29:	45
* 2a:	4b c7 04 3e 00 00 00 	movq   $0x0,(%r14,%r15,1) <-- trapping instruction
  31:	00
  32:	66 43 c7 44 3e 09 00 	movw   $0x0,0x9(%r14,%r15,1)
  39:	00
  3a:	43 c6 44 3e 0b 00    	movb   $0x0,0xb(%r14,%r15,1)