start_kernel+0x3ef/0x4d0 init/main.c:1206 x86_64_start_reservations+0x18/0x30 arch/x86/kernel/head64.c:310 x86_64_start_kernel+0x130/0x190 arch/x86/kernel/head64.c:291 common_startup_64+0x13e/0x148 BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 282s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 284s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=-20 stuck for 241s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=22 refcnt=23 pending: 10*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, drain_local_memcg_stock, vmstat_shepherd, psi_avgs_work, 5*ovs_dp_masks_rebalance, switchdev_deferred_process_work, rht_deferred_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=8 refcnt=9 pending: psi_avgs_work, drm_fb_helper_damage_work, vmpressure_work_fn, 3*psi_avgs_work, 2*ovs_dp_masks_rebalance workqueue events_highpri: flags=0x110 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: fill_page_cache_func pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: fill_page_cache_func workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=12 refcnt=13 pending: 6*defense_work_handler, 4*br_fdb_cleanup, br_multicast_gc_work, br_fdb_cleanup pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: defense_work_handler, 2*br_fdb_cleanup workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: toggle_allocation_gate, flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=18 refcnt=19 in-flight: 78:cfg80211_wiphy_work ,6064:nsim_dev_trap_report_work ,6151:nsim_dev_trap_report_work pending: 5*nsim_dev_trap_report_work, 5*cfg80211_wiphy_work, 3*macvlan_process_broadcast, 2*idle_cull_fn pwq 8: cpus=0-1 flags=0x4 nice=0 active=16 refcnt=17 pending: 3*nsim_dev_trap_report_work, 6*cfg80211_wiphy_work, idle_cull_fn, crng_reseed, 2*macvlan_process_broadcast, idle_cull_fn, macvlan_process_broadcast, idle_cull_fn workqueue events_power_efficient: flags=0x180 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9 pending: hash_netnet6_gc, neigh_managed_work, do_cache_clean, check_lifetime, neigh_periodic_work, reg_check_chans_work, 2*check_lifetime pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=11 refcnt=12 in-flight: 5918:wg_ratelimiter_gc_entries pending: 2*hash_netnet6_gc, gc_worker, fb_flashcursor, neigh_managed_work, check_lifetime, neigh_periodic_work, 3*check_lifetime workqueue kvfree_rcu_reclaim: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_workfn workqueue mld: flags=0x40108 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=3 pending: mld_dad_work inactive: mld_dad_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=10 pending: addrconf_verify_work inactive: 6*addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=10 pending: rxrpc_peer_keepalive_worker inactive: 6*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=51 pending: batadv_mcast_mla_update inactive: 5*batadv_mcast_mla_update, 2*batadv_purge_orig, 9*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_purge_orig, 9*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, 3*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_queued_expired_zero_key_material workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue hci0: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=285s workers=14 idle: 5813 8924 6393 6102 789 24 5985 30 5970 6240 6051 5911 6084 pool 8: cpus=0-1 flags=0x4 nice=0 hung=283s workers=26 idle: 6062 6025 2963 6061 37 3655 60 6039 10880 6055 1909 50 3790 3444 3688 6009 6068 1300 6063 13 6003 6058 6876 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 6: task:kworker/1:5 state:R running task stack:24440 pid:5918 tgid:5918 ppid:2 task_flags:0x4208060 flags:0x00080000 Workqueue: events_power_efficient wg_ratelimiter_gc_entries Call Trace: context_switch kernel/sched/core.c:5256 [inline] __schedule+0x1139/0x6150 kernel/sched/core.c:6863 preempt_schedule_irq+0x51/0x90 kernel/sched/core.c:7190 irqentry_exit+0x1d8/0x8c0 kernel/entry/common.c:216 asm_sysvec_reschedule_ipi+0x1a/0x20 arch/x86/include/asm/idtentry.h:702 RIP: 0010:lock_is_held_type+0x107/0x150 kernel/locking/lockdep.c:5945 Code: 00 00 b8 ff ff ff ff 65 0f c1 05 5c 40 3c 08 83 f8 01 75 2d 9c 58 f6 c4 02 75 43 48 f7 04 24 00 02 00 00 74 01 fb 48 83 c4 08 <44> 89 e8 5b 5d 41 5c 41 5d 41 5e 41 5f e9 f2 69 99 f5 45 31 ed eb RSP: 0018:ffffc9000447fac8 EFLAGS: 00000282 RAX: 0000000000000046 RBX: ffff888031a38b58 RCX: 0000000000000001 RDX: 0000000000000000 RSI: ffffffff8daa6ced RDI: ffffffff8bf2b680 RBP: ffffffff8e3c9640 R08: 0000000000000000 R09: fffffbfff1e95250 R10: ffffffff8f4a9283 R11: ffff888031a38b30 R12: ffff888031a38000 R13: 0000000000000000 R14: 00000000ffffffff R15: 0000000000000002 lock_is_held include/linux/lockdep.h:249 [inline] __might_resched+0x473/0x5e0 kernel/sched/core.c:8789 wg_ratelimiter_gc_entries+0x88/0x520 drivers/net/wireguard/ratelimiter.c:78 process_one_work+0x9ba/0x1b20 kernel/workqueue.c:3257 process_scheduled_works kernel/workqueue.c:3340 [inline] worker_thread+0x6c8/0xf10 kernel/workqueue.c:3421 kthread+0x3c5/0x780 kernel/kthread.c:463 ret_from_fork+0x983/0xb10 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246 ---------------- Code disassembly (best guess): 0: 00 00 add %al,(%rax) 2: b8 ff ff ff ff mov $0xffffffff,%eax 7: 65 0f c1 05 5c 40 3c xadd %eax,%gs:0x83c405c(%rip) # 0x83c406b e: 08 f: 83 f8 01 cmp $0x1,%eax 12: 75 2d jne 0x41 14: 9c pushf 15: 58 pop %rax 16: f6 c4 02 test $0x2,%ah 19: 75 43 jne 0x5e 1b: 48 f7 04 24 00 02 00 testq $0x200,(%rsp) 22: 00 23: 74 01 je 0x26 25: fb sti 26: 48 83 c4 08 add $0x8,%rsp * 2a: 44 89 e8 mov %r13d,%eax <-- trapping instruction 2d: 5b pop %rbx 2e: 5d pop %rbp 2f: 41 5c pop %r12 31: 41 5d pop %r13 33: 41 5e pop %r14 35: 41 5f pop %r15 37: e9 f2 69 99 f5 jmp 0xf5996a2e 3c: 45 31 ed xor %r13d,%r13d 3f: eb .byte 0xeb