start_kernel+0x3ef/0x4d0 init/main.c:1206 x86_64_start_reservations+0x18/0x30 arch/x86/kernel/head64.c:310 x86_64_start_kernel+0x130/0x190 arch/x86/kernel/head64.c:291 common_startup_64+0x13e/0x148 BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 260s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 260s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=-20 stuck for 254s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=13 refcnt=14 pending: psi_avgs_work, 3*nsim_dev_hwstats_traffic_work, vmstat_shepherd, nsim_dev_hwstats_traffic_work, psi_avgs_work, 3*ovs_dp_masks_rebalance, fqdir_free_fn, psi_avgs_work, ovs_dp_masks_rebalance pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=13 refcnt=14 pending: 6*nsim_dev_hwstats_traffic_work, psi_avgs_work, 5*ovs_dp_masks_rebalance, l2cap_info_timeout workqueue events_highpri: flags=0x110 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: fill_page_cache_func pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: fill_page_cache_func workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5 pending: 3*defense_work_handler, br_fdb_cleanup pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=10 refcnt=11 pending: 9*defense_work_handler, br_fdb_cleanup workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 in-flight: 6043:toggle_allocation_gate pending: flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=19 refcnt=20 in-flight: 49:fsnotify_mark_destroy_workfn pending: 5*nsim_dev_trap_report_work, cfg80211_wiphy_work, 5*nsim_dev_trap_report_work, linkwatch_event, cfg80211_wiphy_work, macvlan_process_broadcast, idle_cull_fn, 2*macvlan_process_broadcast, idle_cull_fn pwq 8: cpus=0-1 flags=0x4 nice=0 active=6 refcnt=7 pending: cfg80211_wiphy_work, idle_cull_fn, crng_reseed, 3*idle_cull_fn workqueue events_power_efficient: flags=0x180 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=9 refcnt=10 pending: nf_flow_offload_work_gc, neigh_managed_work, do_cache_clean, neigh_periodic_work, gc_worker, 4*check_lifetime pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=10 refcnt=11 pending: wg_ratelimiter_gc_entries, neigh_managed_work, neigh_periodic_work, 7*check_lifetime workqueue rcu_gp: flags=0x108 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 in-flight: 5983:process_srcu pending: process_srcu workqueue kvfree_rcu_reclaim: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor workqueue netns: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=5 in-flight: 3494:cleanup_net inactive: cleanup_net workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_workfn pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_workfn workqueue kblockd: flags=0x18 pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: blk_mq_timeout_work workqueue mld: flags=0x40108 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: mld_dad_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=14 pending: addrconf_verify_work inactive: 10*addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=12 pending: rxrpc_peer_keepalive_worker inactive: 8*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=53 pending: batadv_mcast_mla_update inactive: 6*batadv_mcast_mla_update, 6*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 7*batadv_iv_send_outstanding_bat_ogm_packet, batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, 6*batadv_tt_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_queued_expired_zero_key_material workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4 pending: 2*wg_packet_handshake_send_worker, wg_queued_expired_zero_key_material workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue hci3: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout workqueue hci4: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout workqueue hci5: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5 pending: hci_cmd_timeout inactive: hci_conn_timeout workqueue hci7: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5 pending: hci_cmd_timeout inactive: hci_conn_timeout pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=261s workers=11 idle: 8473 6282 787 6084 10 6000 6398 5923 5968 5888 pool 8: cpus=0-1 flags=0x4 nice=0 hung=261s workers=28 idle: 6722 6345 1033 6212 12048 3808 1331 10062 6224 7653 3592 3529 1133 4496 37 6484 2006 3739 13 12647 6725 6181 60 11836 10107 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 2: task:kworker/0:8 state:R running task stack:22024 pid:5983 tgid:5983 ppid:2 task_flags:0x4208060 flags:0x00080000 Workqueue: rcu_gp process_srcu Call Trace: context_switch kernel/sched/core.c:5256 [inline] __schedule+0x1139/0x6150 kernel/sched/core.c:6863 preempt_schedule_irq+0x51/0x90 kernel/sched/core.c:7190 irqentry_exit+0x1d8/0x8c0 kernel/entry/common.c:216 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697 RIP: 0010:queue_delayed_work_on+0x89/0x200 kernel/workqueue.c:2560 Code: 3a 00 45 84 e4 74 3a e8 f5 1e 3a 00 9c 5b 81 e3 00 02 00 00 31 ff 48 89 de e8 b3 19 3a 00 48 85 db 0f 85 57 01 00 00 45 31 e4 d2 1e 3a 00 44 89 e0 5b 5d 41 5c 41 5d 41 5e 41 5f e9 eb 84 8b RSP: 0018:ffffc90004b5fba8 EFLAGS: 00000293 RAX: 0000000000000000 RBX: 0000000000000000 RCX: ffffffff8184bcc1 RDX: ffff88801bfe3d00 RSI: ffffffff8184bccb RDI: 0000000000000007 RBP: 0000000000000200 R08: 0000000000000007 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000002b91 R12: 0000000000000001 R13: 0000000000000008 R14: ffff88813ff53800 R15: 0000000000000000 process_one_work+0x9ba/0x1b20 kernel/workqueue.c:3257 process_scheduled_works kernel/workqueue.c:3340 [inline] worker_thread+0x6c8/0xf10 kernel/workqueue.c:3421 kthread+0x3c5/0x780 kernel/kthread.c:463 ret_from_fork+0x983/0xb10 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246 ---------------- Code disassembly (best guess): 0: 3a 00 cmp (%rax),%al 2: 45 84 e4 test %r12b,%r12b 5: 74 3a je 0x41 7: e8 f5 1e 3a 00 call 0x3a1f01 c: 9c pushf d: 5b pop %rbx e: 81 e3 00 02 00 00 and $0x200,%ebx 14: 31 ff xor %edi,%edi 16: 48 89 de mov %rbx,%rsi 19: e8 b3 19 3a 00 call 0x3a19d1 1e: 48 85 db test %rbx,%rbx 21: 0f 85 57 01 00 00 jne 0x17e 27: 45 31 e4 xor %r12d,%r12d * 2a: e8 d2 1e 3a 00 call 0x3a1f01 <-- trapping instruction 2f: 44 89 e0 mov %r12d,%eax 32: 5b pop %rbx 33: 5d pop %rbp 34: 41 5c pop %r12 36: 41 5d pop %r13 38: 41 5e pop %r14 3a: 41 5f pop %r15 3c: e9 .byte 0xe9 3d: eb 84 jmp 0xffffffc3 3f: 8b .byte 0x8b