BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 291s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 257s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=-20 stuck for 165s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: psi_avgs_work, vmstat_shepherd pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=28 refcnt=29 in-flight: 5986:nsim_dev_hwstats_traffic_work pending: 9*nsim_dev_hwstats_traffic_work, 10*ovs_dp_masks_rebalance, rht_deferred_worker, nsim_dev_hwstats_traffic_work, switchdev_deferred_process_work, 2*rht_deferred_worker, rt6_probe_deferred, rht_deferred_worker, debugfs_reap_work workqueue events_highpri: flags=0x110 pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: fill_page_cache_func workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=9 refcnt=10 pending: 5*defense_work_handler, 4*br_fdb_cleanup pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=11 refcnt=12 pending: 5*defense_work_handler, 4*br_multicast_gc_work, 2*br_fdb_cleanup workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: toggle_allocation_gate pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=19 refcnt=20 pending: 10*nsim_dev_trap_report_work, linkwatch_event, crng_reseed, 2*macvlan_process_broadcast, idle_cull_fn, 2*macvlan_process_broadcast, idle_cull_fn, macvlan_process_broadcast pwq 8: cpus=0-1 flags=0x4 nice=0 active=7 refcnt=8 pending: 2*cfg80211_wiphy_work, idle_cull_fn, 2*macvlan_process_broadcast, 2*idle_cull_fn workqueue events_power_efficient: flags=0x180 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=9 refcnt=10 pending: hash_ipportnet6_gc, 2*neigh_managed_work, 2*neigh_periodic_work, do_cache_clean, 3*check_lifetime pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=10 refcnt=11 pending: wg_ratelimiter_gc_entries, reg_check_chans_work, check_lifetime, gc_worker, 6*check_lifetime workqueue kvfree_rcu_reclaim: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor workqueue netns: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=4 pending: cleanup_net workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_workfn workqueue kblockd: flags=0x18 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=2 refcnt=3 pending: 2*blk_mq_timeout_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=13 pending: addrconf_verify_work inactive: 9*addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=13 pending: rxrpc_peer_keepalive_worker inactive: 9*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=62 pending: batadv_iv_send_outstanding_bat_ogm_packet inactive: 9*batadv_mcast_mla_update, batadv_iv_send_outstanding_bat_ogm_packet, 9*batadv_purge_orig, 12*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_tt_purge, batadv_dat_purge, 4*batadv_bla_periodic_work, 2*batadv_dat_purge, batadv_bla_periodic_work, 5*batadv_tt_purge, batadv_bla_periodic_work, 3*batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: 2*wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue hci2: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=258s workers=19 idle: 16367 31975 28527 9167 760 5943 24 21246 10856 6460 16508 6436 5842 89 6182 1790 12162 976 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 6: task:kworker/1:9 state:R running task stack:22728 pid:5986 tgid:5986 ppid:2 task_flags:0x4208060 flags:0x00080000 Workqueue: events nsim_dev_hwstats_traffic_work Call Trace: context_switch kernel/sched/core.c:5256 [inline] __schedule+0x1139/0x6150 kernel/sched/core.c:6863 preempt_schedule_irq+0x51/0x90 kernel/sched/core.c:7190 irqentry_exit+0x1d8/0x8c0 kernel/entry/common.c:216 asm_sysvec_reschedule_ipi+0x1a/0x20 arch/x86/include/asm/idtentry.h:702 RIP: 0010:resched_offsets_ok kernel/sched/core.c:8776 [inline] RIP: 0010:__might_resched+0x92/0x5e0 kernel/sched/core.c:8791 Code: ec 09 85 c0 0f 85 47 04 00 00 65 4c 8b 3d ce 92 28 12 49 8d bf 84 04 00 00 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 <0f> b6 14 02 48 89 f8 83 e0 07 83 c0 03 65 44 8b 35 b9 92 28 12 41 RSP: 0018:ffffc900047dfb58 EFLAGS: 00000217 RAX: dffffc0000000000 RBX: 1ffff920008fbf6d RCX: 0000000000000001 RDX: 1ffff1100519b090 RSI: ffffffff8daa78f3 RDI: ffff888028cd8484 RBP: 0000000000000000 R08: 0000000000000004 R09: 0000000000000000 R10: 0000000000000000 R11: ffffffff9ae4dc70 R12: ffffffff8b8c4cc0 R13: 0000000000000cda R14: 0000000000000000 R15: ffff888028cd8000 process_one_work+0xb83/0x1b20 kernel/workqueue.c:3290 process_scheduled_works kernel/workqueue.c:3340 [inline] worker_thread+0x6c8/0xf10 kernel/workqueue.c:3421 kthread+0x3c5/0x780 kernel/kthread.c:463 ret_from_fork+0x983/0xb10 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246 ---------------- Code disassembly (best guess): 0: ec in (%dx),%al 1: 09 85 c0 0f 85 47 or %eax,0x47850fc0(%rbp) 7: 04 00 add $0x0,%al 9: 00 65 4c add %ah,0x4c(%rbp) c: 8b 3d ce 92 28 12 mov 0x122892ce(%rip),%edi # 0x122892e0 12: 49 8d bf 84 04 00 00 lea 0x484(%r15),%rdi 19: 48 b8 00 00 00 00 00 movabs $0xdffffc0000000000,%rax 20: fc ff df 23: 48 89 fa mov %rdi,%rdx 26: 48 c1 ea 03 shr $0x3,%rdx * 2a: 0f b6 14 02 movzbl (%rdx,%rax,1),%edx <-- trapping instruction 2e: 48 89 f8 mov %rdi,%rax 31: 83 e0 07 and $0x7,%eax 34: 83 c0 03 add $0x3,%eax 37: 65 44 8b 35 b9 92 28 mov %gs:0x122892b9(%rip),%r14d # 0x122892f8 3e: 12 3f: 41 rex.B