BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 156s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=12 refcnt=13 pending: 6*nsim_dev_hwstats_traffic_work, vmstat_shepherd, 4*ovs_dp_masks_rebalance, delayed_vfree_work pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=16 refcnt=17 pending: bpf_prog_free_deferred, free_obj_work, 4*nsim_dev_hwstats_traffic_work, 4*psi_avgs_work, 2*ovs_dp_masks_rebalance, 2*psi_avgs_work, mptcp_worker, psi_avgs_work workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: 3*defense_work_handler pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4 pending: 3*defense_work_handler workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=5 refcnt=6 in-flight: 36:cfg80211_wiphy_work cfg80211_wiphy_work ,79:toggle_allocation_gate pending: 3*cfg80211_wiphy_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=7 refcnt=8 in-flight: 7657:cfg80211_wiphy_work cfg80211_wiphy_work pending: 3*cfg80211_wiphy_work, flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 7659:nsim_dev_trap_report_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=9 refcnt=10 pending: 9*nsim_dev_trap_report_work workqueue events_freezable: flags=0x104 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: update_balloon_stats_func workqueue events_power_efficient: flags=0x180 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6 in-flight: 5896:reg_check_chans_work pending: 2*neigh_managed_work, 2*neigh_periodic_work pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=5 refcnt=6 pending: nf_flow_offload_work_gc, gc_worker, wg_ratelimiter_gc_entries, 2*check_lifetime workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_update_bandwidth_workfn workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=9 in-flight: 7656:addrconf_verify_work inactive: 5*addrconf_verify_work workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=43 in-flight: 12372:batadv_dat_purge inactive: batadv_bla_periodic_work, 6*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 3*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, 7*batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 5*batadv_mcast_mla_update, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_tt_purge workqueue hci4: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 53:hci_cmd_sync_work workqueue hci3: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5828:hci_cmd_sync_work workqueue hci6: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5826:hci_cmd_sync_work workqueue hci0: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5148:hci_cmd_sync_work workqueue hci1: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 in-flight: 5823:hci_cmd_sync_work workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_tx_worker workqueue wg-kex-wg1: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg2: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_tx_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_tx_worker workqueue wg-kex-wg0: flags=0x124 workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 in-flight: 5989:wg_packet_decrypt_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_decrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_decrypt_worker workqueue wg-kex-wg1: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_tx_worker workqueue wg-kex-wg2: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-kex-wg1: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_tx_worker, wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x124 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_receive_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3 pending: wg_packet_encrypt_worker, wg_packet_tx_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_tx_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_tx_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_tx_worker pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=22s workers=8 idle: 49 10 5938 5904 5882 9 6355 pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=0s workers=7 idle: 43 5989 24 6915 985 5934 pool 8: cpus=0-1 flags=0x4 nice=0 hung=0s workers=13 idle: 7659 3533 12372 7657 7660 79 1017 12371 6926 12 13 pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=8 idle: 12373 5832 12370 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 6: task:kworker/1:3 state:R running task stack:23784 pid:983 tgid:983 ppid:2 task_flags:0x4208060 flags:0x00080000 Workqueue: events_power_efficient gc_worker Call Trace: context_switch kernel/sched/core.c:5325 [inline] __schedule+0x1798/0x4cc0 kernel/sched/core.c:6929 preempt_schedule_irq+0xb5/0x150 kernel/sched/core.c:7256 irqentry_exit+0x6f/0x90 kernel/entry/common.c:211 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697 RIP: 0010:seqcount_lockdep_reader_access+0x17e/0x1c0 include/linux/seqlock.h:75 Code: f8 4d 85 e4 75 16 e8 31 01 48 f8 eb 15 e8 2a 01 48 f8 e8 45 bf d3 01 4d 85 e4 74 ea e8 1b 01 48 f8 fb 48 c7 04 24 0e 36 e0 45 <4b> c7 04 3e 00 00 00 00 66 43 c7 44 3e 09 00 00 43 c6 44 3e 0b 00 RSP: 0018:ffffc90003e3f840 EFLAGS: 00000293 RAX: ffffffff897842d5 RBX: 0000000000000000 RCX: ffff888025a98000 RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000 RBP: ffffc90003e3f8f0 R08: ffffffff8f9e1077 R09: 1ffffffff1f3c20e R10: dffffc0000000000 R11: fffffbfff1f3c20f R12: 0000000000000200 R13: ffff88801f7a7b18 R14: 1ffff920007c7f08 R15: dffffc0000000000 nf_conntrack_get_ht include/net/netfilter/nf_conntrack.h:342 [inline] gc_worker+0x308/0x1380 net/netfilter/nf_conntrack_core.c:1548 process_one_work kernel/workqueue.c:3263 [inline] process_scheduled_works+0xae1/0x17b0 kernel/workqueue.c:3346 worker_thread+0x8a0/0xda0 kernel/workqueue.c:3427 kthread+0x711/0x8a0 kernel/kthread.c:463 ret_from_fork+0x4bc/0x870 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 ---------------- Code disassembly (best guess): 0: f8 clc 1: 4d 85 e4 test %r12,%r12 4: 75 16 jne 0x1c 6: e8 31 01 48 f8 call 0xf848013c b: eb 15 jmp 0x22 d: e8 2a 01 48 f8 call 0xf848013c 12: e8 45 bf d3 01 call 0x1d3bf5c 17: 4d 85 e4 test %r12,%r12 1a: 74 ea je 0x6 1c: e8 1b 01 48 f8 call 0xf848013c 21: fb sti 22: 48 c7 04 24 0e 36 e0 movq $0x45e0360e,(%rsp) 29: 45 * 2a: 4b c7 04 3e 00 00 00 movq $0x0,(%r14,%r15,1) <-- trapping instruction 31: 00 32: 66 43 c7 44 3e 09 00 movw $0x0,0x9(%r14,%r15,1) 39: 00 3a: 43 c6 44 3e 0b 00 movb $0x0,0xb(%r14,%r15,1)