start_kernel+0x484/0x490 init/main.c:1220
x86_64_start_reservations+0x24/0x30 arch/x86/kernel/head64.c:310
x86_64_start_kernel+0x12b/0x130 arch/x86/kernel/head64.c:291
common_startup_64+0x13e/0x148
BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 141s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=10 refcnt=11
pending: vmstat_shepherd, 2*psi_avgs_work, 5*ovs_dp_masks_rebalance, xfrm_state_gc_task, nsim_fib_event_work
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=18 refcnt=19
pending: 9*nsim_dev_hwstats_traffic_work, 5*psi_avgs_work, 2*ovs_dp_masks_rebalance, switchdev_deferred_process_work, rht_deferred_worker
workqueue events_long: flags=0x100
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6
pending: 4*defense_work_handler, br_fdb_cleanup
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=7 refcnt=8
pending: 3*defense_work_handler, br_multicast_gc_work, br_fdb_cleanup, br_multicast_gc_work, br_fdb_cleanup
workqueue events_unbound: flags=0x2
pwq 8: cpus=0-1 flags=0x4 nice=0 active=10 refcnt=11
pending: 3*nsim_dev_trap_report_work, macvlan_process_broadcast, crng_reseed, cfg80211_wiphy_work, macvlan_process_broadcast, cfg80211_wiphy_work, 2*macvlan_process_broadcast
pwq 8: cpus=0-1 flags=0x4 nice=0 active=11 refcnt=12
pending: 6*nsim_dev_trap_report_work, toggle_allocation_gate, flush_memcg_stats_dwork, 2*cfg80211_wiphy_work, macvlan_process_broadcast
workqueue events_power_efficient: flags=0x180
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=7 refcnt=8
in-flight: 6008:wg_ratelimiter_gc_entries for 141s
pending: neigh_managed_work, neigh_periodic_work, do_cache_clean, 3*check_lifetime
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=7 refcnt=8
pending: gc_worker, neigh_managed_work, neigh_periodic_work, 4*check_lifetime
workqueue mm_percpu_wq: flags=0x108
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wb_workfn
workqueue kblockd: flags=0x18
pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2
pending: blk_mq_timeout_work
workqueue usb_hub_wq: flags=0x104
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
in-flight: 10:hub_event for 143s hub_event
workqueue ipv6_addrconf: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=10
pending: addrconf_verify_work
inactive: 6*addrconf_verify_work
workqueue krxrpcd: flags=0x2001a
pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=10
pending: rxrpc_peer_keepalive_worker
inactive: 6*rxrpc_peer_keepalive_worker
workqueue bat_events: flags=0x6000a
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=37
pending: batadv_purge_orig
inactive: 4*batadv_purge_orig, 5*batadv_iv_send_outstanding_bat_ogm_packet, 5*batadv_mcast_mla_update, 4*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, 3*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_dat_purge
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
pending: wg_packet_tx_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=142s workers=6 idle: 6009 986 5926 13984
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=0s workers=7 idle: 13960 5946 13421 5823 13680
Showing backtraces of busy workers in stalled worker pools:
pool 6:
task:kworker/1:7 state:R running task stack:23400 pid:5946 tgid:5946 ppid:2 task_flags:0x4208060 flags:0x00080000
Workqueue: events_power_efficient gc_worker
Call Trace:
context_switch kernel/sched/core.c:5387 [inline]
__schedule+0x10e9/0x6820 kernel/sched/core.c:7188
preempt_schedule_irq+0x50/0x90 kernel/sched/core.c:7512
irqentry_exit_to_kernel_mode_preempt include/linux/irq-entry-common.h:476 [inline]
irqentry_exit_to_kernel_mode include/linux/irq-entry-common.h:547 [inline]
irqentry_exit+0x1fe/0x790 kernel/entry/common.c:164
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:__seqprop_spinlock_sequence include/linux/seqlock.h:228 [inline]
RIP: 0010:nf_conntrack_get_ht include/net/netfilter/nf_conntrack.h:343 [inline]
RIP: 0010:gc_worker+0x2bb/0x1630 net/netfilter/nf_conntrack_core.c:1545
Code: 00 00 48 c7 c7 28 04 dd 90 e8 41 f9 2f f8 9c 5b 81 e3 00 02 00 00 31 ff 48 89 de e8 9f 0a 54 f8 58 48 85 db 0f 85 97 0f 00 00 b0 0f 54 f8 8b 1d 0a aa 27 07 89 dd 31 ff 83 e5 01 89 ee e8 4c
RSP: 0018:ffffc90004da7b40 EFLAGS: 00000293
RAX: 0000000000000000 RBX: 0000000000000000 RCX: ffffffff89b564a6
RDX: ffff888037055c40 RSI: ffffffff89b564b5 RDI: ffff888037055c40
RBP: 000000000000b789 R08: 0000000000000007 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000000 R12: dffffc0000000000
R13: 0000000000040000 R14: 0000000000005bc5 R15: 0000000000001770
process_one_work+0xa0e/0x1980 kernel/workqueue.c:3302
process_scheduled_works kernel/workqueue.c:3385 [inline]
worker_thread+0x5ef/0xe50 kernel/workqueue.c:3466
kthread+0x370/0x450 kernel/kthread.c:436
ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
pool 6:
task:kworker/1:3 state:R running task stack:26936 pid:13960 tgid:13960 ppid:2 task_flags:0x4208060 flags:0x00080000
Workqueue: wg-crypt-wg1 wg_packet_decrypt_worker
Call Trace:
context_switch kernel/sched/core.c:5387 [inline]
__schedule+0x10e9/0x6820 kernel/sched/core.c:7188
preempt_schedule_common+0x42/0xc0 kernel/sched/core.c:7369
preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:188 [inline]
_raw_spin_unlock_irq+0x44/0x50 kernel/locking/spinlock.c:206
process_one_work+0x857/0x1980 kernel/workqueue.c:3271
process_scheduled_works kernel/workqueue.c:3385 [inline]
worker_thread+0x5ef/0xe50 kernel/workqueue.c:3466
kthread+0x370/0x450 kernel/kthread.c:436
ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
pool 6:
task:kworker/1:6 state:R running task stack:22168 pid:5939 tgid:5939 ppid:2 task_flags:0x4208060 flags:0x00080000
Workqueue: wg-crypt-wg2 wg_packet_tx_worker
Call Trace:
context_switch kernel/sched/core.c:5387 [inline]
__schedule+0x10e9/0x6820 kernel/sched/core.c:7188
preempt_schedule_common+0x42/0xc0 kernel/sched/core.c:7369
preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:188 [inline]
_raw_spin_unlock_irq+0x44/0x50 kernel/locking/spinlock.c:206
process_one_work+0x857/0x1980 kernel/workqueue.c:3271
process_scheduled_works kernel/workqueue.c:3385 [inline]
worker_thread+0x5ef/0xe50 kernel/workqueue.c:3466
kthread+0x370/0x450 kernel/kthread.c:436
ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
pool 6:
task:kworker/1:2 state:R running task stack:18120 pid:89 tgid:89 ppid:2 task_flags:0x4208060 flags:0x00080000
Workqueue: wg-crypt-wg1 wg_packet_tx_worker
Call Trace:
context_switch kernel/sched/core.c:5387 [inline]
__schedule+0x10e9/0x6820 kernel/sched/core.c:7188
preempt_schedule_common+0x42/0xc0 kernel/sched/core.c:7369
preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
__raw_spin_unlock_irq include/linux/spinlock_api_smp.h:188 [inline]
_raw_spin_unlock_irq+0x44/0x50 kernel/locking/spinlock.c:206
process_one_work+0x857/0x1980 kernel/workqueue.c:3271
process_scheduled_works kernel/workqueue.c:3385 [inline]
worker_thread+0x5ef/0xe50 kernel/workqueue.c:3466
kthread+0x370/0x450 kernel/kthread.c:436
ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158
ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245
----------------
Code disassembly (best guess):
0: 00 00 add %al,(%rax)
2: 48 c7 c7 28 04 dd 90 mov $0xffffffff90dd0428,%rdi
9: e8 41 f9 2f f8 call 0xf82ff94f
e: 9c pushf
f: 5b pop %rbx
10: 81 e3 00 02 00 00 and $0x200,%ebx
16: 31 ff xor %edi,%edi
18: 48 89 de mov %rbx,%rsi
1b: e8 9f 0a 54 f8 call 0xf8540abf
20: 58 pop %rax
21: 48 85 db test %rbx,%rbx
24: 0f 85 97 0f 00 00 jne 0xfc1
* 2a: e8 b0 0f 54 f8 call 0xf8540fdf <-- trapping instruction
2f: 8b 1d 0a aa 27 07 mov 0x727aa0a(%rip),%ebx # 0x727aa3f
35: 89 dd mov %ebx,%ebp
37: 31 ff xor %edi,%edi
39: 83 e5 01 and $0x1,%ebp
3c: 89 ee mov %ebp,%esi
3e: e8 .byte 0xe8
3f: 4c rex.WR