BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 243s!
BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 244s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=16/256 refcnt=17
in-flight: 5846:nsim_fib_event_work
pending: 5*nsim_dev_hwstats_traffic_work, 3*psi_avgs_work, kfree_rcu_monitor, 2*ovs_dp_masks_rebalance, ima_keys_handler, switchdev_deferred_process_work, 2*rht_deferred_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=8/256 refcnt=9
pending: vmstat_shepherd, 2*psi_avgs_work, 3*ovs_dp_masks_rebalance, 2*rht_deferred_worker
workqueue events_long: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=8/256 refcnt=9
pending: 4*defense_work_handler, 2*br_multicast_gc_work, 2*br_fdb_cleanup
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=5/256 refcnt=6
pending: defense_work_handler, 2*br_multicast_gc_work, 2*br_fdb_cleanup
workqueue events_unbound: flags=0x2
pwq 4: cpus=0-1 flags=0x4 nice=0 active=10/512 refcnt=11
pending: 2*cfg80211_wiphy_work, flush_memcg_stats_dwork, crng_reseed, 4*macvlan_process_broadcast, 2*idle_cull_fn
pwq 4: cpus=0-1 flags=0x4 nice=0 active=19/512 refcnt=20
pending: 4*nsim_dev_trap_report_work, toggle_allocation_gate, nsim_dev_trap_report_work, 6*cfg80211_wiphy_work, 4*macvlan_process_broadcast, 3*idle_cull_fn
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
pending: neigh_managed_work, neigh_periodic_work, do_cache_clean, gc_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=5/256 refcnt=6
pending: wg_ratelimiter_gc_entries, neigh_managed_work, neigh_periodic_work, reg_check_chans_work, check_lifetime
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
workqueue dm_bufio_cache: flags=0x8
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: work_fn
workqueue ipv6_addrconf: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=8
pending: addrconf_verify_work
inactive: 4*addrconf_verify_work
workqueue krxrpcd: flags=0xa001a
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=8
pending: rxrpc_peer_keepalive_worker
inactive: 4*rxrpc_peer_keepalive_worker
workqueue bat_events: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=39
in-flight: 2955:batadv_nc_worker
inactive: 3*batadv_nc_worker, 4*batadv_mcast_mla_update, 8*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_tt_purge, 2*batadv_bla_periodic_work, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, 2*batadv_dat_purge
workqueue hci3: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
pending: hci_conn_timeout
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue dm-thin: flags=0xa000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=4
pending: do_waker
pool 2: cpus=1 node=0 flags=0x0 nice=0 hung=245s workers=6 idle: 5807 5809 969 23 27
pool 4: cpus=0-1 flags=0x4 nice=0 hung=245s workers=11 idle: 6786 77 1007 49 1322 11 34 985 12 5912
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/1:5 state:R running task stack:24504 pid:5846 ppid:2 flags:0x00004000
Workqueue: events nsim_fib_event_work
Call Trace:
context_switch kernel/sched/core.c:5382 [inline]
__schedule+0x15ae/0x4660 kernel/sched/core.c:6701
schedule+0xbd/0x170 kernel/sched/core.c:6775
schedule_timeout+0x188/0x2d0 kernel/time/timer.c:2168
schedule_timeout_uninterruptible kernel/time/timer.c:2202 [inline]
msleep+0xa1/0xe0 kernel/time/timer.c:2323
nsim_fib4_rt_add drivers/net/netdevsim/fib.c:369 [inline]
nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:432 [inline]
nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline]
nsim_fib_event drivers/net/netdevsim/fib.c:884 [inline]
nsim_fib_event_work+0x1b70/0x32c0 drivers/net/netdevsim/fib.c:1494
process_one_work kernel/workqueue.c:2657 [inline]
process_scheduled_works+0xa60/0x1600 kernel/workqueue.c:2734
worker_thread+0xa5e/0xfe0 kernel/workqueue.c:2815
kthread+0x2fa/0x390 kernel/kthread.c:388
ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
------------[ cut here ]------------
WARNING: CPU: 0 PID: 5120 at kernel/rcu/tree_stall.h:1001 rcu_check_gp_start_stall+0x2dc/0x460 kernel/rcu/tree_stall.h:993
Modules linked in:
CPU: 0 PID: 5120 Comm: syslogd Not tainted syzkaller #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 07/24/2026
RIP: 0010:rcu_check_gp_start_stall+0x2dc/0x460 kernel/rcu/tree_stall.h:1001
Code: ff ff ff 48 c7 c7 a0 44 33 97 be 04 00 00 00 e8 0a 25 6e 00 48 89 df b8 01 00 00 00 87 05 cc 73 c1 15 85 c0 0f 85 19 ff ff ff <0f> 0b 48 81 ff 00 70 13 8d 74 47 48 c7 c0 ac af 8c 8e 48 c1 e8 03
RSP: 0018:ffffc90000007bb8 EFLAGS: 00010046
RAX: 0000000000000000 RBX: ffffffff8d137000 RCX: ffffffff8171d0c6
RDX: 0000000000000001 RSI: 0000000000000004 RDI: ffffffff8d137000
RBP: ffffc90000007e30 R08: 0000000000000003 R09: 0000000000000004
R10: dffffc0000000000 R11: fffffbfff2e66894 R12: 0000000000002904
R13: 1ffff92000000f94 R14: 0000000000000a02 R15: dffffc0000000000
FS: 00007f0356497c80(0000) GS:ffff8880b8e00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000110c4296a0 CR3: 000000003135e000 CR4: 00000000003506f0
Call Trace:
rcu_core+0x5ec/0x17f0 kernel/rcu/tree.c:2462
handle_softirqs+0x27d/0x820 kernel/softirq.c:578
__do_softirq kernel/softirq.c:612 [inline]
invoke_softirq kernel/softirq.c:452 [inline]
__irq_exit_rcu+0xd3/0x190 kernel/softirq.c:661
irq_exit_rcu+0x9/0x20 kernel/softirq.c:673
instr_sysvec_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1088 [inline]
sysvec_apic_timer_interrupt+0xa4/0xc0 arch/x86/kernel/apic/apic.c:1088
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:687
RIP: 0010:console_flush_all+0x883/0xcf0 arch/x86/include/asm/irqflags.h:-1
Code: ed 01 00 00 e8 5e 7e 1b 00 4d 85 ff 48 8b 5c 24 38 75 07 e8 4f 7e 1b 00 eb 06 e8 48 7e 1b 00 fb 49 bf 00 00 00 00 00 fc ff df <48> 8b 44 24 58 42 0f b6 04 38 84 c0 0f 85 2f 02 00 00 80 3b 01 0f
RSP: 0018:ffffc9000322f460 EFLAGS: 00000293
RAX: ffffffff816bf338 RBX: ffffc9000322f5df RCX: ffff88807e255a00
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc9000322f5b0 R08: ffffffff911eb507 R09: 1ffffffff223d6a0
R10: dffffc0000000000 R11: fffffbfff223d6a1 R12: ffffffff8d8ce1e0
R13: 1ffffffff19f96d8 R14: ffffffff8d8ce238 R15: dffffc0000000000
console_unlock+0xad/0x350 kernel/printk/printk.c:3069
console_device+0x148/0x170 kernel/printk/printk.c:3255
tty_lookup_driver+0xf8/0x470 drivers/tty/tty_io.c:1928
tty_open_by_driver drivers/tty/tty_io.c:2045 [inline]
tty_open+0x224/0xcc0 drivers/tty/tty_io.c:2127
chrdev_open+0x58c/0x600 fs/char_dev.c:414
do_dentry_open+0x880/0x14b0 fs/open.c:929
do_open fs/namei.c:3683 [inline]
path_openat+0x284f/0x3270 fs/namei.c:3844
do_filp_open+0x1f2/0x430 fs/namei.c:3871
do_sys_openat2+0x134/0x1d0 fs/open.c:1422
do_sys_open fs/open.c:1437 [inline]
__do_sys_openat fs/open.c:1453 [inline]
__se_sys_openat fs/open.c:1448 [inline]
__x64_sys_openat+0x139/0x160 fs/open.c:1448
do_syscall_x64 arch/x86/entry/common.c:46 [inline]
do_syscall_64+0x55/0xb0 arch/x86/entry/common.c:76
entry_SYSCALL_64_after_hwframe+0x68/0xd2
RIP: 0033:0x7f03565e7407
Code: 48 89 fa 4c 89 df e8 38 aa 00 00 8b 93 08 03 00 00 59 5e 48 83 f8 fc 74 1a 5b c3 0f 1f 84 00 00 00 00 00 48 8b 44 24 10 0f 05 <5b> c3 0f 1f 80 00 00 00 00 83 e2 39 83 fa 08 75 de e8 23 ff ff ff
RSP: 002b:00007ffccaf43030 EFLAGS: 00000202 ORIG_RAX: 0000000000000101
RAX: ffffffffffffffda RBX: 00007f0356497c80 RCX: 00007f03565e7407
RDX: 0000000000000901 RSI: 00007f03567e1100 RDI: ffffffffffffff9c
RBP: 0000000000000901 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000005
R13: 00007f03567e1100 R14: 0000000000000901 R15: 000055c89bf0bd98
----------------
Code disassembly (best guess):
0: ed in (%dx),%eax
1: 01 00 add %eax,(%rax)
3: 00 e8 add %ch,%al
5: 5e pop %rsi
6: 7e 1b jle 0x23
8: 00 4d 85 add %cl,-0x7b(%rbp)
b: ff 48 8b decl -0x75(%rax)
e: 5c pop %rsp
f: 24 38 and $0x38,%al
11: 75 07 jne 0x1a
13: e8 4f 7e 1b 00 call 0x1b7e67
18: eb 06 jmp 0x20
1a: e8 48 7e 1b 00 call 0x1b7e67
1f: fb sti
20: 49 bf 00 00 00 00 00 movabs $0xdffffc0000000000,%r15
27: fc ff df
* 2a: 48 8b 44 24 58 mov 0x58(%rsp),%rax <-- trapping instruction
2f: 42 0f b6 04 38 movzbl (%rax,%r15,1),%eax
34: 84 c0 test %al,%al
36: 0f 85 2f 02 00 00 jne 0x26b
3c: 80 3b 01 cmpb $0x1,(%rbx)
3f: 0f .byte 0xf