BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 177s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=4/256
in-flight: 8964:rtc_timer_do_work
pending: cache_reap, psi_avgs_work, psi_avgs_work
======================================================
WARNING: possible circular locking dependency detected
5.2.0-rc4 #25 Not tainted
------------------------------------------------------
swapper/0/0 is trying to acquire lock:
00000000cd901cf4 (console_owner){-.-.}, at: log_next kernel/printk/printk.c:506 [inline]
00000000cd901cf4 (console_owner){-.-.}, at: console_unlock+0x3fc/0xec0 kernel/printk/printk.c:2450
but task is already holding lock:
000000007dc7e943 (&(&pool->lock)->rlock){-.-.}, at: show_workqueue_state.cold+0x859/0x11c5 kernel/workqueue.c:4727
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #4 (&(&pool->lock)->rlock){-.-.}:
__raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline]
_raw_spin_lock+0x2f/0x40 kernel/locking/spinlock.c:151
spin_lock include/linux/spinlock.h:338 [inline]
__queue_work+0x285/0x1200 kernel/workqueue.c:1449
queue_work_on+0x192/0x200 kernel/workqueue.c:1518
queue_work include/linux/workqueue.h:494 [inline]
schedule_work include/linux/workqueue.h:552 [inline]
put_pwq kernel/workqueue.c:1118 [inline]
put_pwq+0x178/0x1d0 kernel/workqueue.c:1103
put_pwq_unlocked.part.0+0x34/0x70 kernel/workqueue.c:1135
put_pwq_unlocked kernel/workqueue.c:1129 [inline]
destroy_workqueue+0x623/0x700 kernel/workqueue.c:4385
do_floppy_init drivers/block/floppy.c:4701 [inline]
floppy_async_init+0x2052/0x21a4 drivers/block/floppy.c:4719
async_run_entry_fn+0x124/0x570 kernel/async.c:123
process_one_work+0x989/0x1790 kernel/workqueue.c:2269
worker_thread+0x98/0xe40 kernel/workqueue.c:2415
kthread+0x354/0x420 kernel/kthread.c:255
ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352
-> #3 (&pool->lock/1){..-.}:
__raw_spin_lock include/linux/spinlock_api_smp.h:142 [inline]
_raw_spin_lock+0x2f/0x40 kernel/locking/spinlock.c:151
spin_lock include/linux/spinlock.h:338 [inline]
__queue_work+0x285/0x1200 kernel/workqueue.c:1449
queue_work_on+0x192/0x200 kernel/workqueue.c:1518
queue_work include/linux/workqueue.h:494 [inline]
tty_schedule_flip drivers/tty/tty_buffer.c:413 [inline]
tty_flip_buffer_push+0xc5/0x100 drivers/tty/tty_buffer.c:556
pty_write+0x1a6/0x200 drivers/tty/pty.c:125
n_tty_write+0xb0d/0x1140 drivers/tty/n_tty.c:2356
do_tty_write drivers/tty/tty_io.c:961 [inline]
tty_write+0x458/0x7a0 drivers/tty/tty_io.c:1045
__vfs_write+0x8a/0x110 fs/read_write.c:494
vfs_write+0x20c/0x580 fs/read_write.c:558
ksys_write+0x14f/0x290 fs/read_write.c:611
__do_sys_write fs/read_write.c:623 [inline]
__se_sys_write fs/read_write.c:620 [inline]
__x64_sys_write+0x73/0xb0 fs/read_write.c:620
do_syscall_64+0xfd/0x680 arch/x86/entry/common.c:301
entry_SYSCALL_64_after_hwframe+0x49/0xbe
-> #2 (&(&port->lock)->rlock){-.-.}:
__raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline]
_raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159
tty_port_tty_get+0x22/0x80 drivers/tty/tty_port.c:287
tty_port_default_wakeup+0x16/0x40 drivers/tty/tty_port.c:47
tty_port_tty_wakeup+0x57/0x70 drivers/tty/tty_port.c:387
uart_write_wakeup+0x46/0x70 drivers/tty/serial/serial_core.c:103
serial8250_tx_chars+0x495/0xaf0 drivers/tty/serial/8250/8250_port.c:1806
serial8250_handle_irq.part.0+0x1b9/0x2e0 drivers/tty/serial/8250/8250_port.c:1879
serial8250_handle_irq drivers/tty/serial/8250/8250_port.c:1865 [inline]
serial8250_default_handle_irq+0xc0/0x150 drivers/tty/serial/8250/8250_port.c:1895
serial8250_interrupt+0xf1/0x1a0 drivers/tty/serial/8250/8250_core.c:125
__handle_irq_event_percpu+0x144/0x8f0 kernel/irq/handle.c:149
handle_irq_event_percpu+0x74/0x160 kernel/irq/handle.c:189
handle_irq_event+0xa7/0x134 kernel/irq/handle.c:206
handle_edge_irq+0x25e/0x8d0 kernel/irq/chip.c:822
generic_handle_irq_desc include/linux/irqdesc.h:156 [inline]
handle_irq+0x39/0x50 arch/x86/kernel/irq_64.c:34
do_IRQ+0x99/0x1d0 arch/x86/kernel/irq.c:247
ret_from_intr+0x0/0x1e
native_safe_halt+0xe/0x10 arch/x86/include/asm/irqflags.h:60
arch_cpu_idle+0xa/0x10 arch/x86/kernel/process.c:571
default_idle_call+0x36/0x90 kernel/sched/idle.c:94
cpuidle_idle_call kernel/sched/idle.c:154 [inline]
do_idle+0x377/0x560 kernel/sched/idle.c:263
cpu_startup_entry+0x1b/0x20 kernel/sched/idle.c:354
start_secondary+0x34e/0x4c0 arch/x86/kernel/smpboot.c:265
secondary_startup_64+0xa4/0xb0 arch/x86/kernel/head_64.S:243
-> #1 (&port_lock_key){-.-.}:
__raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline]
_raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159
serial8250_console_write+0x253/0x9a0 drivers/tty/serial/8250/8250_port.c:3245
univ8250_console_write+0x5f/0x70 drivers/tty/serial/8250/8250_core.c:586
call_console_drivers kernel/printk/printk.c:1781 [inline]
console_unlock+0xb3e/0xec0 kernel/printk/printk.c:2463
vprintk_emit+0x2a0/0x700 kernel/printk/printk.c:1986
vprintk_default+0x28/0x30 kernel/printk/printk.c:2013
vprintk_func+0x7e/0x189 kernel/printk/printk_safe.c:386
printk+0xba/0xed kernel/printk/printk.c:2046
register_console+0x745/0xb50 kernel/printk/printk.c:2788
univ8250_console_init+0x3e/0x4b drivers/tty/serial/8250/8250_core.c:681
console_init+0x4f7/0x761 kernel/printk/printk.c:2874
start_kernel+0x5b4/0x893 init/main.c:688
x86_64_start_reservations+0x29/0x2b arch/x86/kernel/head64.c:470
x86_64_start_kernel+0x77/0x7b arch/x86/kernel/head64.c:451
secondary_startup_64+0xa4/0xb0 arch/x86/kernel/head_64.S:243
-> #0 (console_owner){-.-.}:
lock_acquire+0x16f/0x3f0 kernel/locking/lockdep.c:4303
console_lock_spinning_enable kernel/printk/printk.c:1644 [inline]
console_unlock+0x466/0xec0 kernel/printk/printk.c:2460
vprintk_emit+0x2a0/0x700 kernel/printk/printk.c:1986
vprintk_default+0x28/0x30 kernel/printk/printk.c:2013
vprintk_func+0x7e/0x189 kernel/printk/printk_safe.c:386
printk+0xba/0xed kernel/printk/printk.c:2046
show_pwq kernel/workqueue.c:4634 [inline]
show_workqueue_state.cold+0x9c4/0x11c5 kernel/workqueue.c:4729
wq_watchdog_timer_fn+0x516/0x5a0 kernel/workqueue.c:5743
call_timer_fn+0x193/0x720 kernel/time/timer.c:1322
expire_timers kernel/time/timer.c:1366 [inline]
__run_timers kernel/time/timer.c:1685 [inline]
__run_timers kernel/time/timer.c:1653 [inline]
run_timer_softirq+0xd3d/0x1740 kernel/time/timer.c:1700
__do_softirq+0x25c/0x94c kernel/softirq.c:292
invoke_softirq kernel/softirq.c:373 [inline]
irq_exit+0x180/0x1d0 kernel/softirq.c:413
exiting_irq arch/x86/include/asm/apic.h:536 [inline]
smp_apic_timer_interrupt+0x13b/0x550 arch/x86/kernel/apic/apic.c:1068
apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:806
native_safe_halt+0xe/0x10 arch/x86/include/asm/irqflags.h:60
arch_cpu_idle+0xa/0x10 arch/x86/kernel/process.c:571
default_idle_call+0x36/0x90 kernel/sched/idle.c:94
cpuidle_idle_call kernel/sched/idle.c:154 [inline]
do_idle+0x377/0x560 kernel/sched/idle.c:263
cpu_startup_entry+0x1b/0x20 kernel/sched/idle.c:354
rest_init+0x245/0x37b init/main.c:451
arch_call_rest_init+0xe/0x1b
start_kernel+0x854/0x893 init/main.c:761
x86_64_start_reservations+0x29/0x2b arch/x86/kernel/head64.c:470
x86_64_start_kernel+0x77/0x7b arch/x86/kernel/head64.c:451
secondary_startup_64+0xa4/0xb0 arch/x86/kernel/head_64.S:243
other info that might help us debug this:
Chain exists of:
console_owner --> &pool->lock/1 --> &(&pool->lock)->rlock
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(&(&pool->lock)->rlock);
lock(&pool->lock/1);
lock(&(&pool->lock)->rlock);
lock(console_owner);
*** DEADLOCK ***
4 locks held by swapper/0/0:
#0: 00000000a2307206 ((&wq_watchdog_timer)){+.-.}, at: lockdep_copy_map include/linux/lockdep.h:175 [inline]
#0: 00000000a2307206 ((&wq_watchdog_timer)){+.-.}, at: call_timer_fn+0xe0/0x720 kernel/time/timer.c:1312
#1: 0000000037ee3e30 (rcu_read_lock){....}, at: show_workqueue_state+0x0/0x120 kernel/workqueue.c:4599
#2: 000000007dc7e943 (&(&pool->lock)->rlock){-.-.}, at: show_workqueue_state.cold+0x859/0x11c5 kernel/workqueue.c:4727
#3: 0000000053b218da (console_lock){+.+.}, at: console_trylock_spinning kernel/printk/printk.c:1706 [inline]
#3: 0000000053b218da (console_lock){+.+.}, at: vprintk_emit+0x283/0x700 kernel/printk/printk.c:1985
stack backtrace:
CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.2.0-rc4 #25
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Call Trace:
<IRQ>
__dump_stack lib/dump_stack.c:77 [inline]
dump_stack+0x172/0x1f0 lib/dump_stack.c:113
print_circular_bug.cold+0x1cc/0x28f kernel/locking/lockdep.c:1565
check_prev_add kernel/locking/lockdep.c:2310 [inline]
check_prevs_add kernel/locking/lockdep.c:2418 [inline]
validate_chain kernel/locking/lockdep.c:2800 [inline]
__lock_acquire+0x3755/0x5490 kernel/locking/lockdep.c:3793
lock_acquire+0x16f/0x3f0 kernel/locking/lockdep.c:4303
console_lock_spinning_enable kernel/printk/printk.c:1644 [inline]
console_unlock+0x466/0xec0 kernel/printk/printk.c:2460
vprintk_emit+0x2a0/0x700 kernel/printk/printk.c:1986
vprintk_default+0x28/0x30 kernel/printk/printk.c:2013
vprintk_func+0x7e/0x189 kernel/printk/printk_safe.c:386
printk+0xba/0xed kernel/printk/printk.c:2046
show_pwq kernel/workqueue.c:4634 [inline]
show_workqueue_state.cold+0x9c4/0x11c5 kernel/workqueue.c:4729
wq_watchdog_timer_fn+0x516/0x5a0 kernel/workqueue.c:5743
call_timer_fn+0x193/0x720 kernel/time/timer.c:1322
expire_timers kernel/time/timer.c:1366 [inline]
__run_timers kernel/time/timer.c:1685 [inline]
__run_timers kernel/time/timer.c:1653 [inline]
run_timer_softirq+0xd3d/0x1740 kernel/time/timer.c:1700
__do_softirq+0x25c/0x94c kernel/softirq.c:292
invoke_softirq kernel/softirq.c:373 [inline]
irq_exit+0x180/0x1d0 kernel/softirq.c:413
exiting_irq arch/x86/include/asm/apic.h:536 [inline]
smp_apic_timer_interrupt+0x13b/0x550 arch/x86/kernel/apic/apic.c:1068
apic_timer_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:806
</IRQ>
RIP: 0010:native_safe_halt+0xe/0x10 arch/x86/include/asm/irqflags.h:61
Code: ff ff 48 89 df e8 a2 a3 8c fa eb 82 e9 07 00 00 00 0f 00 2d 64 4a 4b 00 f4 c3 66 90 e9 07 00 00 00 0f 00 2d 54 4a 4b 00 fb f4 <c3> 90 55 48 89 e5 41 57 41 56 41 55 41 54 53 e8 de 1b 42 fa e8 99
RSP: 0018:ffffffff88a07cf8 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13
RAX: 1ffffffff1164e81 RBX: ffffffff88a7a100 RCX: 0000000000000000
RDX: dffffc0000000000 RSI: 0000000000000006 RDI: ffffffff88a7a97c
RBP: ffffffff88a07d28 R08: ffffffff88a7a100 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
R13: ffffffff88b273f8 R14: 0000000000000000 R15: 0000000000000000
arch_cpu_idle+0xa/0x10 arch/x86/kernel/process.c:571
default_idle_call+0x36/0x90 kernel/sched/idle.c:94
cpuidle_idle_call kernel/sched/idle.c:154 [inline]
do_idle+0x377/0x560 kernel/sched/idle.c:263
cpu_startup_entry+0x1b/0x20 kernel/sched/idle.c:354
rest_init+0x245/0x37b init/main.c:451
arch_call_rest_init+0xe/0x1b
start_kernel+0x854/0x893 init/main.c:761
x86_64_start_reservations+0x29/0x2b arch/x86/kernel/head64.c:470
x86_64_start_kernel+0x77/0x7b arch/x86/kernel/head64.c:451
secondary_startup_64+0xa4/0xb0 arch/x86/kernel/head_64.S:243
workqueue events_unbound: flags=0x2
pwq 4: cpus=0-1 flags=0x4 nice=0 active=2/512
in-flight: 21:fsnotify_mark_destroy_workfn fsnotify_mark_destroy_workfn
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256
pending: gc_worker, do_cache_clean, check_lifetime
workqueue rcu_gp: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256
pending: srcu_invoke_callbacks
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256
pending: vmstat_update
workqueue dm_bufio_cache: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256
pending: work_fn
workqueue ipv6_addrconf: flags=0x40008
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/1
pending: addrconf_dad_work
delayed: addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_dad_work, addrconf_verify_work
pool 2: cpus=1 node=0 flags=0x0 nice=0 hung=178s workers=5 idle: 8963 23 8965 3486
pool 4: cpus=0-1 flags=0x4 nice=0 hung=1s workers=4 idle: 2200 8998 8961