sched: DL replenish lagged too much
BUG: workqueue lockup - pool cpus=0 node=0 flags=0x2 nice=0 stuck for 37s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=14 refcnt=15
    in-flight: 9:nsim_fib_event_work ,11:nsim_fib_event_work
    pending: psi_avgs_work, vmstat_shepherd, 2*nsim_dev_hwstats_traffic_work, psi_avgs_work, drm_fb_helper_damage_work, 4*ovs_dp_masks_rebalance, free_obj_work, rht_deferred_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=4 refcnt=5
    in-flight: 24:free_ipc
    pending: 3*nsim_dev_hwstats_traffic_work
workqueue events_long: flags=0x0
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: 3*defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: 3*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=8 refcnt=9
    in-flight: 12:cfg80211_wiphy_work cfg80211_wiphy_work ,15:nsim_dev_trap_report_work ,6854:cfg80211_wiphy_work cfg80211_wiphy_work ,42:linkwatch_event ,6671:cfg80211_wiphy_work cfg80211_wiphy_work
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=14 refcnt=15
    in-flight: 6855:cfg80211_wiphy_work cfg80211_wiphy_work ,718:cfg80211_wiphy_work cfg80211_wiphy_work ,2179:cfg80211_wiphy_work cfg80211_wiphy_work ,6859:nsim_dev_trap_report_work
    pending: nsim_dev_trap_report_work, macvlan_process_broadcast, flush_memcg_stats_dwork, macvlan_process_broadcast, toggle_allocation_gate, 2*nsim_dev_trap_report_work
workqueue events_power_efficient: flags=0x82
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    in-flight: 6852:neigh_periodic_work ,6856:gc_worker
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=3 refcnt=4
    in-flight: 6853:reg_check_chans_work
    pending: do_cache_clean, fb_flashcursor
workqueue rcu_gp: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: srcu_invoke_callbacks
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=4 MAYDAY
    pending: 2*kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue writeback: flags=0x4a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    in-flight: 6858:wb_workfn
workqueue mld: flags=0x40008
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=103
    pending: mld_ifc_work
    inactive: 2*mld_dad_work, 4*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 6*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, 2*mld_dad_work, 7*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, mld_dad_work, mld_ifc_work, 2*mld_dad_work, mld_ifc_work, mld_dad_work, 3*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 11*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 6*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 6*mld_ifc_work, mld_dad_work, 10*mld_ifc_work, mld_dad_work, 4*mld_ifc_work, mld_dad_work, 11*mld_ifc_work
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=29
    in-flight: 4199(RESCUER):mld_ifc_work
    inactive: 3*mld_ifc_work, 3*mld_dad_work, 3*mld_ifc_work, mld_dad_work, 8*mld_ifc_work, mld_dad_work, 7*mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=403
    in-flight: 41:addrconf_dad_work
    inactive: 399*addrconf_dad_work
workqueue krxrpcd: flags=0x2001a
  pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5
    in-flight: 52:rxrpc_peer_keepalive_worker
    inactive: rxrpc_peer_keepalive_worker
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=48
    in-flight: 4275(RESCUER):batadv_tt_purge
    inactive: batadv_purge_orig, 2*batadv_tt_purge, 2*batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 2*batadv_nc_worker, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_nc_worker, 3*batadv_mcast_mla_update, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_mcast_mla_update, 11*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=3 MAYDAY
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    in-flight: 6589:wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=3
    in-flight: 6718(RESCUER):wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 26:wg_packet_handshake_receive_worker wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=3 MAYDAY
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x24
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 6804:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x2 nice=0 active=1 refcnt=2
    in-flight: 6666:wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pool 2: cpus=0 node=0 flags=0x2 nice=0 hung=37s workers=9 manager: 6845 idle: 6607 6805 6857 6680 6808
pool 6: cpus=1 node=0 flags=0x2 nice=0 hung=0s workers=5 manager: 6692
pool 8: cpus=0-1 flags=0x4 nice=0 hung=0s workers=15 idle: 6860
pool 9: cpus=0-1 node=0 flags=0x4 nice=-20 hung=0s workers=7 idle: 6132 6694 6682 6632 6690 6686
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/0:4     state:R  running task     stack:0     pid:6804  tgid:6804  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: wg-kex-wg2 wg_packet_handshake_receive_worker
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T)
 context_switch kernel/sched/core.c:5357 [inline]
 __schedule+0x13b0/0x2864 kernel/sched/core.c:6961
 preempt_schedule_common+0xe0/0x194 kernel/sched/core.c:7145
 preempt_schedule+0x60/0x80 kernel/sched/core.c:7169
 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:414
 local_bh_enable+0x28/0x34 include/linux/bottom_half.h:33
 rcu_read_unlock_bh include/linux/rcupdate.h:910 [inline]
 mod_peer_timer+0x1e0/0x220 drivers/net/wireguard/timers.c:38
 wg_timers_any_authenticated_packet_traversal+0x98/0xc0 drivers/net/wireguard/timers.c:218
 wg_packet_send_handshake_response+0x114/0x24c drivers/net/wireguard/send.c:99
 wg_receive_handshake_packet drivers/net/wireguard/receive.c:154 [inline]
 wg_packet_handshake_receive_worker+0x4fc/0xc20 drivers/net/wireguard/receive.c:213
 process_one_work+0x7e8/0x155c kernel/workqueue.c:3236
 process_scheduled_works kernel/workqueue.c:3319 [inline]
 worker_thread+0x958/0xed8 kernel/workqueue.c:3400
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844
pool 2:
task:kworker/0:0     state:R  running task     stack:0     pid:9     tgid:9     ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T)
 context_switch kernel/sched/core.c:5357 [inline]
 __schedule+0x13b0/0x2864 kernel/sched/core.c:6961
 __schedule_loop kernel/sched/core.c:7043 [inline]
 schedule+0xb4/0x230 kernel/sched/core.c:7058
 schedule_timeout+0x128/0x270 kernel/time/sleep_timeout.c:99
 schedule_timeout_uninterruptible+0x7c/0xc0 kernel/time/sleep_timeout.c:158
 msleep+0x3c/0x68 kernel/time/sleep_timeout.c:318
 nsim_fib4_rt_add drivers/net/netdevsim/fib.c:369 [inline]
 nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:432 [inline]
 nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline]
 nsim_fib_event drivers/net/netdevsim/fib.c:884 [inline]
 nsim_fib_event_work+0x13a4/0x25b8 drivers/net/netdevsim/fib.c:1493
 process_one_work+0x7e8/0x155c kernel/workqueue.c:3236
 process_scheduled_works kernel/workqueue.c:3319 [inline]
 worker_thread+0x990/0xed8 kernel/workqueue.c:3400
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844
pool 2:
task:kworker/0:1     state:R  running task     stack:0     pid:11    tgid:11    ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T)
 context_switch kernel/sched/core.c:5357 [inline]
 __schedule+0x13b0/0x2864 kernel/sched/core.c:6961
 preempt_schedule_irq+0x80/0x188 kernel/sched/core.c:7288
 raw_irqentry_exit_cond_resched+0x30/0x44 kernel/entry/common.c:168
 irqentry_exit+0xd4/0x108 kernel/entry/common.c:211
 __exit_to_kernel_mode arch/arm64/kernel/entry-common.c:68 [inline]
 exit_to_kernel_mode+0x10/0x1c arch/arm64/kernel/entry-common.c:75
 __el1_irq arch/arm64/kernel/entry-common.c:522 [inline]
 el1_interrupt+0x4c/0x60 arch/arm64/kernel/entry-common.c:532
 el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:537
 el1h_64_irq+0x6c/0x70 arch/arm64/kernel/entry.S:592
 __sanitizer_cov_trace_pc+0x80/0x84 kernel/kcov.c:235 (P)
 rhashtable_jhash2+0x34/0x48 lib/rhashtable.c:982
 rht_head_hashfn include/linux/rhashtable.h:-1 [inline]
 __rhashtable_insert_fast include/linux/rhashtable.h:725 [inline]
 rhashtable_insert_fast+0x20c/0xb80 include/linux/rhashtable.h:834
 nsim_fib4_rt_add drivers/net/netdevsim/fib.c:362 [inline]
 nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:432 [inline]
 nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline]
 nsim_fib_event drivers/net/netdevsim/fib.c:884 [inline]
 nsim_fib_event_work+0xe00/0x25b8 drivers/net/netdevsim/fib.c:1493
 process_one_work+0x7e8/0x155c kernel/workqueue.c:3236
 process_scheduled_works kernel/workqueue.c:3319 [inline]
 worker_thread+0x990/0xed8 kernel/workqueue.c:3400
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844