bridge0: received packet on veth0_to_bridge with own address as source address (addr:3e:03:ae:c8:ea:85, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)
BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 147s!
BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=-20 stuck for 142s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=12/256 refcnt=13
pending: bpf_prog_free_deferred, 5*nsim_dev_hwstats_traffic_work, psi_avgs_work, ovs_dp_masks_rebalance, psi_avgs_work, ovs_dp_masks_rebalance, kfree_rcu_monitor, psi_avgs_work
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=13/256 refcnt=14
pending: 2*ovs_dp_masks_rebalance, psi_avgs_work, 2*nsim_dev_hwstats_traffic_work, vmstat_shepherd, psi_avgs_work, ovs_dp_masks_rebalance, rht_deferred_worker, ima_keys_handler, kfree_rcu_work, delayed_vfree_work, kfree_rcu_work
workqueue events_long: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=5/256 refcnt=6
pending: 5*defense_work_handler
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: 2*defense_work_handler
workqueue events_unbound: flags=0x2
pwq 4: cpus=0-1 flags=0x4 nice=0 active=15/512 refcnt=16
in-flight: 9414:nsim_dev_trap_report_work, 6733:cfg80211_wiphy_work cfg80211_wiphy_work, 9478:nsim_dev_trap_report_work, 9474:cfg80211_wiphy_work
pending: flush_memcg_stats_dwork, cfg80211_wiphy_work, crng_reseed, cfg80211_wiphy_work, 4*macvlan_process_broadcast, 2*idle_cull_fn
pwq 4: cpus=0-1 flags=0x4 nice=0 active=13/512 refcnt=14
in-flight: 1097:cfg80211_wiphy_work
pending: cfg80211_wiphy_work, 2*macvlan_process_broadcast, toggle_allocation_gate, 5*nsim_dev_trap_report_work, 2*cfg80211_wiphy_work, macvlan_process_broadcast
workqueue events_freezable: flags=0x4
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=5/256 refcnt=6
pending: neigh_managed_work, neigh_periodic_work, gc_worker, reg_check_chans_work, check_lifetime
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=4/256 refcnt=5
in-flight: 966:wg_ratelimiter_gc_entries
pending: neigh_managed_work, neigh_periodic_work, do_cache_clean
workqueue rcu_gp: flags=0x8
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
in-flight: 5763:wait_rcu_exp_gp
workqueue netns: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=4
in-flight: 9472:cleanup_net
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wb_workfn
workqueue kblockd: flags=0x18
pwq 3: cpus=1 node=0 flags=0x0 nice=-20 active=2/256 refcnt=3
pending: blk_mq_timeout_work, blk_mq_requeue_work
workqueue dm_bufio_cache: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: work_fn
workqueue ipv6_addrconf: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=8
pending: addrconf_verify_work
inactive: 4*addrconf_verify_work
workqueue krxrpcd: flags=0xa001a
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=9
pending: rxrpc_peer_keepalive_worker
inactive: 5*rxrpc_peer_keepalive_worker
workqueue bat_events: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=43
in-flight: 9488:batadv_nc_worker
inactive: 2*batadv_nc_worker, batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_nc_worker, batadv_bla_periodic_work, batadv_tt_purge, 4*batadv_mcast_mla_update, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 5*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 8*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_bla_periodic_work, batadv_dat_purge, 3*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
in-flight: 5882:wg_packet_tx_worker
pending: wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
in-flight: 786:wg_packet_tx_worker
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue hci2: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
pending: hci_conn_timeout
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_encrypt_worker, wg_packet_tx_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
in-flight: 5926:wg_packet_tx_worker
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue hci1: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
pending: hci_conn_timeout
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_tx_worker, wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pool 0: cpus=0 node=0 flags=0x0 nice=0 hung=114s workers=8 idle: 7401 9 5846 8 5824
pool 2: cpus=1 node=0 flags=0x0 nice=0 hung=148s workers=8 idle: 5792 28 5881 5825 5811 23
pool 4: cpus=0-1 flags=0x4 nice=0 hung=140s workers=31 idle: 1079 9415 9417 9412 9410 9409 11 64 42 7836 6737 49 6726 6735 6724 1086 6739 1128 6730 12 7835 9507 2897 6742
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/1:2 state:R running task stack:24616 pid:786 ppid:2 flags:0x00004000
Workqueue: wg-crypt-wg0 wg_packet_tx_worker
Call Trace:
context_switch kernel/sched/core.c:5380 [inline]
__schedule+0x14d2/0x44d0 kernel/sched/core.c:6699
preempt_schedule_common+0x82/0xc0 kernel/sched/core.c:6866
preempt_schedule+0xab/0xc0 kernel/sched/core.c:6890
preempt_schedule_thunk+0x1a/0x30 arch/x86/entry/thunk_64.S:45
__local_bh_enable_ip+0x13f/0x1c0 kernel/softirq.c:413
wg_socket_send_skb_to_peer+0x16b/0x1c0 drivers/net/wireguard/socket.c:184
wg_packet_create_data_done drivers/net/wireguard/send.c:251 [inline]
wg_packet_tx_worker+0x1c8/0x7c0 drivers/net/wireguard/send.c:276
process_one_work kernel/workqueue.c:2634 [inline]
process_scheduled_works+0xa45/0x15b0 kernel/workqueue.c:2711
worker_thread+0xa55/0xfc0 kernel/workqueue.c:2792
kthread+0x2fa/0x390 kernel/kthread.c:388
ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
pool 2:
task:kworker/1:7 state:R running task stack:23688 pid:5882 ppid:2 flags:0x00004000
Workqueue: wg-crypt-wg1 wg_packet_tx_worker
Call Trace:
context_switch kernel/sched/core.c:5380 [inline]
__schedule+0x14d2/0x44d0 kernel/sched/core.c:6699
preempt_schedule_common+0x82/0xc0 kernel/sched/core.c:6866
preempt_schedule+0xab/0xc0 kernel/sched/core.c:6890
preempt_schedule_thunk+0x1a/0x30 arch/x86/entry/thunk_64.S:45
__local_bh_enable_ip+0x13f/0x1c0 kernel/softirq.c:413
wg_socket_send_skb_to_peer+0x16b/0x1c0 drivers/net/wireguard/socket.c:184
wg_packet_create_data_done drivers/net/wireguard/send.c:251 [inline]
wg_packet_tx_worker+0x1c8/0x7c0 drivers/net/wireguard/send.c:276
process_one_work kernel/workqueue.c:2634 [inline]
process_scheduled_works+0xa45/0x15b0 kernel/workqueue.c:2711
worker_thread+0xa55/0xfc0 kernel/workqueue.c:2792
kthread+0x2fa/0x390 kernel/kthread.c:388
ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
net_ratelimit: 1677 callbacks suppressed
bridge0: received packet on veth0_to_bridge with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:3e:03:ae:c8:ea:85, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:3e:03:ae:c8:ea:85, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:3e:03:ae:c8:ea:85, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)
net_ratelimit: 1958 callbacks suppressed
bridge0: received packet on veth0_to_bridge with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:3e:03:ae:c8:ea:85, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:3e:03:ae:c8:ea:85, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:aa:aa:aa:aa:aa:0c, vlan:0)
bridge0: received packet on veth0_to_bridge with own address as source address (addr:3e:03:ae:c8:ea:85, vlan:0)
bridge0: received packet on bridge_slave_0 with own address as source address (addr:aa:aa:aa:aa:aa:1b, vlan:0)