diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c
index 3b32a0afa9798d3c416d9ae570e6d529f70e6697..221406acbac8efae295f39097ff995f7017e2483 100644
--- a/net/ipv4/udp_tunnel_nic.c
+++ b/net/ipv4/udp_tunnel_nic.c
@@ -32,7 +32,6 @@ struct udp_tunnel_nic_table_entry {
  * @lock:      protects all fields
  * @need_sync: at least one port start changed
  * @need_replay: space was freed, we need a replay of all ports
- * @work_pending: @work is currently scheduled
  * @n_tables:  number of tables under @entries
  * @missed:    bitmap of tables which overflown
  * @entries:   table of tables of ports currently offloaded
@@ -46,7 +45,6 @@ struct udp_tunnel_nic {
 
        u8 need_sync:1;
        u8 need_replay:1;
-       u8 work_pending:1;
 
        unsigned int n_tables;
        unsigned long missed;
@@ -301,11 +299,10 @@ __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 static void
 udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)
 {
-       if (!utn->need_sync || utn->work_pending)
+       if (!utn->need_sync)
                return;
 
        queue_work(udp_tunnel_nic_workqueue, &utn->work);
-       utn->work_pending = 1;
 }
 
 static bool
@@ -733,10 +730,17 @@ static void udp_tunnel_nic_device_sync_work(struct work_struct *work)
        struct udp_tunnel_nic *utn =
                container_of(work, struct udp_tunnel_nic, work);
 
-       rtnl_lock();
+       /* We cannot block on RTNL here, otherwise we would deadlock with
+        * udp_tunnel_nic_unregister() calling cancel_work_sync() while holding
+        * RTNL. Requeue if RTNL is contended, we will get another chance once
+        * the current holder releases it.
+        */
+       if (!rtnl_trylock()) {
+               queue_work(udp_tunnel_nic_workqueue, work);
+               return;
+       }
        mutex_lock(&utn->lock);
 
-       utn->work_pending = 0;
        __udp_tunnel_nic_device_sync(utn->dev, utn);
 
        if (utn->need_replay)
@@ -901,11 +905,11 @@ udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)
        udp_tunnel_nic_flush(dev, utn);
        udp_tunnel_nic_unlock(dev);
 
-       /* Wait for the work to be done using the state, netdev core will
-        * retry unregister until we give up our reference on this device.
+       /* Make sure no work is running or queued before freeing @utn.
+        * The work handler uses rtnl_trylock(), so it will not deadlock
+        * against the RTNL we are holding here.
         */
-       if (utn->work_pending)
-               return;
+       cancel_work_sync(&utn->work);
 
        udp_tunnel_nic_free(utn);
 release_dev: