diff --git a/drivers/net/caif/caif_serial.c b/drivers/net/caif/caif_serial.c
index c398ac42eae9..700bf2434ad2 100644
--- a/drivers/net/caif/caif_serial.c
+++ b/drivers/net/caif/caif_serial.c
@@ -294,6 +294,11 @@ static void ser_release(struct work_struct *work)
 		list_for_each_entry_safe(ser, tmp, &list, node) {
 			dev_close(ser->dev);
 			unregister_netdevice(ser->dev);
+
+			while (test_bit(CAIF_SENDING, &ser->state))
+				cpu_relax();
+
+			tty_kref_put(ser->tty);
 			debugfs_deinit(ser);
 		}
 		rtnl_unlock();
@@ -355,8 +360,6 @@ static void ldisc_close(struct tty_struct *tty)
 {
 	struct ser_device *ser = tty->disc_data;
 
-	tty_kref_put(ser->tty);
-
 	spin_lock(&ser_lock);
 	list_move(&ser->node, &ser_release_list);
 	spin_unlock(&ser_lock);