diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 054a144cb569..6ce3368bdbee 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2405,6 +2405,9 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
 {
 	pgtable_t pgtable;
 
+	if (unlikely(!pmd_huge_pte(mm, pmd)))
+		return;
+
 	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
 	pte_free(mm, pgtable);
 	mm_dec_nr_ptes(mm);
@@ -2436,7 +2439,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			zap_deposited_table(tlb->mm, pmd);
 		spin_unlock(ptl);
 	} else if (is_huge_zero_pmd(orig_pmd)) {
-		if (!vma_is_dax(vma) || arch_needs_pgtable_deposit())
+		if (!is_vm_hugetlb_page(vma) &&
+		    (!vma_is_dax(vma) || arch_needs_pgtable_deposit()))
 			zap_deposited_table(tlb->mm, pmd);
 		spin_unlock(ptl);
 	} else {
@@ -2447,38 +2451,48 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			struct page *page = pmd_page(orig_pmd);
 
 			folio = page_folio(page);
-			folio_remove_rmap_pmd(folio, page, vma);
+			if (!folio_test_hugetlb(folio))
+				folio_remove_rmap_pmd(folio, page, vma);
 			WARN_ON_ONCE(folio_mapcount(folio) < 0);
 			VM_BUG_ON_PAGE(!PageHead(page), page);
-		} else if (pmd_is_valid_softleaf(orig_pmd)) {
-			const softleaf_t entry = softleaf_from_pmd(orig_pmd);
-
-			folio = softleaf_to_folio(entry);
+		} else {
 			flush_needed = 0;
 
-			if (!thp_migration_supported())
-				WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+			if (pmd_is_valid_softleaf(orig_pmd)) {
+				const softleaf_t entry = softleaf_from_pmd(orig_pmd);
+
+				folio = softleaf_to_folio(entry);
+
+				if (!thp_migration_supported())
+					WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+			}
 		}
 
-		if (folio_test_anon(folio)) {
-			zap_deposited_table(tlb->mm, pmd);
-			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
+		if (folio && folio_test_anon(folio)) {
+			if (!folio_test_hugetlb(folio)) {
+				zap_deposited_table(tlb->mm, pmd);
+				add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
+			}
 		} else {
-			if (arch_needs_pgtable_deposit())
+			if (!is_vm_hugetlb_page(vma) &&
+			    (vma_is_anonymous(vma) || arch_needs_pgtable_deposit()))
 				zap_deposited_table(tlb->mm, pmd);
-			add_mm_counter(tlb->mm, mm_counter_file(folio),
-				       -HPAGE_PMD_NR);
 
-			/*
-			 * Use flush_needed to indicate whether the PMD entry
-			 * is present, instead of checking pmd_present() again.
-			 */
-			if (flush_needed && pmd_young(orig_pmd) &&
-			    likely(vma_has_recency(vma)))
-				folio_mark_accessed(folio);
+			if (folio && !folio_test_hugetlb(folio)) {
+				add_mm_counter(tlb->mm, mm_counter_file(folio),
+					       -HPAGE_PMD_NR);
+
+				/*
+				 * Use flush_needed to indicate whether the PMD entry
+				 * is present, instead of checking pmd_present() again.
+				 */
+				if (flush_needed && pmd_young(orig_pmd) &&
+				    likely(vma_has_recency(vma)))
+					folio_mark_accessed(folio);
+			}
 		}
 
-		if (folio_is_device_private(folio)) {
+		if (folio && folio_is_device_private(folio)) {
 			folio_remove_rmap_pmd(folio, &folio->page, vma);
 			WARN_ON_ONCE(folio_mapcount(folio) < 0);
 			folio_put(folio);