diff --git a/mm/workingset.c b/mm/workingset.c index 0ec205a1ae92..d64490cd987d 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -199,28 +199,49 @@ static unsigned int bucket_order __read_mostly; static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, bool workingset) { + pr_err("PACK_SHADOW: CREATING SHADOW\n"); + pr_err(" memcgid=%d node_id=%d eviction=0x%lx workingset=%d\n", + memcgid, pgdat->node_id, eviction, workingset); eviction &= EVICTION_MASK; eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; eviction = (eviction << NODES_SHIFT) | pgdat->node_id; eviction = (eviction << WORKINGSET_SHIFT) | workingset; - - return xa_mk_value(eviction); + void *shadow = xa_mk_value(eviction); + pr_err(" Final packed shadow=0x%lx (raw eviction=0x%lx)\n", + (unsigned long)shadow, eviction); + if ((unsigned long)shadow == 0x41) { + pr_err("*** BUG: CREATED SHADOW 0x41! ***\n"); + dump_stack(); + } + return shadow; } static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, unsigned long *evictionp, bool *workingsetp) { + pr_err("UNPACK_SHADOW: READING SHADOW\n"); + pr_err(" shadow=0x%lx\n", (unsigned long)shadow); unsigned long entry = xa_to_value(shadow); int memcgid, nid; bool workingset; - + // CRITICAL: Detect if we're reading the bad 0x41 shadow! + if ((unsigned long)shadow == 0x41) { + pr_err("*** BUG: UNPACKING CORRUPTED SHADOW 0x41! ***\n"); + dump_stack(); + } workingset = entry & ((1UL << WORKINGSET_SHIFT) - 1); entry >>= WORKINGSET_SHIFT; nid = entry & ((1UL << NODES_SHIFT) - 1); entry >>= NODES_SHIFT; memcgid = entry & ((1UL << MEM_CGROUP_ID_SHIFT) - 1); entry >>= MEM_CGROUP_ID_SHIFT; - + pr_err(" Unpacked: memcgid=%d nid=%d eviction=0x%lx workingset=%d\n", + memcgid, nid, entry, workingset); + pr_err(" NODE_DATA(%d)=%px\n", nid, NODE_DATA(nid)); + if (nid >= MAX_NUMNODES || !NODE_DATA(nid)) { + pr_err("*** BUG: INVALID NODE ID %d! ***\n", nid); + dump_stack(); + } *memcgidp = memcgid; *pgdat = NODE_DATA(nid); *evictionp = entry; @@ -231,6 +252,8 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, static void *lru_gen_eviction(struct folio *folio) { + pr_err("LRU_GEN_EVICTION: ENTERED\n"); + pr_err(" folio=%px node=%d\n", folio, folio_nid(folio)); int hist; unsigned long token; unsigned long min_seq; @@ -250,11 +273,15 @@ static void *lru_gen_eviction(struct folio *folio) lrugen = &lruvec->lrugen; min_seq = READ_ONCE(lrugen->min_seq[type]); token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); - + pr_err("LRU_GEN_EVICTION: min_seq=0x%lx refs=%d tier=%d\n", + min_seq, refs, tier); + pr_err(" token=0x%lx (will be eviction parameter)\n", token); hist = lru_hist_from_seq(min_seq); atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); - - return pack_shadow(mem_cgroup_id(memcg), pgdat, token, workingset); + void *shadow = pack_shadow(mem_cgroup_id(memcg), pgdat, token, workingset); + pr_err("LRU_GEN_EVICTION: Returning shadow=0x%lx\n", (unsigned long)shadow); + return shadow; + //return pack_shadow(mem_cgroup_id(memcg), pgdat, token, workingset); } /* @@ -289,6 +316,13 @@ static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec, static void lru_gen_refault(struct folio *folio, void *shadow) { + pr_err("LRU_GEN_REFAULT: ENTERED\n"); + pr_err(" folio=%px shadow=0x%lx\n", folio, (unsigned long)shadow); + + if ((unsigned long)shadow == 0x41) { + pr_err("*** BUG: LRU_GEN_REFAULT received corrupted shadow 0x41! ***\n"); + //dump_stack(); + } bool recent; int hist, tier, refs; bool workingset; @@ -299,8 +333,11 @@ static void lru_gen_refault(struct folio *folio, void *shadow) int delta = folio_nr_pages(folio); rcu_read_lock(); + pr_err("LRU_GEN_REFAULT: Calling lru_gen_test_recent\n"); recent = lru_gen_test_recent(shadow, &lruvec, &token, &workingset); + pr_err("LRU_GEN_REFAULT: lru_gen_test_recent returned %d\n", recent); + pr_err(" lruvec=%px token=0x%lx workingset=%d\n", lruvec, token, workingset); if (!lruvec || lruvec != folio_lruvec(folio)) goto unlock; mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); @@ -539,6 +576,12 @@ bool workingset_test_recent(void *shadow, bool file, bool *workingset, */ void workingset_refault(struct folio *folio, void *shadow) { + pr_err("WORKINGSET_REFAULT: ENTERED\n"); + pr_err(" folio=%px shadow=0x%lx\n", folio, (unsigned long)shadow); + if ((unsigned long)shadow == 0x41) { + pr_err("*** BUG: WORKINGSET_REFAULT received corrupted shadow 0x41! ***\n"); + dump_stack(); + } bool file = folio_is_file_lru(folio); struct pglist_data *pgdat; struct mem_cgroup *memcg; @@ -549,9 +592,13 @@ void workingset_refault(struct folio *folio, void *shadow) VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); if (lru_gen_enabled()) { + pr_err("WORKINGSET_REFAULT: LRU_GEN enabled, calling lru_gen_refault\n"); lru_gen_refault(folio, shadow); + pr_err("WORKINGSET_REFAULT: lru_gen_refault returned\n"); + return; } + pr_err("WORKINGSET_REFAULT: Using regular (non-LRU_GEN) path\n"); /* * The activation decision for this folio is made at the level @@ -568,6 +615,7 @@ void workingset_refault(struct folio *folio, void *shadow) lruvec = mem_cgroup_lruvec(memcg, pgdat); mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr); + pr_err("WORKINGSET_REFAULT: Calling workingset_test_recent\n"); if (!workingset_test_recent(shadow, file, &workingset, true)) return; @@ -578,6 +626,7 @@ void workingset_refault(struct folio *folio, void *shadow) /* Folio was active prior to eviction */ if (workingset) { + pr_err("WORKINGSET_REFAULT: Folio was workingset, restoring\n"); folio_set_workingset(folio); /* * XXX: Move to folio_add_lru() when it supports new vs @@ -586,6 +635,7 @@ void workingset_refault(struct folio *folio, void *shadow) lru_note_cost_refault(folio); mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr); } + pr_err("WORKINGSET_REFAULT: EXITING\n"); } /**