MM_PAGED_POOL_INFO MmPagedPoolInfo;
SIZE_T MmAllocatedNonPagedPool;
ULONG MmSpecialPoolTag;
+ULONG MmConsumedPoolPercentage;
+BOOLEAN MmProtectFreedNonPagedPool;
/* PRIVATE FUNCTIONS **********************************************************/
+VOID
+NTAPI
+MiProtectFreeNonPagedPool(IN PVOID VirtualAddress,
+ IN ULONG PageCount)
+{
+ PMMPTE PointerPte, LastPte;
+ MMPTE TempPte;
+
+ /* If pool is physical, can't protect PTEs */
+ if (MI_IS_PHYSICAL_ADDRESS(VirtualAddress)) return;
+
+ /* Get PTE pointers and loop */
+ PointerPte = MiAddressToPte(VirtualAddress);
+ LastPte = PointerPte + PageCount;
+ do
+ {
+ /* Capture the PTE for safety */
+ TempPte = *PointerPte;
+
+ /* Mark it as an invalid PTE, set proto bit to recognize it as pool */
+ TempPte.u.Hard.Valid = 0;
+ TempPte.u.Soft.Prototype = 1;
+ MI_WRITE_INVALID_PTE(PointerPte, TempPte);
+ } while (++PointerPte < LastPte);
+
+ /* Flush the TLB */
+ KeFlushEntireTb(TRUE, TRUE);
+}
+
+BOOLEAN
+NTAPI
+MiUnProtectFreeNonPagedPool(IN PVOID VirtualAddress,
+ IN ULONG PageCount)
+{
+ PMMPTE PointerPte;
+ MMPTE TempPte;
+ PFN_NUMBER UnprotectedPages = 0;
+
+ /* If pool is physical, can't protect PTEs */
+ if (MI_IS_PHYSICAL_ADDRESS(VirtualAddress)) return FALSE;
+
+ /* Get, and capture the PTE */
+ PointerPte = MiAddressToPte(VirtualAddress);
+ TempPte = *PointerPte;
+
+ /* Loop protected PTEs */
+ while ((TempPte.u.Hard.Valid == 0) && (TempPte.u.Soft.Prototype == 1))
+ {
+ /* Unprotect the PTE */
+ TempPte.u.Hard.Valid = 1;
+ TempPte.u.Soft.Prototype = 0;
+ MI_WRITE_VALID_PTE(PointerPte, TempPte);
+
+ /* One more page */
+ if (++UnprotectedPages == PageCount) break;
+
+ /* Capture next PTE */
+ TempPte = *(++PointerPte);
+ }
+
+ /* Return if any pages were unprotected */
+ return UnprotectedPages ? TRUE : FALSE;
+}
+
+VOID
+FORCEINLINE
+MiProtectedPoolUnProtectLinks(IN PLIST_ENTRY Links,
+ OUT PVOID* PoolFlink,
+ OUT PVOID* PoolBlink)
+{
+ BOOLEAN Safe;
+ PVOID PoolVa;
+
+ /* Initialize variables */
+ *PoolFlink = *PoolBlink = NULL;
+
+ /* Check if the list has entries */
+ if (IsListEmpty(Links) == FALSE)
+ {
+ /* We are going to need to forward link to do an insert */
+ PoolVa = Links->Flink;
+
+ /* So make it safe to access */
+ Safe = MiUnProtectFreeNonPagedPool(PoolVa, 1);
+ if (Safe) PoolFlink = PoolVa;
+ }
+
+ /* Are we going to need a backward link too? */
+ if (Links != Links->Blink)
+ {
+ /* Get the head's backward link for the insert */
+ PoolVa = Links->Blink;
+
+ /* Make it safe to access */
+ Safe = MiUnProtectFreeNonPagedPool(PoolVa, 1);
+ if (Safe) PoolBlink = PoolVa;
+ }
+}
+
+VOID
+FORCEINLINE
+MiProtectedPoolProtectLinks(IN PVOID PoolFlink,
+ IN PVOID PoolBlink)
+{
+ /* Reprotect the pages, if they got unprotected earlier */
+ if (PoolFlink) MiProtectFreeNonPagedPool(PoolFlink, 1);
+ if (PoolBlink) MiProtectFreeNonPagedPool(PoolBlink, 1);
+}
+
+VOID
+NTAPI
+MiProtectedPoolInsertList(IN PLIST_ENTRY ListHead,
+ IN PLIST_ENTRY Entry,
+ IN BOOLEAN Critical)
+{
+ PVOID PoolFlink, PoolBlink;
+
+ /* Make the list accessible */
+ MiProtectedPoolUnProtectLinks(ListHead, &PoolFlink, &PoolBlink);
+
+ /* Now insert in the right position */
+ Critical ? InsertHeadList(ListHead, Entry) : InsertTailList(ListHead, Entry);
+
+ /* And reprotect the pages containing the free links */
+ MiProtectedPoolProtectLinks(PoolFlink, PoolBlink);
+}
+
+VOID
+NTAPI
+MiProtectedPoolRemoveEntryList(IN PLIST_ENTRY Entry)
+{
+ PVOID PoolFlink, PoolBlink;
+
+ /* Make the list accessible */
+ MiProtectedPoolUnProtectLinks(Entry, &PoolFlink, &PoolBlink);
+
+ /* Now remove */
+ RemoveEntryList(Entry);
+
+ /* And reprotect the pages containing the free links */
+ if (PoolFlink) MiProtectFreeNonPagedPool(PoolFlink, 1);
+ if (PoolBlink) MiProtectFreeNonPagedPool(PoolBlink, 1);
+}
+
VOID
NTAPI
MiInitializeNonPagedPoolThresholds(VOID)
FreeEntry = MmNonPagedPoolStart;
FirstEntry = FreeEntry;
FreeEntry->Size = PoolPages;
+ FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
FreeEntry->Owner = FirstEntry;
//
//
FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)FreeEntry + PAGE_SIZE);
FreeEntry->Owner = FirstEntry;
+ FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
}
//
KIRQL OldIrql;
PLIST_ENTRY NextEntry, NextHead, LastHead;
PMMPTE PointerPte, StartPte;
+ PMMPDE PointerPde;
+ ULONG EndAllocation;
MMPTE TempPte;
+ MMPDE TempPde;
PMMPFN Pfn1;
PVOID BaseVa, BaseVaStart;
PMMFREE_POOL_ENTRY FreeEntry;
//
// Handle paged pool
//
- if (PoolType == PagedPool)
+ if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
{
//
// Lock the paged pool mutex
//
// Get the page bit count
//
- i = ((SizeInPages - 1) / 1024) + 1;
+ i = ((SizeInPages - 1) / PTE_COUNT) + 1;
+ DPRINT1("Paged pool expansion: %d %x\n", i, SizeInPages);
//
// Check if there is enougn paged pool expansion space left
}
//
- // Get the template PTE we'll use to expand
+ // Get the template PDE we'll use to expand
//
- TempPte = ValidKernelPte;
+ TempPde = ValidKernelPde;
//
// Get the first PTE in expansion space
//
- PointerPte = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
- BaseVa = MiPteToAddress(PointerPte);
+ PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
+ BaseVa = MiPteToAddress(PointerPde);
BaseVaStart = BaseVa;
//
//
// It should not already be valid
//
- ASSERT(PointerPte->u.Hard.Valid == 0);
-
- //
- // Request a paged pool page and write the PFN for it
- //
- PageFrameNumber = MmAllocPage(MC_PPOOL);
- TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
+ ASSERT(PointerPde->u.Hard.Valid == 0);
+ /* Request a page */
+ DPRINT1("Requesting %d PDEs\n", i);
+ PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
+ TempPde.u.Hard.PageFrameNumber = PageFrameNumber;
+ DPRINT1("We have a PDE: %lx\n", PageFrameNumber);
+
+#if (_MI_PAGING_LEVELS >= 3)
+ /* On PAE/x64 systems, there's no double-buffering */
+ ASSERT(FALSE);
+#else
//
// Save it into our double-buffered system page directory
//
- MmSystemPagePtes[(ULONG_PTR)PointerPte & (PAGE_SIZE - 1) /
- sizeof(MMPTE)] = TempPte;
-
- //
- // Write the actual PTE now
- //
- *PointerPte++ = TempPte;
-
+ MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde;
+
+ /* Initialize the PFN */
+ MiInitializePfnForOtherProcess(PageFrameNumber,
+ PointerPde,
+ MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]);
+
+ /* Write the actual PDE now */
+ MI_WRITE_VALID_PTE(PointerPde, TempPde);
+#endif
//
// Move on to the next expansion address
//
+ PointerPde++;
BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
- } while (--i > 0);
+ i--;
+ } while (i > 0);
//
// Release the PFN database lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
-
+
//
// These pages are now available, clear their availablity bits
//
+ EndAllocation = (MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
+ MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
+ PTE_COUNT;
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
- (MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
- MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
- 1024,
- SizeInPages * 1024);
+ EndAllocation,
+ SizeInPages * PTE_COUNT);
//
// Update the next expansion location
// Update the end bitmap so we know the bounds of this allocation when
// the time comes to free it
//
- RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, i + SizeInPages - 1);
+ EndAllocation = i + SizeInPages - 1;
+ RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation);
//
// Now we can release the lock (it mainly protects the bitmap)
//
KeFlushEntireTb(TRUE, TRUE);
- //
- // Setup a demand-zero writable PTE
- //
- TempPte.u.Long = 0;
- MI_MAKE_WRITE_PAGE(&TempPte);
+ /* Setup a demand-zero writable PTE */
+ MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
//
// Find the first and last PTE, then loop them all
//
// Write the demand zero PTE and keep going
//
- *PointerPte++ = TempPte;
- } while (PointerPte < StartPte);
+ MI_WRITE_INVALID_PTE(PointerPte, TempPte);
+ } while (++PointerPte < StartPte);
//
// Return the allocation address to the caller
NextEntry = NextHead->Flink;
while (NextEntry != NextHead)
{
+ /* Is freed non paged pool enabled */
+ if (MmProtectFreedNonPagedPool)
+ {
+ /* We need to be able to touch this page, unprotect it */
+ MiUnProtectFreeNonPagedPool(NextEntry, 0);
+ }
+
//
// Grab the entry and see if it can handle our allocation
//
FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
+ ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
if (FreeEntry->Size >= SizeInPages)
{
//
BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
(FreeEntry->Size << PAGE_SHIFT));
- //
- // This is not a free page segment anymore
- //
- RemoveEntryList(&FreeEntry->List);
+ /* Remove the item from the list, depending if pool is protected */
+ MmProtectFreedNonPagedPool ?
+ MiProtectedPoolRemoveEntryList(&FreeEntry->List) :
+ RemoveEntryList(&FreeEntry->List);
//
// However, check if its' still got space left
//
if (FreeEntry->Size != 0)
{
- //
- // Insert it back into a different list, based on its pages
- //
+ /* Check which list to insert this entry into */
i = FreeEntry->Size - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
- InsertTailList (&MmNonPagedPoolFreeListHead[i],
- &FreeEntry->List);
+
+ /* Insert the entry into the free list head, check for prot. pool */
+ MmProtectFreedNonPagedPool ?
+ MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE) :
+ InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
+
+ /* Is freed non paged pool protected? */
+ if (MmProtectFreedNonPagedPool)
+ {
+ /* Protect the freed pool! */
+ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
+ }
}
//
ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
Pfn1->u3.e1.StartOfAllocation = 1;
+ /* Mark it as special pool if needed */
+ ASSERT(Pfn1->u4.VerifierAllocation == 0);
+ if (PoolType & 64) Pfn1->u4.VerifierAllocation = 1;
+
//
// Check if the allocation is larger than one page
//
// Try the next free page entry
//
NextEntry = FreeEntry->List.Flink;
+
+ /* Is freed non paged pool protected? */
+ if (MmProtectFreedNonPagedPool)
+ {
+ /* Protect the freed pool! */
+ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
+ }
}
} while (++NextHead < LastHead);
TempPte = ValidKernelPte;
do
{
- //
- // Allocate a page
- //
- PageFrameNumber = MmAllocPage(MC_NPPOOL);
+ /* Allocate a page */
+ PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
- //
- // Get the PFN entry for it
- //
+ /* Get the PFN entry for it and fill it out */
Pfn1 = MiGetPfnEntry(PageFrameNumber);
+ Pfn1->u3.e2.ReferenceCount = 1;
+ Pfn1->u2.ShareCount = 1;
+ Pfn1->PteAddress = PointerPte;
+ Pfn1->u3.e1.PageLocation = ActiveAndValid;
+ Pfn1->u4.VerifierAllocation = 0;
- //
- // Write the PTE for it
- //
+ /* Write the PTE for it */
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
- ASSERT(PointerPte->u.Hard.Valid == 0);
- ASSERT(TempPte.u.Hard.Valid == 1);
- *PointerPte++ = TempPte;
+ MI_WRITE_VALID_PTE(PointerPte++, TempPte);
} while (--SizeInPages > 0);
//
Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
Pfn1->u3.e1.StartOfAllocation = 1;
+ /* Mark it as a verifier allocation if needed */
+ ASSERT(Pfn1->u4.VerifierAllocation == 0);
+ if (PoolType & 64) Pfn1->u4.VerifierAllocation = 1;
+
//
// Release the PFN and nonpaged pool lock
//
//
NumberOfPages = End - i + 1;
+ /* Delete the actual pages */
+ PointerPte = MmPagedPoolInfo.FirstPteForPagedPool + i;
+ FreePages = MiDeleteSystemPageableVm(PointerPte, NumberOfPages, 0, NULL);
+ ASSERT(FreePages == NumberOfPages);
+
//
// Acquire the paged pool lock
//
}
else
{
+ /* Sanity check */
+ ASSERT((ULONG_PTR)StartingVa + NumberOfPages <= (ULONG_PTR)MmNonPagedPoolEnd);
+
+ /* Check if protected pool is enabled */
+ if (MmProtectFreedNonPagedPool)
+ {
+ /* The freed block will be merged, it must be made accessible */
+ MiUnProtectFreeNonPagedPool(MiPteToAddress(PointerPte), 0);
+ }
+
//
// Otherwise, our entire allocation must've fit within the initial non
// paged pool, or the expansion nonpaged pool, so get the PFN entry of
// the next allocation
//
- ASSERT((ULONG_PTR)StartingVa + NumberOfPages <= (ULONG_PTR)MmNonPagedPoolEnd);
if (PointerPte->u.Hard.Valid == 1)
{
//
//
FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)StartingVa +
(NumberOfPages << PAGE_SHIFT));
+ ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
ASSERT(FreeEntry->Owner == FreeEntry);
- //
- // Consume this entry's pages, and remove it from its free list
- //
+ /* Consume this entry's pages */
FreePages += FreeEntry->Size;
- RemoveEntryList (&FreeEntry->List);
+
+ /* Remove the item from the list, depending if pool is protected */
+ MmProtectFreedNonPagedPool ?
+ MiProtectedPoolRemoveEntryList(&FreeEntry->List) :
+ RemoveEntryList(&FreeEntry->List);
}
//
// Otherwise, get the PTE for the page right before our allocation
//
PointerPte -= NumberOfPages + 1;
+
+ /* Check if protected pool is enabled */
+ if (MmProtectFreedNonPagedPool)
+ {
+ /* The freed block will be merged, it must be made accessible */
+ MiUnProtectFreeNonPagedPool(MiPteToAddress(PointerPte), 0);
+ }
+
+ /* Check if this is valid pool, or a guard page */
if (PointerPte->u.Hard.Valid == 1)
{
//
// Get the free entry descriptor for that given page range
//
FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)StartingVa - PAGE_SIZE);
+ ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
FreeEntry = FreeEntry->Owner;
+ /* Check if protected pool is enabled */
+ if (MmProtectFreedNonPagedPool)
+ {
+ /* The freed block will be merged, it must be made accessible */
+ MiUnProtectFreeNonPagedPool(FreeEntry, 0);
+ }
+
//
// Check if the entry is small enough to be indexed on a free list
// If it is, we'll want to re-insert it, since we're about to
//
if (FreeEntry->Size < (MI_MAX_FREE_PAGE_LISTS - 1))
{
- //
- // Remove the list from where it is now
- //
- RemoveEntryList(&FreeEntry->List);
+ /* Remove the item from the list, depending if pool is protected */
+ MmProtectFreedNonPagedPool ?
+ MiProtectedPoolRemoveEntryList(&FreeEntry->List) :
+ RemoveEntryList(&FreeEntry->List);
//
// Update its size
i = (ULONG)(FreeEntry->Size - 1);
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
- //
- // Do it
- //
- InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
+ /* Insert the entry into the free list head, check for prot. pool */
+ MmProtectFreedNonPagedPool ?
+ MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE) :
+ InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
}
else
{
i = FreeEntry->Size - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
- //
- // And insert us
- //
- InsertTailList (&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
+ /* Insert the entry into the free list head, check for prot. pool */
+ MmProtectFreedNonPagedPool ?
+ MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE) :
+ InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
}
//
//
// Link back to the parent free entry, and keep going
//
- NextEntry->Owner = FreeEntry;
+ NextEntry->Owner = FreeEntry;
+ NextEntry->Signature = MM_FREE_POOL_SIGNATURE;
NextEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)NextEntry + PAGE_SIZE);
} while (NextEntry != LastEntry);
+ /* Is freed non paged pool protected? */
+ if (MmProtectFreedNonPagedPool)
+ {
+ /* Protect the freed pool! */
+ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
+ }
+
//
// We're done, release the lock and let the caller know how much we freed
//
projects / reactos.git / blobdiff