* PURPOSE: Memory Manager Initialization for amd64
*
* PROGRAMMERS: Timo kreuzer (timo.kreuzer@reactos.org)
+ * ReactOS Portable Systems Group
*/
/* INCLUDES ***************************************************************/
#include <ntoskrnl.h>
-#define NDEBUG
+//#define NDEBUG
#include <debug.h>
#include "../ARM3/miarm.h"
+#ifndef KDBG
extern PMMPTE MmDebugPte;
+#endif
/* GLOBALS *****************************************************************/
-ULONG64 MmUserProbeAddress = 0x7FFFFFF0000ULL;
-PVOID MmHighestUserAddress = (PVOID)0x7FFFFFEFFFFULL;
-PVOID MmSystemRangeStart = (PVOID)KSEG0_BASE; // FFFF080000000000
-
-/* Size of session view, pool, and image */
-ULONG64 MmSessionSize = MI_SESSION_SIZE;
-ULONG64 MmSessionViewSize = MI_SESSION_VIEW_SIZE;
-ULONG64 MmSessionPoolSize = MI_SESSION_POOL_SIZE;
-ULONG64 MmSessionImageSize = MI_SESSION_IMAGE_SIZE;
-
-/* Session space addresses */
-PVOID MiSessionSpaceEnd = MI_SESSION_SPACE_END; // FFFFF98000000000
-PVOID MiSessionImageEnd; // FFFFF98000000000 = MiSessionSpaceEnd
-PVOID MiSessionImageStart; // ?FFFFF97FFF000000 = MiSessionImageEnd - MmSessionImageSize
-PVOID MiSessionViewEnd; // FFFFF97FFF000000
-PVOID MiSessionViewStart; // = MiSessionViewEnd - MmSessionViewSize
-PVOID MiSessionPoolEnd; // = MiSessionViewStart
-PVOID MiSessionPoolStart; // FFFFF90000000000 = MiSessionPoolEnd - MmSessionPoolSize
-PVOID MmSessionBase; // FFFFF90000000000 = MiSessionPoolStart
-
-/* System view */
-ULONG64 MmSystemViewSize = MI_SYSTEM_VIEW_SIZE;
-PVOID MiSystemViewStart;
-
+/* Sizes */
+ULONG64 MmBootImageSize;
ULONG64 MmMinimumNonPagedPoolSize = 256 * 1024;
ULONG64 MmSizeOfNonPagedPoolInBytes;
ULONG64 MmMaximumNonPagedPoolInBytes;
ULONG64 MmMinAdditionNonPagedPoolPerMb = 32 * 1024;
ULONG64 MmMaxAdditionNonPagedPoolPerMb = 400 * 1024;
ULONG64 MmDefaultMaximumNonPagedPool = 1024 * 1024;
-PVOID MmNonPagedSystemStart;
-PVOID MmNonPagedPoolStart;
-PVOID MmNonPagedPoolExpansionStart;
-PVOID MmNonPagedPoolEnd = MI_NONPAGED_POOL_END;
-
+ULONG64 MmSessionSize = MI_SESSION_SIZE;
+ULONG64 MmSessionViewSize = MI_SESSION_VIEW_SIZE;
+ULONG64 MmSessionPoolSize = MI_SESSION_POOL_SIZE;
+ULONG64 MmSessionImageSize = MI_SESSION_IMAGE_SIZE;
+ULONG64 MmSystemViewSize = MI_SYSTEM_VIEW_SIZE;
ULONG64 MmSizeOfPagedPoolInBytes = MI_MIN_INIT_PAGED_POOLSIZE;
-PVOID MmPagedPoolStart = MI_PAGED_POOL_START;
-PVOID MmPagedPoolEnd;
+ULONG64 MiNonPagedSystemSize;
+/* Address ranges */
+ULONG64 MmUserProbeAddress = 0x7FFFFFF0000ULL;
+PVOID MmHighestUserAddress = (PVOID)0x7FFFFFEFFFFULL;
+PVOID MmSystemRangeStart = (PVOID)0xFFFF080000000000ULL;
+PVOID MmSessionBase; // FFFFF90000000000 = MiSessionPoolStart
+PVOID MiSessionPoolStart; // FFFFF90000000000 = MiSessionPoolEnd - MmSessionPoolSize
+PVOID MiSessionPoolEnd; // = MiSessionViewStart
+PVOID MiSessionViewStart; // = MiSessionViewEnd - MmSessionViewSize
+PVOID MiSessionViewEnd; // FFFFF97FFF000000
+PVOID MiSessionImageStart; // ?FFFFF97FFF000000 = MiSessionImageEnd - MmSessionImageSize
+PVOID MiSessionImageEnd; // FFFFF98000000000 = MiSessionSpaceEnd
+PVOID MiSessionSpaceEnd = MI_SESSION_SPACE_END; // FFFFF98000000000
+PVOID MmSystemCacheStart; // FFFFF98000000000
+PVOID MmSystemCacheEnd; // FFFFFA8000000000
+PVOID MmPagedPoolStart = MI_PAGED_POOL_START; // FFFFFA8000000000
+PVOID MmPagedPoolEnd; // FFFFFAA000000000
+PVOID MiSystemViewStart;
+PVOID MmNonPagedSystemStart; // FFFFFAA000000000
+PVOID MmNonPagedPoolStart;
+PVOID MmNonPagedPoolExpansionStart;
+PVOID MmNonPagedPoolEnd = MI_NONPAGED_POOL_END; // 0xFFFFFAE000000000
-ULONG64 MmBootImageSize;
PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock;
-RTL_BITMAP MiPfnBitMap;
ULONG MmNumberOfPhysicalPages, MmHighestPhysicalPage, MmLowestPhysicalPage = -1; // FIXME: ULONG64
-ULONG64 MmNumberOfSystemPtes;
-PMMPTE MmSystemPagePtes;
-ULONG64 MxPfnAllocation;
-ULONG64 MxPfnSizeInBytes;
-PVOID MmSystemCacheStart;
-PVOID MmSystemCacheEnd;
+ULONG MmNumberOfSystemPtes;
+PMMPTE MmSystemPagePtes;
MMSUPPORT MmSystemCacheWs;
-
-///////////////////////////////////////////////
+RTL_BITMAP MiPfnBitMap;
+ULONG64 MxPfnAllocation;
+ULONG64 MxPfnSizeInBytes;
PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor;
MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor;
+ULONG MiNumberDescriptors = 0;
+BOOLEAN MiIncludeType[LoaderMaximum];
PFN_NUMBER MxFreePageBase;
ULONG64 MxFreePageCount = 0;
+ULONG MxPhase = 0;
+
+PFN_NUMBER MmSystemPageDirectory;
+PFN_NUMBER MmSizeOfPagedPoolInPages = MI_MIN_INIT_PAGED_POOLSIZE / PAGE_SIZE;
+
+
+/* FUNCTIONS *****************************************************************/
ULONG
NoDbgPrint(const char *Format, ...)
PMEMORY_ALLOCATION_DESCRIPTOR MdBlock;
PLIST_ENTRY ListEntry;
PFN_NUMBER LastPage;
+ ULONG i;
+
+ /* Get the size of the boot loader's image allocations */
+ MmBootImageSize = KeLoaderBlock->Extension->LoaderPagesSpanned * PAGE_SIZE;
+ MmBootImageSize = ROUND_UP(MmBootImageSize, 4 * 1024 * 1024);
+
+ /* Instantiate memory that we don't consider RAM/usable */
+ for (i = 0; i < LoaderMaximum; i++) MiIncludeType[i] = TRUE;
+ MiIncludeType[LoaderBad] = FALSE;
+ MiIncludeType[LoaderFirmwarePermanent] = FALSE;
+ MiIncludeType[LoaderSpecialMemory] = FALSE;
+ MiIncludeType[LoaderBBTMemory] = FALSE;
/* Loop the memory descriptors */
for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
MEMORY_ALLOCATION_DESCRIPTOR,
ListEntry);
+ /* Count it */
+ MiNumberDescriptors++;
+
/* Skip pages that are not part of the PFN database */
- if ((MdBlock->MemoryType == LoaderFirmwarePermanent) ||
- (MdBlock->MemoryType == LoaderBBTMemory) ||
- (MdBlock->MemoryType == LoaderHALCachedMemory) ||
- (MdBlock->MemoryType == LoaderSpecialMemory) ||
- (MdBlock->MemoryType == LoaderBad))
+ if (!MiIncludeType[MdBlock->MemoryType])
{
continue;
}
PFN_NUMBER
NTAPI
-MxGetNextPage(IN PFN_NUMBER PageCount)
+MxAllocEarlyPage()
{
PFN_NUMBER Pfn;
/* Make sure we have enough pages */
- if (PageCount > MxFreePageCount)
+ if (!MxFreePageCount)
{
/* Crash the system */
KeBugCheckEx(INSTALL_MORE_MEMORY,
MmNumberOfPhysicalPages,
MxFreeDescriptor->PageCount,
MxOldFreeDescriptor.PageCount,
- PageCount);
+ 1);
}
/* Use our lowest usable free pages */
Pfn = MxFreePageBase;
- MxFreePageBase += PageCount;
- MxFreePageCount -= PageCount;
+ MxFreePageBase++;
+ MxFreePageCount--;
return Pfn;
}
+PFN_NUMBER
+NTAPI
+MxAllocPage()
+{
+ return (MxPhase == 0) ? MxAllocEarlyPage() : MmAllocPage(MC_SYSTEM, 0);
+}
+
PMMPTE
NTAPI
MxGetPte(PVOID Address)
{
PMMPTE Pte;
- MMPTE TmpPte;
+ MMPTE TmplPte;
/* Setup template pte */
- TmpPte.u.Long = 0;
- TmpPte.u.Flush.Valid = 1;
- TmpPte.u.Flush.Write = 1;
+ TmplPte.u.Long = 0;
+ TmplPte.u.Flush.Valid = 1;
+ TmplPte.u.Flush.Write = 1;
/* Get a pointer to the PXE */
Pte = MiAddressToPxe(Address);
if (!Pte->u.Hard.Valid)
{
/* It's not valid, map it! */
- TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
- *Pte = TmpPte;
+ TmplPte.u.Hard.PageFrameNumber = MxAllocPage();
+ *Pte = TmplPte;
}
/* Get a pointer to the PPE */
if (!Pte->u.Hard.Valid)
{
/* It's not valid, map it! */
- TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
- *Pte = TmpPte;
+ TmplPte.u.Hard.PageFrameNumber = MxAllocPage();
+ *Pte = TmplPte;
}
/* Get a pointer to the PDE */
if (!Pte->u.Hard.Valid)
{
/* It's not valid, map it! */
- TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
- *Pte = TmpPte;
+ TmplPte.u.Hard.PageFrameNumber = MxAllocPage();
+ *Pte = TmplPte;
}
/* Get a pointer to the PTE */
}
VOID
-MxMapPageRange(PVOID Address, ULONG64 PageCount)
+NTAPI
+MxMapPage(PVOID Address)
{
- MMPTE TmpPte, *Pte;
+ MMPTE TmplPte, *Pte;
/* Setup template pte */
- TmpPte.u.Long = 0;
- TmpPte.u.Flush.Valid = 1;
- TmpPte.u.Flush.Write = 1;
+ TmplPte.u.Long = 0;
+ TmplPte.u.Flush.Valid = 1;
+ TmplPte.u.Flush.Write = 1;
+ TmplPte.u.Hard.PageFrameNumber = MxAllocPage();
+
+ /* Get the PTE for that page */
+ Pte = MxGetPte(Address);
+ ASSERT(Pte->u.Hard.Valid == 0);
+ /* Map a physical page */
+ *Pte = TmplPte;
+}
+
+VOID
+MxMapPageRange(PVOID Address, ULONG64 PageCount)
+{
while (PageCount--)
{
- /* Get the PTE for that page */
- Pte = MxGetPte(Address);
- ASSERT(Pte->u.Hard.Valid == 0);
-
- /* Map a physical page */
- TmpPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
- *Pte = TmpPte;
+ /* Map the page */
+ MxMapPage(Address);
/* Goto next page */
Address = (PVOID)((ULONG64)Address + PAGE_SIZE);
VOID
NTAPI
-MiArmConfigureMemorySizes(IN PLOADER_PARAMETER_BLOCK LoaderBloc)
+MiPreparePfnDatabse(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
- /* Get the size of the boot loader's image allocations */
- MmBootImageSize = KeLoaderBlock->Extension->LoaderPagesSpanned * PAGE_SIZE;
- MmBootImageSize = ROUND_UP(MmBootImageSize, 4 * 1024 * 1024);
+ PMEMORY_ALLOCATION_DESCRIPTOR MdBlock;
+ PLIST_ENTRY ListEntry;
+ PUCHAR Page, FirstPage;
+ SIZE_T Size;
+
+ /* Calculate the size of the PFN database and convert to pages */
+ MxPfnSizeInBytes = ROUND_TO_PAGES((MmHighestPhysicalPage + 1) * sizeof(MMPFN));
+ MxPfnAllocation = MxPfnSizeInBytes >> PAGE_SHIFT;
+
+ /* Simply start at hardcoded address */
+ MmPfnDatabase = MI_PFN_DATABASE;
+
+ /* Loop the memory descriptors */
+ for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
+ ListEntry != &LoaderBlock->MemoryDescriptorListHead;
+ ListEntry = ListEntry->Flink)
+ {
+ /* Get the memory descriptor */
+ MdBlock = CONTAINING_RECORD(ListEntry,
+ MEMORY_ALLOCATION_DESCRIPTOR,
+ ListEntry);
+
+ /* Skip pages that are not part of the PFN database */
+ if (MiIncludeType[MdBlock->MemoryType])
+ {
+ /* Get the base and size of this pfn database entry */
+ FirstPage = PAGE_ALIGN(&MmPfnDatabase[MdBlock->BasePage]);
+ Size = ROUND_TO_PAGES(MdBlock->PageCount * sizeof(MMPFN));
+
+ /* Loop the pages of this Pfn database entry */
+ for (Page = FirstPage; Page < FirstPage + Size; Page += PAGE_SIZE)
+ {
+ /* Is the page already mapped? */
+ if (!MmIsAddressValid(Page))
+ {
+ /* It's not, map it now */
+ MxMapPage(Page);
+ }
+ }
+
+ /* Zero out the pages */
+ RtlZeroMemory(FirstPage, Size);
+ }
+ }
+}
+
+
+VOID
+NTAPI
+MiInitializeSessionSpace(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
+{
+ /* Set up session space */
+ MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END;
+
+ /* This is where we will load Win32k.sys and the video driver */
+ MiSessionImageEnd = MiSessionSpaceEnd;
+ MiSessionImageStart = (PCHAR)MiSessionImageEnd - MmSessionImageSize;
+
+ /* The view starts right below the session working set (itself below
+ * the image area) */
+ MiSessionViewEnd = MI_SESSION_VIEW_END;
+ MiSessionViewStart = (PCHAR)MiSessionViewEnd - MmSessionViewSize;
+ ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart));
+
+ /* Session pool follows */
+ MiSessionPoolEnd = MiSessionViewStart;
+ MiSessionPoolStart = (PCHAR)MiSessionPoolEnd - MmSessionPoolSize;
+ ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart));
+
+ /* And it all begins here */
+ MmSessionBase = MiSessionPoolStart;
+
+ /* System view space ends at session space, so now that we know where
+ * this is, we can compute the base address of system view space itself. */
+ MiSystemViewStart = (PCHAR)MmSessionBase - MmSystemViewSize;
+ ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart));
+
+ /* Sanity checks */
+ ASSERT(MiSessionViewEnd <= MiSessionImageStart);
+ ASSERT(MmSessionBase <= MiSessionPoolStart);
+}
+
+VOID
+MiInitializePageTable()
+{
+ ULONG64 PageFrameOffset;
+ MMPTE TmplPte, *Pte;
+ PFN_NUMBER PageCount;
+
+ /* HACK: don't use freeldr debug print anymore */
+ FrLdrDbgPrint = NoDbgPrint;
+
+ /* Get current directory base */
+ MmSystemPageDirectory = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber;
+ PageFrameOffset = MmSystemPageDirectory << PAGE_SHIFT;
+ ASSERT(PageFrameOffset == __readcr3());
+
+ /* Set directory base for the system process */
+ PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PageFrameOffset;
+
+ /* Enable global pages */
+ __writecr4(__readcr4() | CR4_PGE);
+ ASSERT(__readcr4() & CR4_PGE);
+
+ /* Loop the user mode PXEs */
+ for (Pte = MiAddressToPxe(0);
+ Pte <= MiAddressToPxe(MmHighestUserAddress);
+ Pte++)
+ {
+ /* Zero the PXE, clear all mappings */
+ Pte->u.Long = 0;
+ }
+
+ /* Flush the TLB */
+ KeFlushCurrentTb();
+
+ /* Set up a template PTE */
+ TmplPte.u.Long = 0;
+ TmplPte.u.Flush.Valid = 1;
+ TmplPte.u.Flush.Write = 1;
+ HyperTemplatePte = TmplPte;
+
+ /* Create PDPTs (72 KB) for shared system address space,
+ * skip page tables and hyperspace */
+
+ /* Loop the PXEs */
+ for (Pte = MiAddressToPxe((PVOID)(HYPER_SPACE_END + 1));
+ Pte <= MiAddressToPxe(MI_HIGHEST_SYSTEM_ADDRESS);
+ Pte++)
+ {
+ /* Is the PXE already valid? */
+ if (!Pte->u.Hard.Valid)
+ {
+ /* It's not Initialize it */
+ TmplPte.u.Flush.PageFrameNumber = MxAllocPage();
+ *Pte = TmplPte;
+
+ /* Zero the page. The PXE is the PTE for the PDPT. */
+ RtlZeroMemory(MiPteToAddress(Pte), PAGE_SIZE);
+ }
+ }
+
+ /* Setup the mapping PTEs */
+ MmFirstReservedMappingPte = MxGetPte((PVOID)MI_MAPPING_RANGE_START);
+ MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES;
+ MmLastReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_END);
+
+#ifndef KDBG
+ /* Setup debug mapping PTE */
+ MmDebugPte = MxGetPte(MI_DEBUG_MAPPING);
+#endif
+}
+
+VOID
+NTAPI
+MiBuildNonPagedPool(VOID)
+{
+ PMMPTE Pte;
+ PFN_COUNT PageCount;
/* Check if this is a machine with less than 256MB of RAM, and no overide */
if ((MmNumberOfPhysicalPages <= MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING) &&
/* Check if the registy setting or our dynamic calculation was too high */
if (MmSizeOfNonPagedPoolInBytes > MI_MAX_INIT_NONPAGED_POOL_SIZE)
{
- // Set it to the maximum */
+ /* Set it to the maximum */
MmSizeOfNonPagedPoolInBytes = MI_MAX_INIT_NONPAGED_POOL_SIZE;
}
MmMaximumNonPagedPoolInBytes = MI_MAX_NONPAGED_POOL_SIZE;
}
- // MmSessionImageSize
-}
+ /* Put non paged pool to the end of the region */
+ MmNonPagedPoolStart = (PCHAR)MmNonPagedPoolEnd - MmMaximumNonPagedPoolInBytes;
-VOID
-NTAPI
-MiArmInitializeMemoryLayout(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
-{
- /* Set up session space */
- MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END;
+ /* Make sure it doesn't collide with the PFN database */
+ if ((PCHAR)MmNonPagedPoolStart < (PCHAR)MmPfnDatabase + MxPfnSizeInBytes)
+ {
+ /* Put non paged pool after the PFN database */
+ MmNonPagedPoolStart = (PCHAR)MmPfnDatabase + MxPfnSizeInBytes;
+ MmMaximumNonPagedPoolInBytes = (ULONG64)MmNonPagedPoolEnd -
+ (ULONG64)MmNonPagedPoolStart;
+ }
- /* This is where we will load Win32k.sys and the video driver */
- MiSessionImageEnd = MiSessionSpaceEnd;
- MiSessionImageStart = (PVOID)((ULONG_PTR)MiSessionImageEnd -
- MmSessionImageSize);
+ ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolStart));
- /* The view starts right below the session working set (itself below
- * the image area) */
- MiSessionViewEnd = MI_SESSION_VIEW_END;
- MiSessionViewStart = (PVOID)((ULONG_PTR)MiSessionViewStart -
- MmSessionViewSize);
-
- /* Session pool follows */
- MiSessionPoolEnd = MiSessionViewStart;
- MiSessionPoolStart = (PVOID)((ULONG_PTR)MiSessionPoolEnd -
- MmSessionPoolSize);
+ /* Calculate the nonpaged pool expansion start region */
+ MmNonPagedPoolExpansionStart = (PCHAR)MmNonPagedPoolStart +
+ MmSizeOfNonPagedPoolInBytes;
+ ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolExpansionStart));
- /* And it all begins here */
- MmSessionBase = MiSessionPoolStart;
+ /* Map the nonpaged pool */
+ PageCount = (MmSizeOfNonPagedPoolInBytes + PAGE_SIZE - 1) / PAGE_SIZE;
+ MxMapPageRange(MmNonPagedPoolStart, PageCount);
- /* System view space ends at session space, so now that we know where
- * this is, we can compute the base address of system view space itself. */
- MiSystemViewStart = (PVOID)((ULONG_PTR)MmSessionBase -
- MmSystemViewSize);
+ /* Loop the non paged pool extension PTEs */
+ for (Pte = MiAddressToPte(MmNonPagedPoolExpansionStart);
+ Pte <= MiAddressToPte(MmNonPagedPoolEnd);
+ Pte++)
+ {
+ /* Create PXE, PPE, PDE and zero the PTE */
+ MxGetPte(MiPteToAddress(Pte))->u.Long = 0;
+ }
- /* Use the default */
- MmNumberOfSystemPtes = 22000;
+ /* Initialize the ARM3 nonpaged pool */
+ MiInitializeArmPool();
- ASSERT(MiSessionViewEnd <= MiSessionImageStart);
- ASSERT(MmSessionBase <= MiSessionPoolStart);
+ /* Initialize the nonpaged pool */
+ InitializePool(NonPagedPool, 0);
}
VOID
-MiArmInitializePageTable()
+NTAPI
+MiBuildSystemPteSpace()
{
- ULONG64 PageFrameOffset;
PMMPTE Pte, StartPte, EndPte;
- /* Get current directory base */
- PageFrameOffset = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber << PAGE_SHIFT;
- ASSERT(PageFrameOffset == __readcr3());
-
- /* Set directory base for the system process */
- PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PageFrameOffset;
-
- /* HACK: don't use freeldr debug pront anymore */
- FrLdrDbgPrint = NoDbgPrint;
+ /* Use the default numer of system PTEs */
+ MmNumberOfSystemPtes = MI_NUMBER_SYSTEM_PTES;
-#if 1
- /* Clear user mode mappings in PML4 */
- StartPte = MiAddressToPxe(0);
- EndPte = MiAddressToPxe(MmHighestUserAddress);
+ /* System PTE pool is below the PFN database */
+ MiNonPagedSystemSize = (MmNumberOfSystemPtes + 1) * PAGE_SIZE;
+ MmNonPagedSystemStart = (PCHAR)MmPfnDatabase - MiNonPagedSystemSize;
+ MmNonPagedSystemStart = MM_ROUND_DOWN(MmNonPagedSystemStart, 512 * PAGE_SIZE);
- for (Pte = StartPte; Pte <= EndPte; Pte++)
+ /* Don't let it go below the minimum */
+ if (MmNonPagedSystemStart < (PVOID)MI_NON_PAGED_SYSTEM_START_MIN)
{
- /* Zero the pte */
- Pte->u.Long = 0;
- }
-#else
- /* Clear user mode mappings in PML4 */
- StartPte = MiAddressToPte(0);
- EndPte = MiAddressToPte((PVOID)0xa00000);
+ /* This is a hard-coded limit in the Windows NT address space */
+ MmNonPagedSystemStart = (PVOID)MI_NON_PAGED_SYSTEM_START_MIN;
- for (Pte = StartPte; Pte < EndPte; Pte++)
- {
- /* Zero the pte */
- //Pte->u.Long = 0;
+ /* Reduce the amount of system PTEs to reach this point */
+ MmNumberOfSystemPtes = ((ULONG64)MmPfnDatabase -
+ (ULONG64)MmNonPagedSystemStart) >>
+ PAGE_SHIFT;
+ MmNumberOfSystemPtes--;
+ ASSERT(MmNumberOfSystemPtes > 1000);
}
- /* Flush the TLB */
- KeFlushCurrentTb();
-
-// MiAddressToPde(0)->u.Long = 0;
-// MiAddressToPde((PVOID)0x200000)->u.Long = 0;
-// MiAddressToPde((PVOID)0x400000)->u.Long = 0;
-// MiAddressToPde((PVOID)0x600000)->u.Long = 0;
-// MiAddressToPde((PVOID)0x800000)->u.Long = 0;
+ /* Set the range of system PTEs */
+ StartPte = MiAddressToPte(MI_SYSTEM_PTE_START);
+ EndPte = StartPte + MmNumberOfSystemPtes - 1;
- // MiAddressToPpe->u.Long = 0;
+ /* Loop the system PTEs */
+ for (Pte = StartPte; Pte <= EndPte; Pte++)
+ {
+ /* Create PXE, PPE, PDE and zero the PTE */
+ MxGetPte(MiPteToAddress(Pte))->u.Long = 0;
+ }
-#endif
+ /* Create the system PTE space */
+ Pte = MiAddressToPte(MI_SYSTEM_PTE_START);
+ MiInitializeSystemPtes(Pte, MmNumberOfSystemPtes, SystemPteSpace);
- /* Flush the TLB */
- KeFlushCurrentTb();
+ /* Reserve system PTEs for zeroing PTEs and clear them */
+ MiFirstReservedZeroingPte = MiReserveSystemPtes(MI_ZERO_PTES, SystemPteSpace);
+ RtlZeroMemory(MiFirstReservedZeroingPte, MI_ZERO_PTES * sizeof(MMPTE));
- /* Setup debug mapping pte */
- MmDebugPte = MxGetPte(MI_DEBUG_MAPPING);
+ /* Set the counter to maximum */
+ MiFirstReservedZeroingPte->u.Hard.PageFrameNumber = MI_ZERO_PTES - 1;
}
-
VOID
NTAPI
-MiArmPreparePfnDatabse(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
+MiBuildPhysicalMemoryBlock(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
+ PPHYSICAL_MEMORY_DESCRIPTOR Buffer, NewBuffer;
PMEMORY_ALLOCATION_DESCRIPTOR MdBlock;
PLIST_ENTRY ListEntry;
- PFN_COUNT PageCount;
- PVOID PageBase;
+ PFN_NUMBER NextPage = -1;
+ PULONG Bitmap;
+ ULONG Runs = 0;
+ ULONG Size, i;
+
+ /* Calculate size for the PFN bitmap */
+ Size = ROUND_UP(MmHighestPhysicalPage + 1, sizeof(ULONG));
+
+ /* Allocate the PFN bitmap */
+ Bitmap = ExAllocatePoolWithTag(NonPagedPool, Size, ' mM');
+
+ /* Allocate enough memory for the physical memory block */
+ Buffer = ExAllocatePoolWithTag(NonPagedPool,
+ sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
+ sizeof(PHYSICAL_MEMORY_RUN) *
+ (MiNumberDescriptors - 1),
+ 'lMmM');
+ if (!Bitmap || !Buffer)
+ {
+ /* This is critical */
+ KeBugCheckEx(INSTALL_MORE_MEMORY,
+ MmNumberOfPhysicalPages,
+ MmLowestPhysicalPage,
+ MmHighestPhysicalPage,
+ 0x101);
+ }
- /* The PFN database is at the start of the non paged region */
- MmPfnDatabase = (PVOID)((ULONG64)MmNonPagedPoolEnd - MmMaximumNonPagedPoolInBytes);
+ /* Initialize the bitmap and clear all bits */
+ RtlInitializeBitMap(&MiPfnBitMap, Bitmap, MmHighestPhysicalPage + 1);
+ RtlClearAllBits(&MiPfnBitMap);
/* Loop the memory descriptors */
for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
ListEntry);
/* Skip pages that are not part of the PFN database */
- if ((MdBlock->MemoryType == LoaderFirmwarePermanent) ||
- (MdBlock->MemoryType == LoaderBBTMemory) ||
- (MdBlock->MemoryType == LoaderHALCachedMemory) ||
- (MdBlock->MemoryType == LoaderSpecialMemory) ||
- (MdBlock->MemoryType != LoaderBad))
+ if (!MiIncludeType[MdBlock->MemoryType])
{
continue;
}
- /* Map pages for the PFN database */
- PageCount = ROUND_TO_PAGES(MdBlock->PageCount * sizeof(MMPFN)) / PAGE_SIZE;
- PageBase = PAGE_ALIGN(&MmPfnDatabase[MdBlock->BasePage]);
- MxMapPageRange(PageBase, PageCount);
+ /* Does the memory block begin where the last ended? */
+ if (MdBlock->BasePage == NextPage)
+ {
+ /* Add it to the current run */
+ Buffer->Run[Runs - 1].PageCount += MdBlock->PageCount;
+ }
+ else
+ {
+ /* Create a new run */
+ Runs++;
+ Buffer->Run[Runs - 1].BasePage = MdBlock->BasePage;
+ Buffer->Run[Runs - 1].PageCount = MdBlock->PageCount;
+ }
+
+ /* Set the bits in the PFN bitmap */
+ RtlSetBits(&MiPfnBitMap, MdBlock->BasePage, MdBlock->PageCount);
- /* Zero out the pages */
- RtlZeroMemory(PageBase, PageCount * PAGE_SIZE);
+ /* Set the next page */
+ NextPage = MdBlock->BasePage + MdBlock->PageCount;
}
- /* Calculate the number of bytes, and then convert to pages */
- MxPfnSizeInBytes = ROUND_TO_PAGES(MmHighestPhysicalPage + 1) * sizeof(MMPFN);
- MxPfnAllocation = MxPfnSizeInBytes >> PAGE_SHIFT;
+ // FIXME: allocate a buffer of better size
- /* Reduce maximum pool size */
- MmMaximumNonPagedPoolInBytes -= MxPfnSizeInBytes;
+ Buffer->NumberOfRuns = Runs;
+ Buffer->NumberOfPages = MmNumberOfPhysicalPages;
+ MmPhysicalMemoryBlock = Buffer;
}
-
VOID
NTAPI
-MiArmPrepareNonPagedPool()
+MiBuildPagedPool(VOID)
{
- PFN_NUMBER PageCount;
- PVOID Address;
-
- /* Non paged pool comes after the PFN database */
- MmNonPagedPoolStart = (PVOID)((ULONG64)MmPfnDatabase +
- MxPfnSizeInBytes);
- ASSERT((ULONG64)MmNonPagedPoolEnd == (ULONG64)MmNonPagedPoolStart +
- MmMaximumNonPagedPoolInBytes);
-
- /* Calculate the nonpaged pool expansion start region */
- MmNonPagedPoolExpansionStart = (PVOID)((ULONG_PTR)MmNonPagedPoolEnd -
- MmMaximumNonPagedPoolInBytes +
- MmSizeOfNonPagedPoolInBytes);
- MmNonPagedPoolExpansionStart = (PVOID)PAGE_ALIGN(MmNonPagedPoolExpansionStart);
-
- /* Now calculate the nonpaged system VA region, which includes the
- * nonpaged pool expansion (above) and the system PTEs. Note that it is
- * then aligned to a PDE boundary (4MB). */
- MmNonPagedSystemStart = (PVOID)((ULONG_PTR)MmNonPagedPoolExpansionStart -
- (MmNumberOfSystemPtes + 1) * PAGE_SIZE);
- MmNonPagedSystemStart = (PVOID)((ULONG_PTR)MmNonPagedSystemStart &
- ~((4 * 1024 * 1024) - 1));
+ PMMPTE Pte;
+ MMPTE TmplPte;
+ PFN_NUMBER PageFrameIndex;
+ KIRQL OldIrql;
+ ULONG Size, BitMapSize;
+
+ /* Default size for paged pool is 4 times non paged pool */
+ MmSizeOfPagedPoolInBytes = 4 * MmMaximumNonPagedPoolInBytes;
- /* Don't let it go below the minimum */
- if (MmNonPagedSystemStart < (PVOID)MI_NON_PAGED_SYSTEM_START_MIN)
+ /* Make sure it doesn't overflow */
+ if (MmSizeOfPagedPoolInBytes > ((ULONG64)MmNonPagedSystemStart -
+ (ULONG64)MmPagedPoolStart))
{
- /* This is a hard-coded limit in the Windows NT address space */
- MmNonPagedSystemStart = (PVOID)MI_NON_PAGED_SYSTEM_START_MIN;
+ MmSizeOfPagedPoolInBytes = (ULONG64)MmNonPagedSystemStart -
+ (ULONG64)MmPagedPoolStart;
+ }
- /* Reduce the amount of system PTEs to reach this point */
- MmNumberOfSystemPtes = ((ULONG_PTR)MmNonPagedPoolExpansionStart -
- (ULONG_PTR)MmNonPagedSystemStart) >>
- PAGE_SHIFT;
- MmNumberOfSystemPtes--;
- ASSERT(MmNumberOfSystemPtes > 1000);
+ /* Make sure paged pool is big enough */
+ if (MmSizeOfPagedPoolInBytes < MI_MIN_INIT_PAGED_POOLSIZE)
+ {
+ MmSizeOfPagedPoolInBytes = MI_MIN_INIT_PAGED_POOLSIZE;
}
- /* Map the nonpaged pool */
- PageCount = (MmSizeOfNonPagedPoolInBytes + PAGE_SIZE - 1) / PAGE_SIZE;
- MxMapPageRange(MmNonPagedPoolStart, PageCount);
+ /* Align down to a PDE boundary */
+ MmSizeOfPagedPoolInBytes = ROUND_DOWN(MmSizeOfPagedPoolInBytes,
+ 512 * PAGE_SIZE);
+ MmSizeOfPagedPoolInPages = MmSizeOfPagedPoolInBytes >> PAGE_SHIFT;
+
+ /* This is where paged pool ends */
+ MmPagedPoolEnd = (PCHAR)MmPagedPoolStart + MmSizeOfPagedPoolInBytes - 1;
+
+ /* Sanity check */
+ ASSERT(MmPagedPoolEnd < MmNonPagedSystemStart);
- /* Create PTEs for the paged pool extension */
- for (Address = MmNonPagedPoolExpansionStart;
- Address < MmNonPagedPoolEnd;
- Address = (PVOID)((ULONG64)Address + PAGE_SIZE))
+ /* setup a template PTE */
+ TmplPte.u.Long = 0;
+ TmplPte.u.Flush.Valid = 1;
+ TmplPte.u.Flush.Write = 1;
+
+ /* Make sure the PXE is valid */
+ Pte = MiAddressToPxe(MmPagedPoolStart);
+ if (!Pte->u.Flush.Valid)
{
- /* Create PXE, PPE, PDE and set PTE to 0*/
- MxGetPte(Address)->u.Long = 0;
+ /* Map it! */
+ TmplPte.u.Flush.PageFrameNumber = MmAllocPage(MC_SYSTEM, 0);
+ *Pte = TmplPte;
}
-//DPRINT1("MmNonPagedPoolStart = %p, Pte=%p \n", MmNonPagedPoolStart, MiAddressToPte(MmNonPagedPoolStart));
- /* Sanity check */
- ASSERT(MiAddressToPte(MmNonPagedSystemStart) <
- MiAddressToPte(MmNonPagedPoolExpansionStart));
+ /* Map all page directories (max 128) */
+ for (Pte = MiAddressToPpe(MmPagedPoolStart);
+ Pte <= MiAddressToPpe(MmPagedPoolEnd);
+ Pte++)
+ {
+ if (!Pte->u.Flush.Valid)
+ {
+ /* Map it! */
+ TmplPte.u.Flush.PageFrameNumber = MxAllocPage();
+ *Pte = TmplPte;
+ }
+ }
+
+ /* Create and map the first PTE for paged pool */
+ Pte = MxGetPte(MmPagedPoolStart);
+ TmplPte.u.Flush.PageFrameNumber = MxAllocPage();
+ *Pte = TmplPte;
+
+ /* Save the first and last paged pool PTE */
+ MmPagedPoolInfo.FirstPteForPagedPool = MiAddressToPte(MmPagedPoolStart);
+ MmPagedPoolInfo.LastPteForPagedPool = MiAddressToPte(MmPagedPoolEnd);
+
+ MmPagedPoolInfo.NextPdeForPagedPoolExpansion =
+ MiAddressToPde(MmPagedPoolStart) + 1;
+
+ // We keep track of each page via a bit, so check how big the bitmap will
+ // have to be (make sure to align our page count such that it fits nicely
+ // into a 4-byte aligned bitmap.
+
+ /* The size of the bitmap in bits is the size in pages */
+ BitMapSize = MmSizeOfPagedPoolInPages;
+
+ /* Calculate buffer size in bytes, aligned to 32 bits */
+ Size = sizeof(RTL_BITMAP) + ROUND_UP(BitMapSize, 32) / 8;
+ // Allocate the allocation bitmap, which tells us which regions have not yet
+ // been mapped into memory
+
+ MmPagedPoolInfo.PagedPoolAllocationMap =
+ ExAllocatePoolWithTag(NonPagedPool, Size, ' mM');
+ ASSERT(MmPagedPoolInfo.PagedPoolAllocationMap);
+
+ // Initialize it such that at first, only the first page's worth of PTEs is
+ // marked as allocated (incidentially, the first PDE we allocated earlier).
+ RtlInitializeBitMap(MmPagedPoolInfo.PagedPoolAllocationMap,
+ (PULONG)(MmPagedPoolInfo.PagedPoolAllocationMap + 1),
+ BitMapSize);
+ RtlSetAllBits(MmPagedPoolInfo.PagedPoolAllocationMap);
+ RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, 0, 512);
+
+ // We have a second bitmap, which keeps track of where allocations end.
+ // Given the allocation bitmap and a base address, we can therefore figure
+ // out which page is the last page of that allocation, and thus how big the
+ // entire allocation is.
+ MmPagedPoolInfo.EndOfPagedPoolBitmap =
+ ExAllocatePoolWithTag(NonPagedPool, Size, ' mM');
+ ASSERT(MmPagedPoolInfo.EndOfPagedPoolBitmap);
+
+ /* Initialize the bitmap */
+ RtlInitializeBitMap(MmPagedPoolInfo.EndOfPagedPoolBitmap,
+ (PULONG)(MmPagedPoolInfo.EndOfPagedPoolBitmap + 1),
+ BitMapSize);
+
+ /* No allocations, no allocation ends; clear all bits. */
+ RtlClearAllBits(MmPagedPoolInfo.EndOfPagedPoolBitmap);
+
+ /* Initialize the paged pool mutex */
+ KeInitializeGuardedMutex(&MmPagedPoolMutex);
+
+ /* Initialize the paged pool */
+ InitializePool(PagedPool, 0);
}
+
NTSTATUS
NTAPI
MmArmInitSystem(IN ULONG Phase,
/* Parse memory descriptors */
MiEvaluateMemoryDescriptors(LoaderBlock);
- /* Configure the memory sizes */
- MiArmConfigureMemorySizes(LoaderBlock);
-
- /* Initialize the memory layout */
- MiArmInitializeMemoryLayout(LoaderBlock);
-
/* Prepare PFN database mappings */
- MiArmPreparePfnDatabse(LoaderBlock);
-
- /* Initialize some mappings */
- MiArmInitializePageTable();
+ MiPreparePfnDatabse(LoaderBlock);
- /* Prepare paged pool mappings */
- MiArmPrepareNonPagedPool();
+ /* Initialize the session space */
+ MiInitializeSessionSpace(LoaderBlock);
- /* Initialize the ARM3 nonpaged pool */
- MiInitializeArmPool();
+ /* Initialize some mappings */
+ MiInitializePageTable();
/* Update the memory descriptor, to make sure the pages we used
won't get inserted into the PFN database */
}
else if (Phase == 1)
{
+ PMMPTE Pte;
+ ULONG OldCount;
+ PPHYSICAL_MEMORY_RUN Run;
+
/* The PFN database was created, restore the free descriptor */
*MxFreeDescriptor = MxOldFreeDescriptor;
- ASSERT(FALSE);
+ /* Switch to phase 1 */
+ MxPhase = 1;
+
+ /* Initialize the nonpaged pool */
+ MiBuildNonPagedPool();
+
+ /* Initialize system PTE handling */
+ MiBuildSystemPteSpace();
+
+ /* Build the physical memory block */
+ MiBuildPhysicalMemoryBlock(LoaderBlock);
+ /* Size up paged pool and build the shadow system page directory */
+ MiBuildPagedPool();
}
return STATUS_SUCCESS;