ntoskrnl/mm/amd64/init.c

   1 /*
   2  * COPYRIGHT:       GPL, See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/amd64/init.c
   5  * PURPOSE:         Memory Manager Initialization for amd64
   6  *
   7  * PROGRAMMERS:     Timo kreuzer (timo.kreuzer@reactos.org)
   8  *                  ReactOS Portable Systems Group
   9  */
  10
  11 /* INCLUDES ***************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 //#define NDEBUG
  15 #include <debug.h>
  16
  17 #include <mm/ARM3/miarm.h>
  18
  19 extern PMMPTE MmDebugPte;
  20
  21 /* Helper macros */
  22 #define IS_ALIGNED(addr, align) (((ULONG64)(addr) & (align - 1)) == 0)
  23 #define IS_PAGE_ALIGNED(addr) IS_ALIGNED(addr, PAGE_SIZE)
  24
  25 /* GLOBALS *****************************************************************/
  26
  27 /* Template PTE and PDE for a kernel page */
  28 MMPTE ValidKernelPde = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  29 MMPTE ValidKernelPte = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  30
  31 /* The same, but for local pages */
  32 MMPTE ValidKernelPdeLocal = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  33 MMPTE ValidKernelPteLocal = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  34
  35 /* Template PDE for a demand-zero page */
  36 MMPDE DemandZeroPde  = {{MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  37 MMPTE DemandZeroPte  = {{MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  38
  39 /* Template PTE for prototype page */
  40 MMPTE PrototypePte = {{(MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS) |
  41                       PTE_PROTOTYPE | (MI_PTE_LOOKUP_NEEDED << 32)}};
  42
  43 /* Template PTE for decommited page */
  44 MMPTE MmDecommittedPte = {{MM_DECOMMIT << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  45
  46 /* Address ranges */
  47 PVOID MiSessionViewEnd;
  48 PVOID MiSystemPteSpaceStart;
  49 PVOID MiSystemPteSpaceEnd;
  50
  51 ULONG64 MxPfnSizeInBytes;
  52 BOOLEAN MiIncludeType[LoaderMaximum];
  53 PFN_NUMBER MxFreePageBase;
  54 ULONG64 MxFreePageCount = 0;
  55
  56 BOOLEAN MiPfnsInitialized = FALSE;
  57
  58 /* FUNCTIONS *****************************************************************/
  59
  60 VOID
  61 NTAPI
  62 INIT_FUNCTION
  63 MiInitializeSessionSpaceLayout(VOID)
  64 {
  65     MmSessionSize = MI_SESSION_SIZE;
  66     MmSessionViewSize = MI_SESSION_VIEW_SIZE;
  67     MmSessionPoolSize = MI_SESSION_POOL_SIZE;
  68     MmSessionImageSize = MI_SESSION_IMAGE_SIZE;
  69     MmSystemViewSize = MI_SYSTEM_VIEW_SIZE;
  70
  71     /* Set up session space */
  72     MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END;
  73
  74     /* This is where we will load Win32k.sys and the video driver */
  75     MiSessionImageEnd = MiSessionSpaceEnd;
  76     MiSessionImageStart = (PCHAR)MiSessionImageEnd - MmSessionImageSize;
  77
  78     /* The view starts right below the session working set (itself below
  79      * the image area) */
  80     MiSessionViewEnd = (PVOID)MI_SESSION_VIEW_END;
  81     MiSessionViewStart = (PCHAR)MiSessionViewEnd - MmSessionViewSize;
  82     ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart));
  83
  84     /* Session pool follows */
  85     MiSessionPoolEnd = MiSessionViewStart;
  86     MiSessionPoolStart = (PCHAR)MiSessionPoolEnd - MmSessionPoolSize;
  87     ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart));
  88
  89     /* And it all begins here */
  90     MmSessionBase = MiSessionPoolStart;
  91
  92     /* System view space ends at session space, so now that we know where
  93      * this is, we can compute the base address of system view space itself. */
  94     MiSystemViewStart = (PCHAR)MmSessionBase - MmSystemViewSize;
  95     ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart));
  96
  97     /* Sanity checks */
  98     ASSERT(MiSessionViewEnd <= MiSessionImageStart);
  99     ASSERT(MmSessionBase <= MiSessionPoolStart);
 100 }
 101
 102 VOID
 103 NTAPI
 104 MiMapPPEs(
 105     PVOID StartAddress,
 106     PVOID EndAddress)
 107 {
 108     PMMPDE PointerPpe;
 109     MMPDE TmplPde = ValidKernelPde;
 110
 111     /* Loop the PPEs */
 112     for (PointerPpe = MiAddressToPpe(StartAddress);
 113          PointerPpe <= MiAddressToPpe(EndAddress);
 114          PointerPpe++)
 115     {
 116         /* Check if its already mapped */
 117         if (!PointerPpe->u.Hard.Valid)
 118         {
 119             /* No, map it! */
 120             TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
 121             MI_WRITE_VALID_PTE(PointerPpe, TmplPde);
 122
 123             /* Zero out the page table */
 124             RtlZeroMemory(MiPteToAddress(PointerPpe), PAGE_SIZE);
 125         }
 126     }
 127 }
 128
 129 VOID
 130 NTAPI
 131 MiMapPDEs(
 132     PVOID StartAddress,
 133     PVOID EndAddress)
 134 {
 135     PMMPDE PointerPde;
 136     MMPDE TmplPde = ValidKernelPde;
 137
 138     /* Loop the PDEs */
 139     for (PointerPde = MiAddressToPde(StartAddress);
 140          PointerPde <= MiAddressToPde(EndAddress);
 141          PointerPde++)
 142     {
 143         /* Check if its already mapped */
 144         if (!PointerPde->u.Hard.Valid)
 145         {
 146             /* No, map it! */
 147             TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
 148             MI_WRITE_VALID_PTE(PointerPde, TmplPde);
 149
 150             /* Zero out the page table */
 151             RtlZeroMemory(MiPteToAddress(PointerPde), PAGE_SIZE);
 152         }
 153     }
 154 }
 155
 156 VOID
 157 NTAPI
 158 MiMapPTEs(
 159     PVOID StartAddress,
 160     PVOID EndAddress)
 161 {
 162     PMMPTE PointerPte;
 163     MMPTE TmplPte = ValidKernelPte;
 164
 165     /* Loop the PTEs */
 166     for (PointerPte = MiAddressToPte(StartAddress);
 167          PointerPte <= MiAddressToPte(EndAddress);
 168          PointerPte++)
 169     {
 170         /* Check if its already mapped */
 171         if (!PointerPte->u.Hard.Valid)
 172         {
 173             /* No, map it! */
 174             TmplPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
 175             MI_WRITE_VALID_PTE(PointerPte, TmplPte);
 176
 177             /* Zero out the page (FIXME: not always neccessary) */
 178             RtlZeroMemory(MiPteToAddress(PointerPte), PAGE_SIZE);
 179         }
 180     }
 181 }
 182
 183 VOID
 184 NTAPI
 185 INIT_FUNCTION
 186 MiInitializePageTable(VOID)
 187 {
 188     ULONG64 PxePhysicalAddress;
 189     MMPTE TmplPte, *PointerPxe;
 190     PFN_NUMBER PxePfn;
 191
 192     /* Get current directory base */
 193     PxePfn = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber;
 194     PxePhysicalAddress = PxePfn << PAGE_SHIFT;
 195     ASSERT(PxePhysicalAddress == __readcr3());
 196
 197     /* Set directory base for the system process */
 198     PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PxePhysicalAddress;
 199
 200     /* Enable global pages */
 201     __writecr4(__readcr4() | CR4_PGE);
 202     ASSERT(__readcr4() & CR4_PGE);
 203
 204     /* Loop the user mode PXEs */
 205     for (PointerPxe = MiAddressToPxe(0);
 206          PointerPxe <= MiAddressToPxe(MmHighestUserAddress);
 207          PointerPxe++)
 208     {
 209         /* Zero the PXE, clear all mappings */
 210         PointerPxe->u.Long = 0;
 211     }
 212
 213     /* Flush the TLB */
 214     KeFlushCurrentTb();
 215
 216     /* Set up a template PTE */
 217     TmplPte.u.Long = 0;
 218     TmplPte.u.Flush.Valid = 1;
 219     TmplPte.u.Flush.Write = 1;
 220     HyperTemplatePte = TmplPte;
 221
 222     /* Create PDPTs (72 KB) for shared system address space,
 223      * skip page tables TODO: use global pages. */
 224
 225     /* Loop the PXEs */
 226     for (PointerPxe = MiAddressToPxe((PVOID)HYPER_SPACE);
 227          PointerPxe <= MiAddressToPxe(MI_HIGHEST_SYSTEM_ADDRESS);
 228          PointerPxe++)
 229     {
 230         /* Is the PXE already valid? */
 231         if (!PointerPxe->u.Hard.Valid)
 232         {
 233             /* It's not Initialize it */
 234             TmplPte.u.Flush.PageFrameNumber = MxGetNextPage(1);
 235             *PointerPxe = TmplPte;
 236
 237             /* Zero the page. The PXE is the PTE for the PDPT. */
 238             RtlZeroMemory(MiPteToAddress(PointerPxe), PAGE_SIZE);
 239         }
 240     }
 241
 242     /* Map PPEs for paged pool */
 243     MiMapPPEs(MmPagedPoolStart, MmPagedPoolEnd);
 244
 245     /* Setup 1 PPE for hyper space */
 246     MiMapPPEs((PVOID)HYPER_SPACE, (PVOID)HYPER_SPACE_END);
 247
 248     /* Setup PPEs for system space view */
 249     MiMapPPEs(MiSystemViewStart, (PCHAR)MiSystemViewStart + MmSystemViewSize);
 250
 251     /* Setup the mapping PDEs */
 252     MiMapPDEs((PVOID)MI_MAPPING_RANGE_START, (PVOID)MI_MAPPING_RANGE_END);
 253
 254     /* Setup the mapping PTEs */
 255     MmFirstReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_START);
 256     MmLastReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_END);
 257     MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES;
 258
 259     /* Setup debug mapping PTE */
 260     MiMapPPEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
 261     MiMapPDEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
 262     MmDebugPte = MiAddressToPte((PVOID)MI_DEBUG_MAPPING);
 263
 264     /* Setup PDE and PTEs for VAD bitmap and working set list */
 265     MiMapPDEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
 266     MiMapPTEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
 267 }
 268
 269 VOID
 270 NTAPI
 271 INIT_FUNCTION
 272 MiBuildNonPagedPool(VOID)
 273 {
 274     /* Check if this is a machine with less than 256MB of RAM, and no overide */
 275     if ((MmNumberOfPhysicalPages <= MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING) &&
 276         !(MmSizeOfNonPagedPoolInBytes))
 277     {
 278         /* Force the non paged pool to be 2MB so we can reduce RAM usage */
 279         MmSizeOfNonPagedPoolInBytes = 2 * 1024 * 1024;
 280     }
 281
 282     /* Check if the user gave a ridicuously large nonpaged pool RAM size */
 283     if ((MmSizeOfNonPagedPoolInBytes >> PAGE_SHIFT) >
 284         (MmNumberOfPhysicalPages * 7 / 8))
 285     {
 286         /* More than 7/8ths of RAM was dedicated to nonpaged pool, ignore! */
 287         MmSizeOfNonPagedPoolInBytes = 0;
 288     }
 289
 290     /* Check if no registry setting was set, or if the setting was too low */
 291     if (MmSizeOfNonPagedPoolInBytes < MmMinimumNonPagedPoolSize)
 292     {
 293         /* Start with the minimum (256 KB) and add 32 KB for each MB above 4 */
 294         MmSizeOfNonPagedPoolInBytes = MmMinimumNonPagedPoolSize;
 295         MmSizeOfNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
 296                                        256 * MmMinAdditionNonPagedPoolPerMb;
 297     }
 298
 299     /* Check if the registy setting or our dynamic calculation was too high */
 300     if (MmSizeOfNonPagedPoolInBytes > MI_MAX_INIT_NONPAGED_POOL_SIZE)
 301     {
 302         /* Set it to the maximum */
 303         MmSizeOfNonPagedPoolInBytes = MI_MAX_INIT_NONPAGED_POOL_SIZE;
 304     }
 305
 306     /* Check if a percentage cap was set through the registry */
 307     if (MmMaximumNonPagedPoolPercent)
 308     {
 309         /* Don't feel like supporting this right now */
 310         UNIMPLEMENTED;
 311     }
 312
 313     /* Page-align the nonpaged pool size */
 314     MmSizeOfNonPagedPoolInBytes &= ~(PAGE_SIZE - 1);
 315
 316     /* Now, check if there was a registry size for the maximum size */
 317     if (!MmMaximumNonPagedPoolInBytes)
 318     {
 319         /* Start with the default (1MB) and add 400 KB for each MB above 4 */
 320         MmMaximumNonPagedPoolInBytes = MmDefaultMaximumNonPagedPool;
 321         MmMaximumNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
 322                                          256 * MmMaxAdditionNonPagedPoolPerMb;
 323     }
 324
 325     /* Don't let the maximum go too high */
 326     if (MmMaximumNonPagedPoolInBytes > MI_MAX_NONPAGED_POOL_SIZE)
 327     {
 328         /* Set it to the upper limit */
 329         MmMaximumNonPagedPoolInBytes = MI_MAX_NONPAGED_POOL_SIZE;
 330     }
 331
 332     /* Convert nonpaged pool size from bytes to pages */
 333     MmMaximumNonPagedPoolInPages = MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT;
 334
 335     /* Non paged pool starts after the PFN database */
 336     MmNonPagedPoolStart = MmPfnDatabase + MxPfnAllocation * PAGE_SIZE;
 337
 338     /* Calculate the nonpaged pool expansion start region */
 339     MmNonPagedPoolExpansionStart = (PCHAR)MmNonPagedPoolStart +
 340                                           MmSizeOfNonPagedPoolInBytes;
 341     ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolExpansionStart));
 342
 343     /* And this is where the none paged pool ends */
 344     MmNonPagedPoolEnd = (PCHAR)MmNonPagedPoolStart + MmMaximumNonPagedPoolInBytes;
 345     ASSERT(MmNonPagedPoolEnd < (PVOID)MM_HAL_VA_START);
 346
 347     /* Map PPEs and PDEs for non paged pool (including expansion) */
 348     MiMapPPEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
 349     MiMapPDEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
 350
 351     /* Map the nonpaged pool PTEs (without expansion) */
 352     MiMapPTEs(MmNonPagedPoolStart, (PCHAR)MmNonPagedPoolExpansionStart - 1);
 353
 354     /* Initialize the ARM3 nonpaged pool */
 355     MiInitializeNonPagedPool();
 356     MiInitializeNonPagedPoolThresholds();
 357
 358 }
 359
 360 VOID
 361 NTAPI
 362 INIT_FUNCTION
 363 MiBuildSystemPteSpace(VOID)
 364 {
 365     PMMPTE PointerPte;
 366     SIZE_T NonPagedSystemSize;
 367
 368     /* Use the default number of system PTEs */
 369     MmNumberOfSystemPtes = MI_NUMBER_SYSTEM_PTES;
 370     NonPagedSystemSize = (MmNumberOfSystemPtes + 1) * PAGE_SIZE;
 371
 372     /* Put system PTEs at the start of the system VA space */
 373     MiSystemPteSpaceStart = MmNonPagedSystemStart;
 374     MiSystemPteSpaceEnd = (PUCHAR)MiSystemPteSpaceStart + NonPagedSystemSize;
 375
 376     /* Map the PPEs and PDEs for the system PTEs */
 377     MiMapPPEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
 378     MiMapPDEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
 379
 380     /* Initialize the system PTE space */
 381     PointerPte = MiAddressToPte(MiSystemPteSpaceStart);
 382     MiInitializeSystemPtes(PointerPte, MmNumberOfSystemPtes, SystemPteSpace);
 383
 384     /* Reserve system PTEs for zeroing PTEs and clear them */
 385     MiFirstReservedZeroingPte = MiReserveSystemPtes(MI_ZERO_PTES, SystemPteSpace);
 386     RtlZeroMemory(MiFirstReservedZeroingPte, MI_ZERO_PTES * sizeof(MMPTE));
 387
 388     /* Set the counter to maximum */
 389     MiFirstReservedZeroingPte->u.Hard.PageFrameNumber = MI_ZERO_PTES - 1;
 390 }
 391
 392 static
 393 VOID
 394 MiSetupPfnForPageTable(
 395     PFN_NUMBER PageFrameIndex,
 396     PMMPTE PointerPte)
 397 {
 398     PMMPFN Pfn;
 399     PMMPDE PointerPde;
 400
 401     /* Get the pfn entry for this page */
 402     Pfn = MiGetPfnEntry(PageFrameIndex);
 403
 404     /* Check if it's valid memory */
 405     if ((PageFrameIndex <= MmHighestPhysicalPage) &&
 406         (MmIsAddressValid(Pfn)) &&
 407         (Pfn->u3.e1.PageLocation == ActiveAndValid))
 408     {
 409         /* Setup the PFN entry */
 410         Pfn->u1.WsIndex = 0;
 411         Pfn->u2.ShareCount++;
 412         Pfn->PteAddress = PointerPte;
 413         Pfn->OriginalPte = *PointerPte;
 414         Pfn->u3.e1.PageLocation = ActiveAndValid;
 415         Pfn->u3.e1.CacheAttribute = MiNonCached;
 416         Pfn->u3.e2.ReferenceCount = 1;
 417         Pfn->u4.PteFrame = PFN_FROM_PTE(MiAddressToPte(PointerPte));
 418     }
 419
 420     /* Increase the shared count of the PFN entry for the PDE */
 421     PointerPde = MiAddressToPde(MiPteToAddress(PointerPte));
 422     Pfn = MiGetPfnEntry(PFN_FROM_PTE(PointerPde));
 423     Pfn->u2.ShareCount++;
 424 }
 425
 426 VOID
 427 NTAPI
 428 MiBuildPfnDatabaseFromPageTables(VOID)
 429 {
 430     PVOID Address = NULL;
 431     PFN_NUMBER PageFrameIndex;
 432     PMMPDE PointerPde;
 433     PMMPTE PointerPte;
 434     ULONG k, l;
 435     PMMPFN Pfn;
 436 #if (_MI_PAGING_LEVELS >= 3)
 437     PMMPDE PointerPpe;
 438     ULONG j;
 439 #endif
 440 #if (_MI_PAGING_LEVELS == 4)
 441     PMMPDE PointerPxe;
 442     ULONG i;
 443 #endif
 444
 445     /* Manual setup of the top level page directory */
 446 #if (_MI_PAGING_LEVELS == 4)
 447     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PXE_BASE));
 448 #elif (_MI_PAGING_LEVELS == 3)
 449     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PPE_BASE));
 450 #else
 451     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PDE_BASE));
 452 #endif
 453     Pfn = MiGetPfnEntry(PageFrameIndex);
 454     ASSERT(Pfn->u3.e1.PageLocation == ActiveAndValid);
 455     Pfn->u1.WsIndex = 0;
 456     Pfn->u2.ShareCount = 1;
 457     Pfn->PteAddress = NULL;
 458     Pfn->u3.e1.CacheAttribute = MiNonCached;
 459     Pfn->u3.e2.ReferenceCount = 1;
 460     Pfn->u4.PteFrame = 0;
 461
 462 #if (_MI_PAGING_LEVELS == 4)
 463     /* Loop all PXEs in the PML4 */
 464     PointerPxe = MiAddressToPxe(Address);
 465     for (i = 0; i < PXE_PER_PAGE; i++, PointerPxe++)
 466     {
 467         /* Skip invalid PXEs */
 468         if (!PointerPxe->u.Hard.Valid) continue;
 469
 470         /* Handle the PFN */
 471         PageFrameIndex = PFN_FROM_PXE(PointerPxe);
 472         MiSetupPfnForPageTable(PageFrameIndex, PointerPxe);
 473
 474         /* Get starting VA for this PXE */
 475         Address = MiPxeToAddress(PointerPxe);
 476 #endif
 477 #if (_MI_PAGING_LEVELS >= 3)
 478         /* Loop all PPEs in this PDP */
 479         PointerPpe = MiAddressToPpe(Address);
 480         for (j = 0; j < PPE_PER_PAGE; j++, PointerPpe++)
 481         {
 482             /* Skip invalid PPEs */
 483             if (!PointerPpe->u.Hard.Valid) continue;
 484
 485             /* Handle the PFN */
 486             PageFrameIndex = PFN_FROM_PPE(PointerPpe);
 487             MiSetupPfnForPageTable(PageFrameIndex, PointerPpe);
 488
 489             /* Get starting VA for this PPE */
 490             Address = MiPpeToAddress(PointerPpe);
 491 #endif
 492             /* Loop all PDEs in this PD */
 493             PointerPde = MiAddressToPde(Address);
 494             for (k = 0; k < PDE_PER_PAGE; k++, PointerPde++)
 495             {
 496                 /* Skip invalid PDEs */
 497                 if (!PointerPde->u.Hard.Valid) continue;
 498
 499                 /* Handle the PFN */
 500                 PageFrameIndex = PFN_FROM_PDE(PointerPde);
 501                 MiSetupPfnForPageTable(PageFrameIndex, PointerPde);
 502
 503                 /* Get starting VA for this PDE */
 504                 Address = MiPdeToAddress(PointerPde);
 505
 506                 /* Loop all PTEs in this PT */
 507                 PointerPte = MiAddressToPte(Address);
 508                 for (l = 0; l < PTE_PER_PAGE; l++, PointerPte++)
 509                 {
 510                     /* Skip invalid PTEs */
 511                     if (!PointerPte->u.Hard.Valid) continue;
 512
 513                     /* Handle the PFN */
 514                     PageFrameIndex = PFN_FROM_PTE(PointerPte);
 515                     MiSetupPfnForPageTable(PageFrameIndex, PointerPte);
 516                 }
 517             }
 518 #if (_MI_PAGING_LEVELS >= 3)
 519         }
 520 #endif
 521 #if (_MI_PAGING_LEVELS == 4)
 522     }
 523 #endif
 524 }
 525
 526 VOID
 527 NTAPI
 528 INIT_FUNCTION
 529 MiAddDescriptorToDatabase(
 530     PFN_NUMBER BasePage,
 531     PFN_NUMBER PageCount,
 532     TYPE_OF_MEMORY MemoryType)
 533 {
 534     PMMPFN Pfn;
 535
 536     ASSERT(!MiIsMemoryTypeInvisible(MemoryType));
 537
 538     /* Check if the memory is free */
 539     if (MiIsMemoryTypeFree(MemoryType))
 540     {
 541         /* Get the last pfn of this descriptor. Note we loop backwards */
 542         Pfn = &MmPfnDatabase[BasePage + PageCount - 1];
 543
 544         /* Loop all pages */
 545         while (PageCount--)
 546         {
 547             /* Add it to the free list */
 548             Pfn->u3.e1.CacheAttribute = MiNonCached;
 549             MiInsertPageInFreeList(BasePage + PageCount);
 550
 551             /* Go to the previous page */
 552             Pfn--;
 553         }
 554     }
 555     else if (MemoryType == LoaderXIPRom)
 556     {
 557         Pfn = &MmPfnDatabase[BasePage];
 558         while (PageCount--)
 559         {
 560             /* Make it a pseudo-I/O ROM mapping */
 561             Pfn->PteAddress = 0;
 562             Pfn->u1.Flink = 0;
 563             Pfn->u2.ShareCount = 0;
 564             Pfn->u3.e1.PageLocation = 0;
 565             Pfn->u3.e1.CacheAttribute = MiNonCached;
 566             Pfn->u3.e1.Rom = 1;
 567             Pfn->u3.e1.PrototypePte = 1;
 568             Pfn->u3.e2.ReferenceCount = 0;
 569             Pfn->u4.InPageError = 0;
 570             Pfn->u4.PteFrame = 0;
 571
 572             /* Advance one */
 573             Pfn++;
 574         }
 575     }
 576     else if (MemoryType == LoaderBad)
 577     {
 578         // FIXME: later
 579         ASSERT(FALSE);
 580     }
 581     else
 582     {
 583         /* For now skip it */
 584         DbgPrint("Skipping BasePage=0x%lx, PageCount=0x%lx, MemoryType=%lx\n",
 585                  BasePage, PageCount, MemoryType);
 586         Pfn = &MmPfnDatabase[BasePage];
 587         while (PageCount--)
 588         {
 589             /* Make an active PFN */
 590             Pfn->u3.e1.PageLocation = ActiveAndValid;
 591
 592             /* Advance one */
 593             Pfn++;
 594         }
 595     }
 596 }
 597
 598 VOID
 599 NTAPI
 600 INIT_FUNCTION
 601 MiBuildPfnDatabase(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 602 {
 603     PLIST_ENTRY ListEntry;
 604     PMEMORY_ALLOCATION_DESCRIPTOR Descriptor;
 605     PFN_NUMBER BasePage, PageCount;
 606
 607     /* Map the PDEs and PPEs for the pfn database (ignore holes) */
 608 #if (_MI_PAGING_LEVELS >= 3)
 609     MiMapPPEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + (MxPfnAllocation * PAGE_SIZE) - 1);
 610 #endif
 611     MiMapPDEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + (MxPfnAllocation * PAGE_SIZE) - 1);
 612
 613     /* First initialize the color tables */
 614     MiInitializeColorTables();
 615
 616     /* Loop the memory descriptors */
 617     for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
 618          ListEntry != &LoaderBlock->MemoryDescriptorListHead;
 619          ListEntry = ListEntry->Flink)
 620     {
 621         /* Get the descriptor */
 622         Descriptor = CONTAINING_RECORD(ListEntry,
 623                                        MEMORY_ALLOCATION_DESCRIPTOR,
 624                                        ListEntry);
 625
 626         /* Skip invisible memory */
 627         if (MiIsMemoryTypeInvisible(Descriptor->MemoryType)) continue;
 628
 629         /* If this is the free descriptor, use the copy instead */
 630         if (Descriptor == MxFreeDescriptor) Descriptor = &MxOldFreeDescriptor;
 631
 632         /* Get the range for this descriptor */
 633         BasePage = Descriptor->BasePage;
 634         PageCount = Descriptor->PageCount;
 635
 636         /* Map the pages for the database */
 637         MiMapPTEs(&MmPfnDatabase[BasePage],
 638                   (PUCHAR)(&MmPfnDatabase[BasePage + PageCount]) - 1);
 639
 640         /* If this was the free descriptor, skip the next step */
 641         if (Descriptor == &MxOldFreeDescriptor) continue;
 642
 643         /* Add this descriptor to the database */
 644         MiAddDescriptorToDatabase(BasePage, PageCount, Descriptor->MemoryType);
 645     }
 646
 647     /* At this point the whole pfn database is mapped. We are about to add the
 648        pages from the free descriptor to the database, so from now on we cannot
 649        use it anymore. */
 650
 651     /* Now add the free descriptor */
 652     BasePage = MxFreeDescriptor->BasePage;
 653     PageCount = MxFreeDescriptor->PageCount;
 654     MiAddDescriptorToDatabase(BasePage, PageCount, LoaderFree);
 655
 656     /* And finally the memory we used */
 657     BasePage = MxOldFreeDescriptor.BasePage;
 658     PageCount = MxFreeDescriptor->BasePage - BasePage;
 659     MiAddDescriptorToDatabase(BasePage, PageCount, LoaderMemoryData);
 660
 661     /* Reset the descriptor back so we can create the correct memory blocks */
 662     *MxFreeDescriptor = MxOldFreeDescriptor;
 663 }
 664
 665 NTSTATUS
 666 NTAPI
 667 INIT_FUNCTION
 668 MiInitMachineDependent(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 669 {
 670     KIRQL OldIrql;
 671
 672     ASSERT(MxPfnAllocation != 0);
 673
 674     /* Set some hardcoded addresses */
 675     MmHyperSpaceEnd = (PVOID)HYPER_SPACE_END;
 676     MmNonPagedSystemStart = (PVOID)MM_SYSTEM_SPACE_START;
 677     MmPfnDatabase = (PVOID)MI_PFN_DATABASE;
 678     MmWorkingSetList = (PVOID)MI_WORKING_SET_LIST;
 679
 680
 681 //    PrototypePte.u.Proto.Valid = 1
 682 //    PrototypePte.u.ReadOnly
 683 //    PrototypePte.u.Prototype
 684 //    PrototypePte.u.Protection = MM_READWRITE;
 685 //    PrototypePte.u.ProtoAddress
 686     PrototypePte.u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
 687
 688
 689     MiInitializePageTable();
 690
 691     MiBuildNonPagedPool();
 692
 693     MiBuildSystemPteSpace();
 694
 695     /* Need to be at DISPATCH_LEVEL for MiInsertPageInFreeList */
 696     KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
 697
 698     /* Map the PFN database pages */
 699     MiBuildPfnDatabase(LoaderBlock);
 700
 701     /* Now process the page tables */
 702     MiBuildPfnDatabaseFromPageTables();
 703
 704     /* PFNs are initialized now! */
 705     MiPfnsInitialized = TRUE;
 706
 707     //KeLowerIrql(OldIrql);
 708
 709     /* Need to be at DISPATCH_LEVEL for InitializePool */
 710     //KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
 711
 712     /* Initialize the nonpaged pool */
 713     InitializePool(NonPagedPool, 0);
 714
 715     KeLowerIrql(OldIrql);
 716
 717     /* Initialize the balancer */
 718     MmInitializeBalancer((ULONG)MmAvailablePages, 0);
 719
 720     /* Make sure we have everything we need */
 721     ASSERT(MmPfnDatabase);
 722     ASSERT(MmNonPagedSystemStart);
 723     ASSERT(MmNonPagedPoolStart);
 724     ASSERT(MmSizeOfNonPagedPoolInBytes);
 725     ASSERT(MmMaximumNonPagedPoolInBytes);
 726     ASSERT(MmNonPagedPoolExpansionStart);
 727     ASSERT(MmHyperSpaceEnd);
 728     ASSERT(MmNumberOfSystemPtes);
 729     ASSERT(MiAddressToPde(MmNonPagedPoolStart)->u.Hard.Valid);
 730     ASSERT(MiAddressToPte(MmNonPagedPoolStart)->u.Hard.Valid);
 731
 732     return STATUS_SUCCESS;
 733 }