ntoskrnl/mm/amd64/init.c

   1 /*
   2  * COPYRIGHT:       GPL, See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/amd64/init.c
   5  * PURPOSE:         Memory Manager Initialization for amd64
   6  *
   7  * PROGRAMMERS:     Timo kreuzer (timo.kreuzer@reactos.org)
   8  *                  ReactOS Portable Systems Group
   9  */
  10
  11 /* INCLUDES ***************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 //#define NDEBUG
  15 #include <debug.h>
  16
  17 #include <mm/ARM3/miarm.h>
  18
  19 #ifdef _WINKD_
  20 extern PMMPTE MmDebugPte;
  21 #endif
  22
  23 /* Helper macros */
  24 #define IS_ALIGNED(addr, align) (((ULONG64)(addr) & (align - 1)) == 0)
  25 #define IS_PAGE_ALIGNED(addr) IS_ALIGNED(addr, PAGE_SIZE)
  26
  27 /* GLOBALS *****************************************************************/
  28
  29 /* Template PTE and PDE for a kernel page */
  30 MMPTE ValidKernelPde = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  31 MMPTE ValidKernelPte = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  32
  33 /* The same, but for local pages */
  34 MMPTE ValidKernelPdeLocal = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  35 MMPTE ValidKernelPteLocal = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  36
  37 /* Template PDE for a demand-zero page */
  38 MMPDE DemandZeroPde  = {{MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  39 MMPTE DemandZeroPte  = {{MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  40
  41 /* Template PTE for prototype page */
  42 MMPTE PrototypePte = {{(MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS) |
  43                       PTE_PROTOTYPE | (MI_PTE_LOOKUP_NEEDED << 32)}};
  44
  45 /* Template PTE for decommited page */
  46 MMPTE MmDecommittedPte = {{MM_DECOMMIT << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  47
  48 /* Address ranges */
  49 PVOID MiSessionViewEnd;
  50 PVOID MiSystemPteSpaceStart;
  51 PVOID MiSystemPteSpaceEnd;
  52
  53 ULONG64 MxPfnSizeInBytes;
  54 BOOLEAN MiIncludeType[LoaderMaximum];
  55 PFN_NUMBER MxFreePageBase;
  56 ULONG64 MxFreePageCount = 0;
  57
  58 BOOLEAN MiPfnsInitialized = FALSE;
  59
  60 /* FUNCTIONS *****************************************************************/
  61
  62 VOID
  63 NTAPI
  64 INIT_FUNCTION
  65 MiInitializeSessionSpaceLayout(VOID)
  66 {
  67     MmSessionSize = MI_SESSION_SIZE;
  68     MmSessionViewSize = MI_SESSION_VIEW_SIZE;
  69     MmSessionPoolSize = MI_SESSION_POOL_SIZE;
  70     MmSessionImageSize = MI_SESSION_IMAGE_SIZE;
  71     MmSystemViewSize = MI_SYSTEM_VIEW_SIZE;
  72
  73     /* Set up session space */
  74     MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END;
  75
  76     /* This is where we will load Win32k.sys and the video driver */
  77     MiSessionImageEnd = MiSessionSpaceEnd;
  78     MiSessionImageStart = (PCHAR)MiSessionImageEnd - MmSessionImageSize;
  79
  80     /* The view starts right below the session working set (itself below
  81      * the image area) */
  82     MiSessionViewEnd = (PVOID)MI_SESSION_VIEW_END;
  83     MiSessionViewStart = (PCHAR)MiSessionViewEnd - MmSessionViewSize;
  84     ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart));
  85
  86     /* Session pool follows */
  87     MiSessionPoolEnd = MiSessionViewStart;
  88     MiSessionPoolStart = (PCHAR)MiSessionPoolEnd - MmSessionPoolSize;
  89     ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart));
  90
  91     /* And it all begins here */
  92     MmSessionBase = MiSessionPoolStart;
  93
  94     /* System view space ends at session space, so now that we know where
  95      * this is, we can compute the base address of system view space itself. */
  96     MiSystemViewStart = (PCHAR)MmSessionBase - MmSystemViewSize;
  97     ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart));
  98
  99     /* Sanity checks */
 100     ASSERT(MiSessionViewEnd <= MiSessionImageStart);
 101     ASSERT(MmSessionBase <= MiSessionPoolStart);
 102 }
 103
 104 VOID
 105 NTAPI
 106 MiMapPPEs(
 107     PVOID StartAddress,
 108     PVOID EndAddress)
 109 {
 110     PMMPDE PointerPpe;
 111     MMPDE TmplPde = ValidKernelPde;
 112
 113     /* Loop the PPEs */
 114     for (PointerPpe = MiAddressToPpe(StartAddress);
 115          PointerPpe <= MiAddressToPpe(EndAddress);
 116          PointerPpe++)
 117     {
 118         /* Check if its already mapped */
 119         if (!PointerPpe->u.Hard.Valid)
 120         {
 121             /* No, map it! */
 122             TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
 123             MI_WRITE_VALID_PTE(PointerPpe, TmplPde);
 124
 125             /* Zero out the page table */
 126             RtlZeroMemory(MiPteToAddress(PointerPpe), PAGE_SIZE);
 127         }
 128     }
 129 }
 130
 131 VOID
 132 NTAPI
 133 MiMapPDEs(
 134     PVOID StartAddress,
 135     PVOID EndAddress)
 136 {
 137     PMMPDE PointerPde;
 138     MMPDE TmplPde = ValidKernelPde;
 139
 140     /* Loop the PDEs */
 141     for (PointerPde = MiAddressToPde(StartAddress);
 142          PointerPde <= MiAddressToPde(EndAddress);
 143          PointerPde++)
 144     {
 145         /* Check if its already mapped */
 146         if (!PointerPde->u.Hard.Valid)
 147         {
 148             /* No, map it! */
 149             TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
 150             MI_WRITE_VALID_PTE(PointerPde, TmplPde);
 151
 152             /* Zero out the page table */
 153             RtlZeroMemory(MiPteToAddress(PointerPde), PAGE_SIZE);
 154         }
 155     }
 156 }
 157
 158 VOID
 159 NTAPI
 160 MiMapPTEs(
 161     PVOID StartAddress,
 162     PVOID EndAddress)
 163 {
 164     PMMPTE PointerPte;
 165     MMPTE TmplPte = ValidKernelPte;
 166
 167     /* Loop the PTEs */
 168     for (PointerPte = MiAddressToPte(StartAddress);
 169          PointerPte <= MiAddressToPte(EndAddress);
 170          PointerPte++)
 171     {
 172         /* Check if its already mapped */
 173         if (!PointerPte->u.Hard.Valid)
 174         {
 175             /* No, map it! */
 176             TmplPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
 177             MI_WRITE_VALID_PTE(PointerPte, TmplPte);
 178
 179             /* Zero out the page (FIXME: not always neccessary) */
 180             RtlZeroMemory(MiPteToAddress(PointerPte), PAGE_SIZE);
 181         }
 182     }
 183 }
 184
 185 VOID
 186 NTAPI
 187 INIT_FUNCTION
 188 MiInitializePageTable(VOID)
 189 {
 190     ULONG64 PxePhysicalAddress;
 191     MMPTE TmplPte, *PointerPxe;
 192     PFN_NUMBER PxePfn;
 193
 194     /* Get current directory base */
 195     PxePfn = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber;
 196     PxePhysicalAddress = PxePfn << PAGE_SHIFT;
 197     ASSERT(PxePhysicalAddress == __readcr3());
 198
 199     /* Set directory base for the system process */
 200     PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PxePhysicalAddress;
 201
 202     /* Enable global pages */
 203     __writecr4(__readcr4() | CR4_PGE);
 204     ASSERT(__readcr4() & CR4_PGE);
 205
 206     /* Enable no execute */
 207     __writemsr(X86_MSR_EFER, __readmsr(X86_MSR_EFER) | EFER_NXE);
 208
 209     /* Loop the user mode PXEs */
 210     for (PointerPxe = MiAddressToPxe(0);
 211          PointerPxe <= MiAddressToPxe(MmHighestUserAddress);
 212          PointerPxe++)
 213     {
 214         /* Zero the PXE, clear all mappings */
 215         PointerPxe->u.Long = 0;
 216     }
 217
 218     /* Flush the TLB */
 219     KeFlushCurrentTb();
 220
 221     /* Set up a template PTE */
 222     TmplPte.u.Long = 0;
 223     TmplPte.u.Flush.Valid = 1;
 224     TmplPte.u.Flush.Write = 1;
 225     HyperTemplatePte = TmplPte;
 226
 227     /* Create PDPTs (72 KB) for shared system address space,
 228      * skip page tables TODO: use global pages. */
 229
 230     /* Loop the PXEs */
 231     for (PointerPxe = MiAddressToPxe((PVOID)HYPER_SPACE);
 232          PointerPxe <= MiAddressToPxe(MI_HIGHEST_SYSTEM_ADDRESS);
 233          PointerPxe++)
 234     {
 235         /* Is the PXE already valid? */
 236         if (!PointerPxe->u.Hard.Valid)
 237         {
 238             /* It's not Initialize it */
 239             TmplPte.u.Flush.PageFrameNumber = MxGetNextPage(1);
 240             *PointerPxe = TmplPte;
 241
 242             /* Zero the page. The PXE is the PTE for the PDPT. */
 243             RtlZeroMemory(MiPteToAddress(PointerPxe), PAGE_SIZE);
 244         }
 245     }
 246
 247     /* Map PPEs for paged pool */
 248     MiMapPPEs(MmPagedPoolStart, MmPagedPoolEnd);
 249
 250     /* Setup 1 PPE for hyper space */
 251     MiMapPPEs((PVOID)HYPER_SPACE, (PVOID)HYPER_SPACE_END);
 252
 253     /* Setup PPEs for system space view */
 254     MiMapPPEs(MiSystemViewStart, (PCHAR)MiSystemViewStart + MmSystemViewSize);
 255
 256     /* Setup the mapping PDEs */
 257     MiMapPDEs((PVOID)MI_MAPPING_RANGE_START, (PVOID)MI_MAPPING_RANGE_END);
 258
 259     /* Setup the mapping PTEs */
 260     MmFirstReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_START);
 261     MmLastReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_END);
 262     MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES;
 263
 264 #ifdef _WINKD_
 265     /* Setup debug mapping PTE */
 266     MiMapPPEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
 267     MiMapPDEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
 268     MmDebugPte = MiAddressToPte((PVOID)MI_DEBUG_MAPPING);
 269 #endif
 270
 271     /* Setup PDE and PTEs for VAD bitmap and working set list */
 272     MiMapPDEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
 273     MiMapPTEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
 274 }
 275
 276 VOID
 277 NTAPI
 278 INIT_FUNCTION
 279 MiBuildNonPagedPool(VOID)
 280 {
 281     /* Check if this is a machine with less than 256MB of RAM, and no overide */
 282     if ((MmNumberOfPhysicalPages <= MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING) &&
 283         !(MmSizeOfNonPagedPoolInBytes))
 284     {
 285         /* Force the non paged pool to be 2MB so we can reduce RAM usage */
 286         MmSizeOfNonPagedPoolInBytes = 2 * 1024 * 1024;
 287     }
 288
 289     /* Check if the user gave a ridicuously large nonpaged pool RAM size */
 290     if ((MmSizeOfNonPagedPoolInBytes >> PAGE_SHIFT) >
 291         (MmNumberOfPhysicalPages * 7 / 8))
 292     {
 293         /* More than 7/8ths of RAM was dedicated to nonpaged pool, ignore! */
 294         MmSizeOfNonPagedPoolInBytes = 0;
 295     }
 296
 297     /* Check if no registry setting was set, or if the setting was too low */
 298     if (MmSizeOfNonPagedPoolInBytes < MmMinimumNonPagedPoolSize)
 299     {
 300         /* Start with the minimum (256 KB) and add 32 KB for each MB above 4 */
 301         MmSizeOfNonPagedPoolInBytes = MmMinimumNonPagedPoolSize;
 302         MmSizeOfNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
 303                                        256 * MmMinAdditionNonPagedPoolPerMb;
 304     }
 305
 306     /* Check if the registy setting or our dynamic calculation was too high */
 307     if (MmSizeOfNonPagedPoolInBytes > MI_MAX_INIT_NONPAGED_POOL_SIZE)
 308     {
 309         /* Set it to the maximum */
 310         MmSizeOfNonPagedPoolInBytes = MI_MAX_INIT_NONPAGED_POOL_SIZE;
 311     }
 312
 313     /* Check if a percentage cap was set through the registry */
 314     if (MmMaximumNonPagedPoolPercent)
 315     {
 316         /* Don't feel like supporting this right now */
 317         UNIMPLEMENTED;
 318     }
 319
 320     /* Page-align the nonpaged pool size */
 321     MmSizeOfNonPagedPoolInBytes &= ~(PAGE_SIZE - 1);
 322
 323     /* Now, check if there was a registry size for the maximum size */
 324     if (!MmMaximumNonPagedPoolInBytes)
 325     {
 326         /* Start with the default (1MB) and add 400 KB for each MB above 4 */
 327         MmMaximumNonPagedPoolInBytes = MmDefaultMaximumNonPagedPool;
 328         MmMaximumNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
 329                                          256 * MmMaxAdditionNonPagedPoolPerMb;
 330     }
 331
 332     /* Don't let the maximum go too high */
 333     if (MmMaximumNonPagedPoolInBytes > MI_MAX_NONPAGED_POOL_SIZE)
 334     {
 335         /* Set it to the upper limit */
 336         MmMaximumNonPagedPoolInBytes = MI_MAX_NONPAGED_POOL_SIZE;
 337     }
 338
 339     /* Convert nonpaged pool size from bytes to pages */
 340     MmMaximumNonPagedPoolInPages = MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT;
 341
 342     /* Non paged pool starts after the PFN database */
 343     MmNonPagedPoolStart = MmPfnDatabase + MxPfnAllocation * PAGE_SIZE;
 344
 345     /* Calculate the nonpaged pool expansion start region */
 346     MmNonPagedPoolExpansionStart = (PCHAR)MmNonPagedPoolStart +
 347                                           MmSizeOfNonPagedPoolInBytes;
 348     ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolExpansionStart));
 349
 350     /* And this is where the none paged pool ends */
 351     MmNonPagedPoolEnd = (PCHAR)MmNonPagedPoolStart + MmMaximumNonPagedPoolInBytes;
 352     ASSERT(MmNonPagedPoolEnd < (PVOID)MM_HAL_VA_START);
 353
 354     /* Map PPEs and PDEs for non paged pool (including expansion) */
 355     MiMapPPEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
 356     MiMapPDEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
 357
 358     /* Map the nonpaged pool PTEs (without expansion) */
 359     MiMapPTEs(MmNonPagedPoolStart, (PCHAR)MmNonPagedPoolExpansionStart - 1);
 360
 361     /* Initialize the ARM3 nonpaged pool */
 362     MiInitializeNonPagedPool();
 363     MiInitializeNonPagedPoolThresholds();
 364
 365 }
 366
 367 VOID
 368 NTAPI
 369 INIT_FUNCTION
 370 MiBuildSystemPteSpace(VOID)
 371 {
 372     PMMPTE PointerPte;
 373     SIZE_T NonPagedSystemSize;
 374
 375     /* Use the default number of system PTEs */
 376     MmNumberOfSystemPtes = MI_NUMBER_SYSTEM_PTES;
 377     NonPagedSystemSize = (MmNumberOfSystemPtes + 1) * PAGE_SIZE;
 378
 379     /* Put system PTEs at the start of the system VA space */
 380     MiSystemPteSpaceStart = MmNonPagedSystemStart;
 381     MiSystemPteSpaceEnd = (PUCHAR)MiSystemPteSpaceStart + NonPagedSystemSize;
 382
 383     /* Map the PPEs and PDEs for the system PTEs */
 384     MiMapPPEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
 385     MiMapPDEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
 386
 387     /* Initialize the system PTE space */
 388     PointerPte = MiAddressToPte(MiSystemPteSpaceStart);
 389     MiInitializeSystemPtes(PointerPte, MmNumberOfSystemPtes, SystemPteSpace);
 390
 391     /* Reserve system PTEs for zeroing PTEs and clear them */
 392     MiFirstReservedZeroingPte = MiReserveSystemPtes(MI_ZERO_PTES, SystemPteSpace);
 393     RtlZeroMemory(MiFirstReservedZeroingPte, MI_ZERO_PTES * sizeof(MMPTE));
 394
 395     /* Set the counter to maximum */
 396     MiFirstReservedZeroingPte->u.Hard.PageFrameNumber = MI_ZERO_PTES - 1;
 397 }
 398
 399 static
 400 VOID
 401 MiSetupPfnForPageTable(
 402     PFN_NUMBER PageFrameIndex,
 403     PMMPTE PointerPte)
 404 {
 405     PMMPFN Pfn;
 406     PMMPDE PointerPde;
 407
 408     /* Get the pfn entry for this page */
 409     Pfn = MiGetPfnEntry(PageFrameIndex);
 410
 411     /* Check if it's valid memory */
 412     if ((PageFrameIndex <= MmHighestPhysicalPage) &&
 413         (MmIsAddressValid(Pfn)) &&
 414         (Pfn->u3.e1.PageLocation == ActiveAndValid))
 415     {
 416         /* Setup the PFN entry */
 417         Pfn->u1.WsIndex = 0;
 418         Pfn->u2.ShareCount++;
 419         Pfn->PteAddress = PointerPte;
 420         Pfn->OriginalPte = *PointerPte;
 421         Pfn->u3.e1.PageLocation = ActiveAndValid;
 422         Pfn->u3.e1.CacheAttribute = MiNonCached;
 423         Pfn->u3.e2.ReferenceCount = 1;
 424         Pfn->u4.PteFrame = PFN_FROM_PTE(MiAddressToPte(PointerPte));
 425     }
 426
 427     /* Increase the shared count of the PFN entry for the PDE */
 428     PointerPde = MiAddressToPde(MiPteToAddress(PointerPte));
 429     Pfn = MiGetPfnEntry(PFN_FROM_PTE(PointerPde));
 430     Pfn->u2.ShareCount++;
 431 }
 432
 433 VOID
 434 NTAPI
 435 MiBuildPfnDatabaseFromPageTables(VOID)
 436 {
 437     PVOID Address = NULL;
 438     PFN_NUMBER PageFrameIndex;
 439     PMMPDE PointerPde;
 440     PMMPTE PointerPte;
 441     ULONG k, l;
 442     PMMPFN Pfn;
 443 #if (_MI_PAGING_LEVELS >= 3)
 444     PMMPDE PointerPpe;
 445     ULONG j;
 446 #endif
 447 #if (_MI_PAGING_LEVELS == 4)
 448     PMMPDE PointerPxe;
 449     ULONG i;
 450 #endif
 451
 452     /* Manual setup of the top level page directory */
 453 #if (_MI_PAGING_LEVELS == 4)
 454     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PXE_BASE));
 455 #elif (_MI_PAGING_LEVELS == 3)
 456     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PPE_BASE));
 457 #else
 458     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PDE_BASE));
 459 #endif
 460     Pfn = MiGetPfnEntry(PageFrameIndex);
 461     ASSERT(Pfn->u3.e1.PageLocation == ActiveAndValid);
 462     Pfn->u1.WsIndex = 0;
 463     Pfn->u2.ShareCount = 1;
 464     Pfn->PteAddress = NULL;
 465     Pfn->u3.e1.CacheAttribute = MiNonCached;
 466     Pfn->u3.e2.ReferenceCount = 1;
 467     Pfn->u4.PteFrame = 0;
 468
 469 #if (_MI_PAGING_LEVELS == 4)
 470     /* Loop all PXEs in the PML4 */
 471     PointerPxe = MiAddressToPxe(Address);
 472     for (i = 0; i < PXE_PER_PAGE; i++, PointerPxe++)
 473     {
 474         /* Skip invalid PXEs */
 475         if (!PointerPxe->u.Hard.Valid) continue;
 476
 477         /* Handle the PFN */
 478         PageFrameIndex = PFN_FROM_PXE(PointerPxe);
 479         MiSetupPfnForPageTable(PageFrameIndex, PointerPxe);
 480
 481         /* Get starting VA for this PXE */
 482         Address = MiPxeToAddress(PointerPxe);
 483 #endif
 484 #if (_MI_PAGING_LEVELS >= 3)
 485         /* Loop all PPEs in this PDP */
 486         PointerPpe = MiAddressToPpe(Address);
 487         for (j = 0; j < PPE_PER_PAGE; j++, PointerPpe++)
 488         {
 489             /* Skip invalid PPEs */
 490             if (!PointerPpe->u.Hard.Valid) continue;
 491
 492             /* Handle the PFN */
 493             PageFrameIndex = PFN_FROM_PPE(PointerPpe);
 494             MiSetupPfnForPageTable(PageFrameIndex, PointerPpe);
 495
 496             /* Get starting VA for this PPE */
 497             Address = MiPpeToAddress(PointerPpe);
 498 #endif
 499             /* Loop all PDEs in this PD */
 500             PointerPde = MiAddressToPde(Address);
 501             for (k = 0; k < PDE_PER_PAGE; k++, PointerPde++)
 502             {
 503                 /* Skip invalid PDEs */
 504                 if (!PointerPde->u.Hard.Valid) continue;
 505
 506                 /* Handle the PFN */
 507                 PageFrameIndex = PFN_FROM_PDE(PointerPde);
 508                 MiSetupPfnForPageTable(PageFrameIndex, PointerPde);
 509
 510                 /* Get starting VA for this PDE */
 511                 Address = MiPdeToAddress(PointerPde);
 512
 513                 /* Loop all PTEs in this PT */
 514                 PointerPte = MiAddressToPte(Address);
 515                 for (l = 0; l < PTE_PER_PAGE; l++, PointerPte++)
 516                 {
 517                     /* Skip invalid PTEs */
 518                     if (!PointerPte->u.Hard.Valid) continue;
 519
 520                     /* Handle the PFN */
 521                     PageFrameIndex = PFN_FROM_PTE(PointerPte);
 522                     MiSetupPfnForPageTable(PageFrameIndex, PointerPte);
 523                 }
 524             }
 525 #if (_MI_PAGING_LEVELS >= 3)
 526         }
 527 #endif
 528 #if (_MI_PAGING_LEVELS == 4)
 529     }
 530 #endif
 531 }
 532
 533 VOID
 534 NTAPI
 535 INIT_FUNCTION
 536 MiAddDescriptorToDatabase(
 537     PFN_NUMBER BasePage,
 538     PFN_NUMBER PageCount,
 539     TYPE_OF_MEMORY MemoryType)
 540 {
 541     PMMPFN Pfn;
 542
 543     ASSERT(!MiIsMemoryTypeInvisible(MemoryType));
 544
 545     /* Check if the memory is free */
 546     if (MiIsMemoryTypeFree(MemoryType))
 547     {
 548         /* Get the last pfn of this descriptor. Note we loop backwards */
 549         Pfn = &MmPfnDatabase[BasePage + PageCount - 1];
 550
 551         /* Loop all pages */
 552         while (PageCount--)
 553         {
 554             /* Add it to the free list */
 555             Pfn->u3.e1.CacheAttribute = MiNonCached;
 556             MiInsertPageInFreeList(BasePage + PageCount);
 557
 558             /* Go to the previous page */
 559             Pfn--;
 560         }
 561     }
 562     else if (MemoryType == LoaderXIPRom)
 563     {
 564         Pfn = &MmPfnDatabase[BasePage];
 565         while (PageCount--)
 566         {
 567             /* Make it a pseudo-I/O ROM mapping */
 568             Pfn->PteAddress = 0;
 569             Pfn->u1.Flink = 0;
 570             Pfn->u2.ShareCount = 0;
 571             Pfn->u3.e1.PageLocation = 0;
 572             Pfn->u3.e1.CacheAttribute = MiNonCached;
 573             Pfn->u3.e1.Rom = 1;
 574             Pfn->u3.e1.PrototypePte = 1;
 575             Pfn->u3.e2.ReferenceCount = 0;
 576             Pfn->u4.InPageError = 0;
 577             Pfn->u4.PteFrame = 0;
 578
 579             /* Advance one */
 580             Pfn++;
 581         }
 582     }
 583     else if (MemoryType == LoaderBad)
 584     {
 585         // FIXME: later
 586         ASSERT(FALSE);
 587     }
 588     else
 589     {
 590         /* For now skip it */
 591         DbgPrint("Skipping BasePage=0x%lx, PageCount=0x%lx, MemoryType=%lx\n",
 592                  BasePage, PageCount, MemoryType);
 593         Pfn = &MmPfnDatabase[BasePage];
 594         while (PageCount--)
 595         {
 596             /* Make an active PFN */
 597             Pfn->u3.e1.PageLocation = ActiveAndValid;
 598
 599             /* Advance one */
 600             Pfn++;
 601         }
 602     }
 603 }
 604
 605 VOID
 606 NTAPI
 607 INIT_FUNCTION
 608 MiBuildPfnDatabase(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 609 {
 610     PLIST_ENTRY ListEntry;
 611     PMEMORY_ALLOCATION_DESCRIPTOR Descriptor;
 612     PFN_NUMBER BasePage, PageCount;
 613
 614     /* Map the PDEs and PPEs for the pfn database (ignore holes) */
 615 #if (_MI_PAGING_LEVELS >= 3)
 616     MiMapPPEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + MxPfnAllocation - 1);
 617 #endif
 618     MiMapPDEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + MxPfnAllocation - 1);
 619
 620     /* First initialize the color tables */
 621     MiInitializeColorTables();
 622
 623     /* Loop the memory descriptors */
 624     for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
 625          ListEntry != &LoaderBlock->MemoryDescriptorListHead;
 626          ListEntry = ListEntry->Flink)
 627     {
 628         /* Get the descriptor */
 629         Descriptor = CONTAINING_RECORD(ListEntry,
 630                                        MEMORY_ALLOCATION_DESCRIPTOR,
 631                                        ListEntry);
 632
 633         /* Skip invisible memory */
 634         if (MiIsMemoryTypeInvisible(Descriptor->MemoryType)) continue;
 635
 636         /* If this is the free descriptor, use the copy instead */
 637         if (Descriptor == MxFreeDescriptor) Descriptor = &MxOldFreeDescriptor;
 638
 639         /* Get the range for this descriptor */
 640         BasePage = Descriptor->BasePage;
 641         PageCount = Descriptor->PageCount;
 642
 643         /* Map the pages for the database */
 644         MiMapPTEs(&MmPfnDatabase[BasePage],
 645                   (PUCHAR)(&MmPfnDatabase[BasePage + PageCount]) - 1);
 646
 647         /* If this was the free descriptor, skip the next step */
 648         if (Descriptor == &MxOldFreeDescriptor) continue;
 649
 650         /* Add this descriptor to the database */
 651         MiAddDescriptorToDatabase(BasePage, PageCount, Descriptor->MemoryType);
 652     }
 653
 654     /* At this point the whole pfn database is mapped. We are about to add the
 655        pages from the free descriptor to the database, so from now on we cannot
 656        use it anymore. */
 657
 658     /* Now add the free descriptor */
 659     BasePage = MxFreeDescriptor->BasePage;
 660     PageCount = MxFreeDescriptor->PageCount;
 661     MiAddDescriptorToDatabase(BasePage, PageCount, LoaderFree);
 662
 663     /* And finally the memory we used */
 664     BasePage = MxOldFreeDescriptor.BasePage;
 665     PageCount = MxFreeDescriptor->BasePage - BasePage;
 666     MiAddDescriptorToDatabase(BasePage, PageCount, LoaderMemoryData);
 667
 668     /* Reset the descriptor back so we can create the correct memory blocks */
 669     *MxFreeDescriptor = MxOldFreeDescriptor;
 670 }
 671
 672 NTSTATUS
 673 NTAPI
 674 INIT_FUNCTION
 675 MiInitMachineDependent(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 676 {
 677     KIRQL OldIrql;
 678
 679     ASSERT(MxPfnAllocation != 0);
 680
 681     /* Set some hardcoded addresses */
 682     MmHyperSpaceEnd = (PVOID)HYPER_SPACE_END;
 683     MmNonPagedSystemStart = (PVOID)MM_SYSTEM_SPACE_START;
 684     MmPfnDatabase = (PVOID)MI_PFN_DATABASE;
 685     MmWorkingSetList = (PVOID)MI_WORKING_SET_LIST;
 686
 687
 688 //    PrototypePte.u.Proto.Valid = 1
 689 //    PrototypePte.u.ReadOnly
 690 //    PrototypePte.u.Prototype
 691 //    PrototypePte.u.Protection = MM_READWRITE;
 692 //    PrototypePte.u.ProtoAddress
 693     PrototypePte.u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
 694
 695
 696     MiInitializePageTable();
 697
 698     MiBuildNonPagedPool();
 699
 700     MiBuildSystemPteSpace();
 701
 702     /* Need to be at DISPATCH_LEVEL for MiInsertPageInFreeList */
 703     KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
 704
 705     /* Map the PFN database pages */
 706     MiBuildPfnDatabase(LoaderBlock);
 707
 708     /* Now process the page tables */
 709     MiBuildPfnDatabaseFromPageTables();
 710
 711     /* PFNs are initialized now! */
 712     MiPfnsInitialized = TRUE;
 713
 714     //KeLowerIrql(OldIrql);
 715
 716     /* Need to be at DISPATCH_LEVEL for InitializePool */
 717     //KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
 718
 719     /* Initialize the nonpaged pool */
 720     InitializePool(NonPagedPool, 0);
 721
 722     KeLowerIrql(OldIrql);
 723
 724     /* Initialize the balancer */
 725     MmInitializeBalancer((ULONG)MmAvailablePages, 0);
 726
 727     /* Make sure we have everything we need */
 728     ASSERT(MmPfnDatabase);
 729     ASSERT(MmNonPagedSystemStart);
 730     ASSERT(MmNonPagedPoolStart);
 731     ASSERT(MmSizeOfNonPagedPoolInBytes);
 732     ASSERT(MmMaximumNonPagedPoolInBytes);
 733     ASSERT(MmNonPagedPoolExpansionStart);
 734     ASSERT(MmHyperSpaceEnd);
 735     ASSERT(MmNumberOfSystemPtes);
 736     ASSERT(MiAddressToPde(MmNonPagedPoolStart)->u.Hard.Valid);
 737     ASSERT(MiAddressToPte(MmNonPagedPoolStart)->u.Hard.Valid);
 738
 739     return STATUS_SUCCESS;
 740 }