reactos/ntoskrnl/mm/amd64/init.c

   1 /*
   2  * COPYRIGHT:       GPL, See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/amd64/init.c
   5  * PURPOSE:         Memory Manager Initialization for amd64
   6  *
   7  * PROGRAMMERS:     Timo kreuzer (timo.kreuzer@reactos.org)
   8  *                  ReactOS Portable Systems Group
   9  */
  10
  11 /* INCLUDES ***************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 //#define NDEBUG
  15 #include <debug.h>
  16
  17 #include "../ARM3/miarm.h"
  18
  19 #ifdef _WINKD_
  20 extern PMMPTE MmDebugPte;
  21 #endif
  22
  23 /* GLOBALS *****************************************************************/
  24
  25 /* Template PTE and PDE for a kernel page */
  26 MMPTE ValidKernelPde = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  27 MMPTE ValidKernelPte = {{PTE_VALID|PTE_READWRITE|PTE_DIRTY|PTE_ACCESSED}};
  28
  29 /* Template PDE for a demand-zero page */
  30 MMPDE DemandZeroPde  = {{MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  31 MMPTE DemandZeroPte  = {{MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  32
  33 /* Template PTE for prototype page */
  34 MMPTE PrototypePte = {{(MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS) |
  35                       PTE_PROTOTYPE | (MI_PTE_LOOKUP_NEEDED << 32)}};
  36
  37 /* Template PTE for decommited page */
  38 MMPTE MmDecommittedPte = {{MM_DECOMMIT << MM_PTE_SOFTWARE_PROTECTION_BITS}};
  39
  40 /* Address ranges */
  41 PVOID MiSessionViewEnd;
  42 PVOID MiSystemPteSpaceStart;
  43 PVOID MiSystemPteSpaceEnd;
  44
  45 ULONG64 MxPfnSizeInBytes;
  46 BOOLEAN MiIncludeType[LoaderMaximum];
  47 PFN_NUMBER MxFreePageBase;
  48 ULONG64 MxFreePageCount = 0;
  49
  50 BOOLEAN MiPfnsInitialized = FALSE;
  51
  52 /* FUNCTIONS *****************************************************************/
  53
  54 VOID
  55 NTAPI
  56 INIT_FUNCTION
  57 MiInitializeSessionSpaceLayout()
  58 {
  59     MmSessionSize = MI_SESSION_SIZE;
  60     MmSessionViewSize = MI_SESSION_VIEW_SIZE;
  61     MmSessionPoolSize = MI_SESSION_POOL_SIZE;
  62     MmSessionImageSize = MI_SESSION_IMAGE_SIZE;
  63     MmSystemViewSize = MI_SYSTEM_VIEW_SIZE;
  64
  65     /* Set up session space */
  66     MiSessionSpaceEnd = (PVOID)MI_SESSION_SPACE_END;
  67
  68     /* This is where we will load Win32k.sys and the video driver */
  69     MiSessionImageEnd = MiSessionSpaceEnd;
  70     MiSessionImageStart = (PCHAR)MiSessionImageEnd - MmSessionImageSize;
  71
  72     /* The view starts right below the session working set (itself below
  73      * the image area) */
  74     MiSessionViewEnd = (PVOID)MI_SESSION_VIEW_END;
  75     MiSessionViewStart = (PCHAR)MiSessionViewEnd - MmSessionViewSize;
  76     ASSERT(IS_PAGE_ALIGNED(MiSessionViewStart));
  77
  78     /* Session pool follows */
  79     MiSessionPoolEnd = MiSessionViewStart;
  80     MiSessionPoolStart = (PCHAR)MiSessionPoolEnd - MmSessionPoolSize;
  81     ASSERT(IS_PAGE_ALIGNED(MiSessionPoolStart));
  82
  83     /* And it all begins here */
  84     MmSessionBase = MiSessionPoolStart;
  85
  86     /* System view space ends at session space, so now that we know where
  87      * this is, we can compute the base address of system view space itself. */
  88     MiSystemViewStart = (PCHAR)MmSessionBase - MmSystemViewSize;
  89     ASSERT(IS_PAGE_ALIGNED(MiSystemViewStart));
  90
  91     /* Sanity checks */
  92     ASSERT(MiSessionViewEnd <= MiSessionImageStart);
  93     ASSERT(MmSessionBase <= MiSessionPoolStart);
  94 }
  95
  96 VOID
  97 NTAPI
  98 MiMapPPEs(
  99     PVOID StartAddress,
 100     PVOID EndAddress)
 101 {
 102     PMMPDE PointerPpe;
 103     MMPDE TmplPde = ValidKernelPde;
 104
 105     /* Loop the PPEs */
 106     for (PointerPpe = MiAddressToPpe(StartAddress);
 107          PointerPpe <= MiAddressToPpe(EndAddress);
 108          PointerPpe++)
 109     {
 110         /* Check if its already mapped */
 111         if (!PointerPpe->u.Hard.Valid)
 112         {
 113             /* No, map it! */
 114             TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
 115             MI_WRITE_VALID_PTE(PointerPpe, TmplPde);
 116
 117             /* Zero out the page table */
 118             RtlZeroMemory(MiPteToAddress(PointerPpe), PAGE_SIZE);
 119         }
 120     }
 121 }
 122
 123 VOID
 124 NTAPI
 125 MiMapPDEs(
 126     PVOID StartAddress,
 127     PVOID EndAddress)
 128 {
 129     PMMPDE PointerPde;
 130     MMPDE TmplPde = ValidKernelPde;
 131
 132     /* Loop the PDEs */
 133     for (PointerPde = MiAddressToPde(StartAddress);
 134          PointerPde <= MiAddressToPde(EndAddress);
 135          PointerPde++)
 136     {
 137         /* Check if its already mapped */
 138         if (!PointerPde->u.Hard.Valid)
 139         {
 140             /* No, map it! */
 141             TmplPde.u.Hard.PageFrameNumber = MxGetNextPage(1);
 142             MI_WRITE_VALID_PTE(PointerPde, TmplPde);
 143
 144             /* Zero out the page table */
 145             RtlZeroMemory(MiPteToAddress(PointerPde), PAGE_SIZE);
 146         }
 147     }
 148 }
 149
 150 VOID
 151 NTAPI
 152 MiMapPTEs(
 153     PVOID StartAddress,
 154     PVOID EndAddress)
 155 {
 156     PMMPTE PointerPte;
 157     MMPTE TmplPte = ValidKernelPte;
 158
 159     /* Loop the PTEs */
 160     for (PointerPte = MiAddressToPte(StartAddress);
 161          PointerPte <= MiAddressToPte(EndAddress);
 162          PointerPte++)
 163     {
 164         /* Check if its already mapped */
 165         if (!PointerPte->u.Hard.Valid)
 166         {
 167             /* No, map it! */
 168             TmplPte.u.Hard.PageFrameNumber = MxGetNextPage(1);
 169             MI_WRITE_VALID_PTE(PointerPte, TmplPte);
 170
 171             /* Zero out the page (FIXME: not always neccessary) */
 172             RtlZeroMemory(MiPteToAddress(PointerPte), PAGE_SIZE);
 173         }
 174     }
 175 }
 176
 177 VOID
 178 NTAPI
 179 INIT_FUNCTION
 180 MiInitializePageTable()
 181 {
 182     ULONG64 PxePhysicalAddress;
 183     MMPTE TmplPte, *PointerPxe;
 184     PFN_NUMBER PxePfn;
 185
 186     /* Get current directory base */
 187     PxePfn = ((PMMPTE)PXE_SELFMAP)->u.Hard.PageFrameNumber;
 188     PxePhysicalAddress = PxePfn << PAGE_SHIFT;
 189     ASSERT(PxePhysicalAddress == __readcr3());
 190
 191     /* Set directory base for the system process */
 192     PsGetCurrentProcess()->Pcb.DirectoryTableBase[0] = PxePhysicalAddress;
 193
 194     /* Enable global pages */
 195     __writecr4(__readcr4() | CR4_PGE);
 196     ASSERT(__readcr4() & CR4_PGE);
 197
 198     /* Enable no execute */
 199     __writemsr(X86_MSR_EFER, __readmsr(X86_MSR_EFER) | EFER_NXE);
 200
 201     /* Loop the user mode PXEs */
 202     for (PointerPxe = MiAddressToPxe(0);
 203          PointerPxe <= MiAddressToPxe(MmHighestUserAddress);
 204          PointerPxe++)
 205     {
 206         /* Zero the PXE, clear all mappings */
 207         PointerPxe->u.Long = 0;
 208     }
 209
 210     /* Flush the TLB */
 211     KeFlushCurrentTb();
 212
 213     /* Set up a template PTE */
 214     TmplPte.u.Long = 0;
 215     TmplPte.u.Flush.Valid = 1;
 216     TmplPte.u.Flush.Write = 1;
 217     HyperTemplatePte = TmplPte;
 218
 219     /* Create PDPTs (72 KB) for shared system address space,
 220      * skip page tables TODO: use global pages. */
 221
 222     /* Loop the PXEs */
 223     for (PointerPxe = MiAddressToPxe((PVOID)HYPER_SPACE);
 224          PointerPxe <= MiAddressToPxe(MI_HIGHEST_SYSTEM_ADDRESS);
 225          PointerPxe++)
 226     {
 227         /* Is the PXE already valid? */
 228         if (!PointerPxe->u.Hard.Valid)
 229         {
 230             /* It's not Initialize it */
 231             TmplPte.u.Flush.PageFrameNumber = MxGetNextPage(1);
 232             *PointerPxe = TmplPte;
 233
 234             /* Zero the page. The PXE is the PTE for the PDPT. */
 235             RtlZeroMemory(MiPteToAddress(PointerPxe), PAGE_SIZE);
 236         }
 237     }
 238
 239     /* Map PPEs for paged pool */
 240     MiMapPPEs(MmPagedPoolStart, MmPagedPoolEnd);
 241
 242     /* Setup 1 PPE for hyper space */
 243     MiMapPPEs((PVOID)HYPER_SPACE, (PVOID)HYPER_SPACE_END);
 244
 245     /* Setup the mapping PDEs */
 246     MiMapPDEs((PVOID)MI_MAPPING_RANGE_START, (PVOID)MI_MAPPING_RANGE_END);
 247
 248     /* Setup the mapping PTEs */
 249     MmFirstReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_START);
 250     MmLastReservedMappingPte = MiAddressToPte((PVOID)MI_MAPPING_RANGE_END);
 251     MmFirstReservedMappingPte->u.Hard.PageFrameNumber = MI_HYPERSPACE_PTES;
 252
 253 #ifdef _WINKD_
 254     /* Setup debug mapping PTE */
 255     MiMapPPEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
 256     MiMapPDEs((PVOID)MI_DEBUG_MAPPING, (PVOID)MI_DEBUG_MAPPING);
 257     MmDebugPte = MiAddressToPte((PVOID)MI_DEBUG_MAPPING);
 258 #endif
 259
 260     /* Setup PDE and PTEs for VAD bitmap and working set list */
 261     MiMapPDEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
 262     MiMapPTEs((PVOID)MI_VAD_BITMAP, (PVOID)(MI_WORKING_SET_LIST + PAGE_SIZE - 1));
 263 }
 264
 265 VOID
 266 NTAPI
 267 INIT_FUNCTION
 268 MiBuildNonPagedPool(VOID)
 269 {
 270     /* Check if this is a machine with less than 256MB of RAM, and no overide */
 271     if ((MmNumberOfPhysicalPages <= MI_MIN_PAGES_FOR_NONPAGED_POOL_TUNING) &&
 272         !(MmSizeOfNonPagedPoolInBytes))
 273     {
 274         /* Force the non paged pool to be 2MB so we can reduce RAM usage */
 275         MmSizeOfNonPagedPoolInBytes = 2 * 1024 * 1024;
 276     }
 277
 278     /* Check if the user gave a ridicuously large nonpaged pool RAM size */
 279     if ((MmSizeOfNonPagedPoolInBytes >> PAGE_SHIFT) >
 280         (MmNumberOfPhysicalPages * 7 / 8))
 281     {
 282         /* More than 7/8ths of RAM was dedicated to nonpaged pool, ignore! */
 283         MmSizeOfNonPagedPoolInBytes = 0;
 284     }
 285
 286     /* Check if no registry setting was set, or if the setting was too low */
 287     if (MmSizeOfNonPagedPoolInBytes < MmMinimumNonPagedPoolSize)
 288     {
 289         /* Start with the minimum (256 KB) and add 32 KB for each MB above 4 */
 290         MmSizeOfNonPagedPoolInBytes = MmMinimumNonPagedPoolSize;
 291         MmSizeOfNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
 292                                        256 * MmMinAdditionNonPagedPoolPerMb;
 293     }
 294
 295     /* Check if the registy setting or our dynamic calculation was too high */
 296     if (MmSizeOfNonPagedPoolInBytes > MI_MAX_INIT_NONPAGED_POOL_SIZE)
 297     {
 298         /* Set it to the maximum */
 299         MmSizeOfNonPagedPoolInBytes = MI_MAX_INIT_NONPAGED_POOL_SIZE;
 300     }
 301
 302     /* Check if a percentage cap was set through the registry */
 303     if (MmMaximumNonPagedPoolPercent)
 304     {
 305         /* Don't feel like supporting this right now */
 306         UNIMPLEMENTED;
 307     }
 308
 309     /* Page-align the nonpaged pool size */
 310     MmSizeOfNonPagedPoolInBytes &= ~(PAGE_SIZE - 1);
 311
 312     /* Now, check if there was a registry size for the maximum size */
 313     if (!MmMaximumNonPagedPoolInBytes)
 314     {
 315         /* Start with the default (1MB) and add 400 KB for each MB above 4 */
 316         MmMaximumNonPagedPoolInBytes = MmDefaultMaximumNonPagedPool;
 317         MmMaximumNonPagedPoolInBytes += (MmNumberOfPhysicalPages - 1024) /
 318                                          256 * MmMaxAdditionNonPagedPoolPerMb;
 319     }
 320
 321     /* Don't let the maximum go too high */
 322     if (MmMaximumNonPagedPoolInBytes > MI_MAX_NONPAGED_POOL_SIZE)
 323     {
 324         /* Set it to the upper limit */
 325         MmMaximumNonPagedPoolInBytes = MI_MAX_NONPAGED_POOL_SIZE;
 326     }
 327
 328     /* Convert nonpaged pool size from bytes to pages */
 329     MmMaximumNonPagedPoolInPages = MmMaximumNonPagedPoolInBytes >> PAGE_SHIFT;
 330
 331     /* Non paged pool starts after the PFN database */
 332     MmNonPagedPoolStart = MmPfnDatabase + MxPfnAllocation * PAGE_SIZE;
 333
 334     /* Calculate the nonpaged pool expansion start region */
 335     MmNonPagedPoolExpansionStart = (PCHAR)MmNonPagedPoolStart +
 336                                           MmSizeOfNonPagedPoolInBytes;
 337     ASSERT(IS_PAGE_ALIGNED(MmNonPagedPoolExpansionStart));
 338
 339     /* And this is where the none paged pool ends */
 340     MmNonPagedPoolEnd = (PCHAR)MmNonPagedPoolStart + MmMaximumNonPagedPoolInBytes;
 341     ASSERT(MmNonPagedPoolEnd < (PVOID)MM_HAL_VA_START);
 342
 343     /* Map PPEs and PDEs for non paged pool (including expansion) */
 344     MiMapPPEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
 345     MiMapPDEs(MmNonPagedPoolStart, MmNonPagedPoolEnd);
 346
 347     /* Map the nonpaged pool PTEs (without expansion) */
 348     MiMapPTEs(MmNonPagedPoolStart, (PCHAR)MmNonPagedPoolExpansionStart - 1);
 349
 350     /* Initialize the ARM3 nonpaged pool */
 351     MiInitializeNonPagedPool();
 352     MiInitializeNonPagedPoolThresholds();
 353
 354     /* Initialize the nonpaged pool */
 355     InitializePool(NonPagedPool, 0);
 356 }
 357
 358 VOID
 359 NTAPI
 360 INIT_FUNCTION
 361 MiBuildSystemPteSpace()
 362 {
 363     PMMPTE PointerPte;
 364
 365     /* Use the default numer of system PTEs */
 366     MmNumberOfSystemPtes = MI_NUMBER_SYSTEM_PTES;
 367     MiNonPagedSystemSize = (MmNumberOfSystemPtes + 1) * PAGE_SIZE;
 368
 369     /* Put system PTEs at the start of the system VA space */
 370     MiSystemPteSpaceStart = MmNonPagedSystemStart;
 371     MiSystemPteSpaceEnd = (PUCHAR)MiSystemPteSpaceStart + MiNonPagedSystemSize;
 372
 373     /* Map the PPEs and PDEs for the system PTEs */
 374     MiMapPPEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
 375     MiMapPDEs(MiSystemPteSpaceStart, MiSystemPteSpaceEnd);
 376
 377     /* Initialize the system PTE space */
 378     PointerPte = MiAddressToPte(MiSystemPteSpaceStart);
 379     MiInitializeSystemPtes(PointerPte, MmNumberOfSystemPtes, SystemPteSpace);
 380
 381     /* Reserve system PTEs for zeroing PTEs and clear them */
 382     MiFirstReservedZeroingPte = MiReserveSystemPtes(MI_ZERO_PTES, SystemPteSpace);
 383     RtlZeroMemory(MiFirstReservedZeroingPte, MI_ZERO_PTES * sizeof(MMPTE));
 384
 385     /* Set the counter to maximum */
 386     MiFirstReservedZeroingPte->u.Hard.PageFrameNumber = MI_ZERO_PTES - 1;
 387 }
 388
 389 static
 390 VOID
 391 MiSetupPfnForPageTable(
 392     PFN_NUMBER PageFrameIndex,
 393     PMMPTE PointerPte)
 394 {
 395     PMMPFN Pfn;
 396     PMMPDE PointerPde;
 397
 398     /* Get the pfn entry for this page */
 399     Pfn = MiGetPfnEntry(PageFrameIndex);
 400
 401     /* Check if it's valid memory */
 402     if ((PageFrameIndex <= MmHighestPhysicalPage) &&
 403         (MmIsAddressValid(Pfn)) &&
 404         (Pfn->u3.e1.PageLocation == ActiveAndValid))
 405     {
 406         /* Setup the PFN entry */
 407         Pfn->u1.WsIndex = 0;
 408         Pfn->u2.ShareCount++;
 409         Pfn->PteAddress = PointerPte;
 410         Pfn->OriginalPte = *PointerPte;
 411         Pfn->u3.e1.PageLocation = ActiveAndValid;
 412         Pfn->u3.e1.CacheAttribute = MiNonCached;
 413         Pfn->u3.e2.ReferenceCount = 1;
 414         Pfn->u4.PteFrame = PFN_FROM_PTE(MiAddressToPte(PointerPte));
 415     }
 416
 417     /* Increase the shared count of the PFN entry for the PDE */
 418     PointerPde = MiAddressToPde(MiPteToAddress(PointerPte));
 419     Pfn = MiGetPfnEntry(PFN_FROM_PTE(PointerPde));
 420     Pfn->u2.ShareCount++;
 421 }
 422
 423 VOID
 424 NTAPI
 425 MiBuildPfnDatabaseFromPageTables(VOID)
 426 {
 427     PVOID Address = NULL;
 428     PFN_NUMBER PageFrameIndex;
 429     PMMPDE PointerPde;
 430     PMMPTE PointerPte;
 431     ULONG k, l;
 432     PMMPFN Pfn;
 433 #if (_MI_PAGING_LEVELS >= 3)
 434     PMMPDE PointerPpe;
 435     ULONG j;
 436 #endif
 437 #if (_MI_PAGING_LEVELS == 4)
 438     PMMPDE PointerPxe;
 439     ULONG i;
 440 #endif
 441
 442     /* Manual setup of the top level page directory */
 443 #if (_MI_PAGING_LEVELS == 4)
 444     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PXE_BASE));
 445 #elif (_MI_PAGING_LEVELS == 3)
 446     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PPE_BASE));
 447 #else
 448     PageFrameIndex = PFN_FROM_PTE(MiAddressToPte(PDE_BASE));
 449 #endif
 450     Pfn = MiGetPfnEntry(PageFrameIndex);
 451     ASSERT(Pfn->u3.e1.PageLocation == ActiveAndValid);
 452     Pfn->u1.WsIndex = 0;
 453     Pfn->u2.ShareCount = 1;
 454     Pfn->PteAddress = NULL;
 455     Pfn->u3.e1.CacheAttribute = MiNonCached;
 456     Pfn->u3.e2.ReferenceCount = 1;
 457     Pfn->u4.PteFrame = 0;
 458
 459 #if (_MI_PAGING_LEVELS == 4)
 460     /* Loop all PXEs in the PML4 */
 461     PointerPxe = MiAddressToPxe(Address);
 462     for (i = 0; i < PXE_PER_PAGE; i++, PointerPxe++)
 463     {
 464         /* Skip invalid PXEs */
 465         if (!PointerPxe->u.Hard.Valid) continue;
 466
 467         /* Handle the PFN */
 468         PageFrameIndex = PFN_FROM_PXE(PointerPxe);
 469         MiSetupPfnForPageTable(PageFrameIndex, PointerPxe);
 470
 471         /* Get starting VA for this PXE */
 472         Address = MiPxeToAddress(PointerPxe);
 473 #endif
 474 #if (_MI_PAGING_LEVELS >= 3)
 475         /* Loop all PPEs in this PDP */
 476         PointerPpe = MiAddressToPpe(Address);
 477         for (j = 0; j < PPE_PER_PAGE; j++, PointerPpe++)
 478         {
 479             /* Skip invalid PPEs */
 480             if (!PointerPpe->u.Hard.Valid) continue;
 481
 482             /* Handle the PFN */
 483             PageFrameIndex = PFN_FROM_PPE(PointerPpe);
 484             MiSetupPfnForPageTable(PageFrameIndex, PointerPpe);
 485
 486             /* Get starting VA for this PPE */
 487             Address = MiPpeToAddress(PointerPpe);
 488 #endif
 489             /* Loop all PDEs in this PD */
 490             PointerPde = MiAddressToPde(Address);
 491             for (k = 0; k < PDE_PER_PAGE; k++, PointerPde++)
 492             {
 493                 /* Skip invalid PDEs */
 494                 if (!PointerPde->u.Hard.Valid) continue;
 495
 496                 /* Handle the PFN */
 497                 PageFrameIndex = PFN_FROM_PDE(PointerPde);
 498                 MiSetupPfnForPageTable(PageFrameIndex, PointerPde);
 499
 500                 /* Get starting VA for this PDE */
 501                 Address = MiPdeToAddress(PointerPde);
 502
 503                 /* Loop all PTEs in this PT */
 504                 PointerPte = MiAddressToPte(Address);
 505                 for (l = 0; l < PTE_PER_PAGE; l++, PointerPte++)
 506                 {
 507                     /* Skip invalid PTEs */
 508                     if (!PointerPte->u.Hard.Valid) continue;
 509
 510                     /* Handle the PFN */
 511                     PageFrameIndex = PFN_FROM_PTE(PointerPte);
 512                     MiSetupPfnForPageTable(PageFrameIndex, PointerPte);
 513                 }
 514             }
 515 #if (_MI_PAGING_LEVELS >= 3)
 516         }
 517 #endif
 518 #if (_MI_PAGING_LEVELS == 4)
 519     }
 520 #endif
 521 }
 522
 523 VOID
 524 NTAPI
 525 INIT_FUNCTION
 526 MiAddDescriptorToDatabase(
 527     PFN_NUMBER BasePage,
 528     PFN_NUMBER PageCount,
 529     TYPE_OF_MEMORY MemoryType)
 530 {
 531     PMMPFN Pfn;
 532
 533     ASSERT(!MiIsMemoryTypeInvisible(MemoryType));
 534
 535     /* Check if the memory is free */
 536     if (MiIsMemoryTypeFree(MemoryType))
 537     {
 538         /* Get the last pfn of this descriptor. Note we loop backwards */
 539         Pfn = &MmPfnDatabase[BasePage + PageCount - 1];
 540
 541         /* Loop all pages */
 542         while (PageCount--)
 543         {
 544             /* Add it to the free list */
 545             Pfn->u3.e1.CacheAttribute = MiNonCached;
 546             MiInsertPageInFreeList(BasePage + PageCount);
 547
 548             /* Go to the previous page */
 549             Pfn--;
 550         }
 551     }
 552     else if (MemoryType == LoaderXIPRom)
 553     {
 554         Pfn = &MmPfnDatabase[BasePage];
 555         while (PageCount--)
 556         {
 557             /* Make it a pseudo-I/O ROM mapping */
 558             Pfn->PteAddress = 0;
 559             Pfn->u1.Flink = 0;
 560             Pfn->u2.ShareCount = 0;
 561             Pfn->u3.e1.PageLocation = 0;
 562             Pfn->u3.e1.CacheAttribute = MiNonCached;
 563             Pfn->u3.e1.Rom = 1;
 564             Pfn->u3.e1.PrototypePte = 1;
 565             Pfn->u3.e2.ReferenceCount = 0;
 566             Pfn->u4.InPageError = 0;
 567             Pfn->u4.PteFrame = 0;
 568
 569             /* Advance one */
 570             Pfn++;
 571         }
 572     }
 573     else if (MemoryType == LoaderBad)
 574     {
 575         // FIXME: later
 576         ASSERT(FALSE);
 577     }
 578     else
 579     {
 580         /* For now skip it */
 581         DbgPrint("Skipping BasePage=0x%lx, PageCount=0x%lx, MemoryType=%lx\n",
 582                  BasePage, PageCount, MemoryType);
 583         Pfn = &MmPfnDatabase[BasePage];
 584         while (PageCount--)
 585         {
 586             /* Make an active PFN */
 587             Pfn->u3.e1.PageLocation = ActiveAndValid;
 588
 589             /* Advance one */
 590             Pfn++;
 591         }
 592     }
 593 }
 594
 595 VOID
 596 NTAPI
 597 INIT_FUNCTION
 598 MiBuildPfnDatabase(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 599 {
 600     PLIST_ENTRY ListEntry;
 601     PMEMORY_ALLOCATION_DESCRIPTOR Descriptor;
 602     PFN_NUMBER BasePage, PageCount;
 603
 604     /* Map the PDEs and PPEs for the pfn database (ignore holes) */
 605 #if (_MI_PAGING_LEVELS >= 3)
 606     MiMapPPEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + MxPfnAllocation - 1);
 607 #endif
 608     MiMapPDEs(MmPfnDatabase, (PUCHAR)MmPfnDatabase + MxPfnAllocation - 1);
 609
 610     /* First initialize the color tables */
 611     MiInitializeColorTables();
 612
 613     /* Loop the memory descriptors */
 614     for (ListEntry = LoaderBlock->MemoryDescriptorListHead.Flink;
 615          ListEntry != &LoaderBlock->MemoryDescriptorListHead;
 616          ListEntry = ListEntry->Flink)
 617     {
 618         /* Get the descriptor */
 619         Descriptor = CONTAINING_RECORD(ListEntry,
 620                                        MEMORY_ALLOCATION_DESCRIPTOR,
 621                                        ListEntry);
 622
 623         /* Skip invisible memory */
 624         if (MiIsMemoryTypeInvisible(Descriptor->MemoryType)) continue;
 625
 626         /* If this is the free descriptor, use the copy instead */
 627         if (Descriptor == MxFreeDescriptor) Descriptor = &MxOldFreeDescriptor;
 628
 629         /* Get the range for this descriptor */
 630         BasePage = Descriptor->BasePage;
 631         PageCount = Descriptor->PageCount;
 632
 633         /* Map the pages for the database */
 634         MiMapPTEs(&MmPfnDatabase[BasePage],
 635                   (PUCHAR)(&MmPfnDatabase[BasePage + PageCount]) - 1);
 636
 637         /* If this was the free descriptor, skip the next step */
 638         if (Descriptor == &MxOldFreeDescriptor) continue;
 639
 640         /* Add this descriptor to the database */
 641         MiAddDescriptorToDatabase(BasePage, PageCount, Descriptor->MemoryType);
 642     }
 643
 644     /* At this point the whole pfn database is mapped. We are about to add the
 645        pages from the free descriptor to the database, so from now on we cannot
 646        use it anymore. */
 647
 648     /* Now add the free descriptor */
 649     BasePage = MxFreeDescriptor->BasePage;
 650     PageCount = MxFreeDescriptor->PageCount;
 651     MiAddDescriptorToDatabase(BasePage, PageCount, LoaderFree);
 652
 653     /* And finally the memory we used */
 654     BasePage = MxOldFreeDescriptor.BasePage;
 655     PageCount = MxFreeDescriptor->BasePage - BasePage;
 656     MiAddDescriptorToDatabase(BasePage, PageCount, LoaderMemoryData);
 657
 658     // Reset the descriptor back so we can create the correct memory blocks
 659     *MxFreeDescriptor = MxOldFreeDescriptor;
 660 }
 661
 662 NTSTATUS
 663 NTAPI
 664 INIT_FUNCTION
 665 MiInitMachineDependent(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
 666 {
 667     KIRQL OldIrql;
 668
 669     ASSERT(MxPfnAllocation != 0);
 670
 671     /* Set some hardcoded addresses */
 672     MmHyperSpaceEnd = (PVOID)HYPER_SPACE_END;
 673     MmNonPagedSystemStart = (PVOID)MM_SYSTEM_SPACE_START;
 674     MmPfnDatabase = (PVOID)MI_PFN_DATABASE;
 675     MmWorkingSetList = (PVOID)MI_WORKING_SET_LIST;
 676
 677
 678 //    PrototypePte.u.Proto.Valid = 1
 679 //    PrototypePte.u.ReadOnly
 680 //    PrototypePte.u.Prototype
 681 //    PrototypePte.u.Protection = MM_READWRITE;
 682 //    PrototypePte.u.ProtoAddress
 683     PrototypePte.u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
 684
 685
 686     MiInitializePageTable();
 687
 688     MiBuildNonPagedPool();
 689
 690     MiBuildSystemPteSpace();
 691
 692     /* Need to be at DISPATCH_LEVEL for MiInsertPageInFreeList */
 693     KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
 694
 695     /* Map the PFN database pages */
 696     MiBuildPfnDatabase(LoaderBlock);
 697
 698     /* Now process the page tables */
 699     MiBuildPfnDatabaseFromPageTables();
 700
 701     MiPfnsInitialized = TRUE;
 702
 703     KeLowerIrql(OldIrql);
 704
 705     /* Initialize the balancer */
 706     MmInitializeBalancer((ULONG)MmAvailablePages, 0);
 707
 708     /* Make sure we have everything we need */
 709     ASSERT(MmPfnDatabase);
 710     ASSERT(MmNonPagedSystemStart);
 711     ASSERT(MmNonPagedPoolStart);
 712     ASSERT(MmSizeOfNonPagedPoolInBytes);
 713     ASSERT(MmMaximumNonPagedPoolInBytes);
 714     ASSERT(MmNonPagedPoolExpansionStart);
 715     ASSERT(MmHyperSpaceEnd);
 716     ASSERT(MmNumberOfSystemPtes);
 717     ASSERT(MiAddressToPde(MmNonPagedPoolStart)->u.Hard.Valid);
 718     ASSERT(MiAddressToPte(MmNonPagedPoolStart)->u.Hard.Valid);
 719
 720     return STATUS_SUCCESS;
 721 }