boot/environ/lib/mm/i386/mmx86.c

   1 /*
   2 * COPYRIGHT:       See COPYING.ARM in the top level directory
   3 * PROJECT:         ReactOS UEFI Boot Library
   4 * FILE:            boot/environ/lib/mm/i386/mmx86.c
   5 * PURPOSE:         Boot Library Memory Manager x86-Specific Code
   6 * PROGRAMMER:      Alex Ionescu (alex.ionescu@reactos.org)
   7 */
   8
   9 /* INCLUDES ******************************************************************/
  10
  11 #include "bl.h"
  12 #include "bcd.h"
  13
  14 #define PTE_BASE                0xC0000000
  15
  16 //
  17 // Specific PDE/PTE macros to be used inside the boot library environment
  18 //
  19 #define MiAddressToPte(x)       ((PMMPTE)(((((ULONG)(x)) >> 12) << 2) + (ULONG_PTR)MmPteBase))
  20 #define MiAddressToPde(x)       ((PMMPDE)(((((ULONG)(x)) >> 22) << 2) + (ULONG_PTR)MmPdeBase))
  21 #define MiAddressToPteOffset(x) ((((ULONG)(x)) << 10) >> 22)
  22 #define MiAddressToPdeOffset(x) (((ULONG)(x)) / (1024 * PAGE_SIZE))
  23
  24 /* DATA VARIABLES ************************************************************/
  25
  26 ULONG_PTR MmArchKsegBase;
  27 ULONG_PTR MmArchKsegBias;
  28 ULONG MmArchLargePageSize;
  29 BL_ADDRESS_RANGE MmArchKsegAddressRange;
  30 ULONG_PTR MmArchTopOfApplicationAddressSpace;
  31 PHYSICAL_ADDRESS Mmx86SelfMapBase;
  32 ULONG MmDeferredMappingCount;
  33 PMMPTE MmPdpt;
  34 PULONG MmArchReferencePage;
  35 PVOID MmPteBase;
  36 PVOID MmPdeBase;
  37 ULONG MmArchReferencePageSize;
  38
  39 typedef VOID
  40 (*PBL_MM_FLUSH_TLB) (
  41     VOID
  42     );
  43
  44 typedef VOID
  45 (*PBL_MM_RELOCATE_SELF_MAP) (
  46     VOID
  47     );
  48
  49 typedef NTSTATUS
  50 (*PBL_MM_MOVE_VIRTUAL_ADDRESS_RANGE) (
  51     _In_ PVOID DestinationAddress,
  52     _In_ PVOID SourceAddress,
  53     _In_ ULONGLONG Size
  54     );
  55
  56 typedef NTSTATUS
  57 (*PBL_MM_ZERO_VIRTUAL_ADDRESS_RANGE) (
  58     _In_ PVOID DestinationAddress,
  59     _In_ ULONGLONG Size
  60     );
  61
  62 typedef VOID
  63 (*PBL_MM_DESTROY_SELF_MAP) (
  64     VOID
  65     );
  66
  67 typedef VOID
  68 (*PBL_MM_FLUSH_TLB_ENTRY) (
  69     _In_ PVOID VirtualAddress
  70     );
  71
  72 typedef VOID
  73 (*PBL_MM_FLUSH_TLB) (
  74     VOID
  75     );
  76
  77 typedef NTSTATUS
  78 (*PBL_MM_UNMAP_VIRTUAL_ADDRESS) (
  79     _In_ PVOID VirtualAddress,
  80     _In_ ULONG Size
  81     );
  82
  83 typedef NTSTATUS
  84 (*PBL_MM_REMAP_VIRTUAL_ADDRESS) (
  85     _In_ PPHYSICAL_ADDRESS PhysicalAddress,
  86     _Out_ PVOID VirtualAddress,
  87     _In_ ULONG Size,
  88     _In_ ULONG CacheAttributes
  89     );
  90
  91 typedef NTSTATUS
  92 (*PBL_MM_MAP_PHYSICAL_ADDRESS) (
  93     _In_ PHYSICAL_ADDRESS PhysicalAddress,
  94     _Out_ PVOID VirtualAddress,
  95     _In_ ULONG Size,
  96     _In_ ULONG CacheAttributes
  97     );
  98
  99 typedef BOOLEAN
 100 (*PBL_MM_TRANSLATE_VIRTUAL_ADDRESS) (
 101     _In_ PVOID VirtualAddress,
 102     _Out_ PPHYSICAL_ADDRESS PhysicalAddress,
 103     _Out_opt_ PULONG CacheAttributes
 104     );
 105
 106 PBL_MM_TRANSLATE_VIRTUAL_ADDRESS Mmx86TranslateVirtualAddress;
 107 PBL_MM_MAP_PHYSICAL_ADDRESS Mmx86MapPhysicalAddress;
 108 PBL_MM_REMAP_VIRTUAL_ADDRESS Mmx86RemapVirtualAddress;
 109 PBL_MM_UNMAP_VIRTUAL_ADDRESS Mmx86UnmapVirtualAddress;
 110 PBL_MM_FLUSH_TLB Mmx86FlushTlb;
 111 PBL_MM_FLUSH_TLB_ENTRY Mmx86FlushTlbEntry;
 112 PBL_MM_DESTROY_SELF_MAP Mmx86DestroySelfMap;
 113
 114 PBL_MM_RELOCATE_SELF_MAP BlMmRelocateSelfMap;
 115 PBL_MM_FLUSH_TLB BlMmFlushTlb;
 116 PBL_MM_MOVE_VIRTUAL_ADDRESS_RANGE BlMmMoveVirtualAddressRange;
 117 PBL_MM_ZERO_VIRTUAL_ADDRESS_RANGE BlMmZeroVirtualAddressRange;
 118
 119 PBL_MM_FLUSH_TLB Mmx86FlushTlb;
 120
 121 /* FUNCTIONS *****************************************************************/
 122
 123 BOOLEAN
 124 BlMmIsTranslationEnabled (
 125     VOID
 126     )
 127 {
 128     /* Return if paging is on */
 129     return ((CurrentExecutionContext) &&
 130             (CurrentExecutionContext->ContextFlags & BL_CONTEXT_PAGING_ON));
 131 }
 132
 133 VOID
 134 MmArchNullFunction (
 135     VOID
 136     )
 137 {
 138     /* Nothing to do */
 139     return;
 140 }
 141
 142 VOID
 143 MmDefRelocateSelfMap (
 144     VOID
 145     )
 146 {
 147     if (MmPteBase != (PVOID)PTE_BASE)
 148     {
 149         EfiPrintf(L"Supposed to relocate CR3\r\n");
 150     }
 151 }
 152
 153 NTSTATUS
 154 MmDefMoveVirtualAddressRange (
 155     _In_ PVOID DestinationAddress,
 156     _In_ PVOID SourceAddress,
 157     _In_ ULONGLONG Size
 158     )
 159 {
 160     EfiPrintf(L"Supposed to move shit\r\n");
 161     return STATUS_NOT_IMPLEMENTED;
 162 }
 163
 164 NTSTATUS
 165 MmDefZeroVirtualAddressRange (
 166     _In_ PVOID DestinationAddress,
 167     _In_ ULONGLONG Size
 168     )
 169 {
 170     EfiPrintf(L"Supposed to zero shit\r\n");
 171     return STATUS_NOT_IMPLEMENTED;
 172 }
 173
 174 BOOLEAN
 175 MmArchTranslateVirtualAddress (
 176     _In_ PVOID VirtualAddress,
 177     _Out_opt_ PPHYSICAL_ADDRESS PhysicalAddress,
 178     _Out_opt_ PULONG CachingFlags
 179     )
 180 {
 181     PBL_MEMORY_DESCRIPTOR Descriptor;
 182
 183     /* Check if paging is on */
 184     if ((CurrentExecutionContext) &&
 185         (CurrentExecutionContext->ContextFlags & BL_CONTEXT_PAGING_ON))
 186     {
 187         /* Yes -- we have to translate this from virtual */
 188         return Mmx86TranslateVirtualAddress(VirtualAddress,
 189                                             PhysicalAddress,
 190                                             CachingFlags);
 191     }
 192
 193     /* Look in all descriptors except truncated and firmware ones */
 194     Descriptor = MmMdFindDescriptor(BL_MM_INCLUDE_NO_FIRMWARE_MEMORY &
 195                                     ~BL_MM_INCLUDE_TRUNCATED_MEMORY,
 196                                     BL_MM_REMOVE_PHYSICAL_REGION_FLAG,
 197                                     (ULONG_PTR)VirtualAddress >> PAGE_SHIFT);
 198
 199     /* Return the virtual address as the physical address */
 200     if (PhysicalAddress)
 201     {
 202         PhysicalAddress->HighPart = 0;
 203         PhysicalAddress->LowPart = (ULONG_PTR)VirtualAddress;
 204     }
 205
 206     /* There's no caching on physical memory */
 207     if (CachingFlags)
 208     {
 209         *CachingFlags = 0;
 210     }
 211
 212     /* Success is if we found a descriptor */
 213     return Descriptor != NULL;
 214 }
 215
 216 VOID
 217 MmDefpDestroySelfMap (
 218     VOID
 219     )
 220 {
 221     EfiPrintf(L"No destroy\r\n");
 222 }
 223
 224 VOID
 225 MmDefpFlushTlbEntry (
 226     _In_ PVOID VirtualAddress
 227     )
 228 {
 229     /* Flush the TLB */
 230     __invlpg(VirtualAddress);
 231 }
 232
 233 VOID
 234 MmDefpFlushTlb (
 235     VOID
 236     )
 237 {
 238     /* Flush the TLB */
 239     __writecr3(__readcr3());
 240 }
 241
 242 NTSTATUS
 243 MmDefpUnmapVirtualAddress (
 244     _In_ PVOID VirtualAddress,
 245     _In_ ULONG Size
 246     )
 247 {
 248     EfiPrintf(L"No unmap\r\n");
 249     return STATUS_NOT_IMPLEMENTED;
 250 }
 251
 252 NTSTATUS
 253 MmDefpRemapVirtualAddress (
 254     _In_ PPHYSICAL_ADDRESS PhysicalAddress,
 255     _Out_ PVOID VirtualAddress,
 256     _In_ ULONG Size,
 257     _In_ ULONG CacheAttributes
 258     )
 259 {
 260     EfiPrintf(L"No remap\r\n");
 261     return STATUS_NOT_IMPLEMENTED;
 262 }
 263
 264 NTSTATUS
 265 MmDefpMapPhysicalAddress (
 266     _In_ PHYSICAL_ADDRESS PhysicalAddress,
 267     _In_ PVOID VirtualAddress,
 268     _In_ ULONG Size,
 269     _In_ ULONG CacheAttributes
 270     )
 271 {
 272     BOOLEAN Enabled;
 273     ULONG i, PageCount, PdeOffset;
 274     ULONGLONG CurrentAddress;
 275     PMMPDE Pde;
 276     PMMPTE Pte;
 277     PMMPTE PageTable;
 278     PHYSICAL_ADDRESS PageTableAddress;
 279     NTSTATUS Status;
 280
 281     /* Check if paging is on yet */
 282     Enabled = BlMmIsTranslationEnabled();
 283
 284     /* Get the physical address aligned */
 285     CurrentAddress = (PhysicalAddress.QuadPart >> PAGE_SHIFT) << PAGE_SHIFT;
 286
 287     /* Get the number of pages and loop through each one */
 288     PageCount = Size >> PAGE_SHIFT;
 289     for (i = 0; i < PageCount; i++)
 290     {
 291         /* Check if translation already exists for this page */
 292         if (Mmx86TranslateVirtualAddress(VirtualAddress, NULL, NULL))
 293         {
 294             /* Ignore it and move to the next one */
 295             VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress + PAGE_SIZE);
 296             CurrentAddress += PAGE_SIZE;
 297             continue;
 298         }
 299
 300         /* Get the PDE offset */
 301         PdeOffset = MiAddressToPdeOffset(VirtualAddress);
 302
 303         /* Check if paging is actually turned on */
 304         if (Enabled)
 305         {
 306             /* Get the PDE entry using the self-map */
 307             Pde = MiAddressToPde(VirtualAddress);
 308         }
 309         else
 310         {
 311             /* Get it using our physical mappings */
 312             Pde = &MmPdpt[PdeOffset];
 313             PageTable = (PMMPDE)(Pde->u.Hard.PageFrameNumber << PAGE_SHIFT);
 314         }
 315
 316         /* Check if we don't yet have a PDE */
 317         if (!Pde->u.Hard.Valid)
 318         {
 319             /* Allocate a page table */
 320             Status = MmPapAllocatePhysicalPagesInRange(&PageTableAddress,
 321                                                        BlLoaderPageDirectory,
 322                                                        1,
 323                                                        0,
 324                                                        0,
 325                                                        &MmMdlUnmappedAllocated,
 326                                                        0,
 327                                                        0);
 328             if (!NT_SUCCESS(Status))
 329             {
 330                 EfiPrintf(L"PDE alloc failed!\r\n");
 331                 EfiStall(1000000);
 332                 return STATUS_NO_MEMORY;
 333             }
 334
 335             /* This is our page table */
 336             PageTable = (PVOID)(ULONG_PTR)PageTableAddress.QuadPart;
 337
 338             /* Build the PDE for it */
 339             Pde->u.Hard.PageFrameNumber = PageTableAddress.QuadPart >> PAGE_SHIFT;
 340             Pde->u.Hard.Write = 1;
 341             Pde->u.Hard.CacheDisable = 1;
 342             Pde->u.Hard.WriteThrough = 1;
 343             Pde->u.Hard.Valid = 1;
 344
 345             /* Check if paging is enabled */
 346             if (Enabled)
 347             {
 348                 /* Then actually, get the page table's virtual address */
 349                 PageTable = (PVOID)PAGE_ROUND_DOWN(MiAddressToPte(VirtualAddress));
 350
 351                 /* Flush the TLB */
 352                 Mmx86FlushTlb();
 353             }
 354
 355             /* Zero out the page table */
 356             RtlZeroMemory(PageTable, PAGE_SIZE);
 357
 358             /* Reset caching attributes now */
 359             Pde->u.Hard.CacheDisable = 0;
 360             Pde->u.Hard.WriteThrough = 0;
 361
 362             /* Check for paging again */
 363             if (Enabled)
 364             {
 365                 /* Flush the TLB entry for the page table only */
 366                 Mmx86FlushTlbEntry(PageTable);
 367             }
 368         }
 369
 370         /* Add a reference to this page table */
 371         MmArchReferencePage[PdeOffset]++;
 372
 373         /* Check if a physical address was given */
 374         if (PhysicalAddress.QuadPart != -1)
 375         {
 376             /* Check if paging is turned on */
 377             if (Enabled)
 378             {
 379                 /* Get the PTE using the self-map */
 380                 Pte = MiAddressToPte(VirtualAddress);
 381             }
 382             else
 383             {
 384                 /* Get the PTE using physical addressing */
 385                 Pte = &PageTable[MiAddressToPteOffset(VirtualAddress)];
 386             }
 387
 388             /* Build a valid PTE for it */
 389             Pte->u.Hard.PageFrameNumber = CurrentAddress >> PAGE_SHIFT;
 390             Pte->u.Hard.Write = 1;
 391             Pte->u.Hard.Valid = 1;
 392
 393             /* Check if this is uncached */
 394             if (CacheAttributes == BlMemoryUncached)
 395             {
 396                 /* Set the flags */
 397                 Pte->u.Hard.CacheDisable = 1;
 398                 Pte->u.Hard.WriteThrough = 1;
 399             }
 400             else if (CacheAttributes == BlMemoryWriteThrough)
 401             {
 402                 /* It's write-through, set the flag */
 403                 Pte->u.Hard.WriteThrough = 1;
 404             }
 405         }
 406
 407         /* Move to the next physical/virtual address */
 408         VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress + PAGE_SIZE);
 409         CurrentAddress += PAGE_SIZE;
 410     }
 411
 412     /* All done! */
 413     return STATUS_SUCCESS;
 414 }
 415
 416 BOOLEAN
 417 MmDefpTranslateVirtualAddress (
 418     _In_ PVOID VirtualAddress,
 419     _Out_ PPHYSICAL_ADDRESS PhysicalAddress,
 420     _Out_opt_ PULONG CacheAttributes
 421     )
 422 {
 423     PMMPDE Pde;
 424     PMMPTE Pte;
 425     PMMPTE PageTable;
 426     BOOLEAN Enabled;
 427
 428     /* Is there no page directory yet? */
 429     if (!MmPdpt)
 430     {
 431         return FALSE;
 432     }
 433
 434     /* Is paging enabled? */
 435     Enabled = BlMmIsTranslationEnabled();
 436
 437     /* Check if paging is actually turned on */
 438     if (Enabled)
 439     {
 440         /* Get the PDE entry using the self-map */
 441         Pde = MiAddressToPde(VirtualAddress);
 442     }
 443     else
 444     {
 445         /* Get it using our physical mappings */
 446         Pde = &MmPdpt[MiAddressToPdeOffset(VirtualAddress)];
 447     }
 448
 449     /* Is the PDE valid? */
 450     if (!Pde->u.Hard.Valid)
 451     {
 452         return FALSE;
 453     }
 454
 455     /* Check if paging is turned on */
 456     if (Enabled)
 457     {
 458         /* Get the PTE using the self-map */
 459         Pte = MiAddressToPte(VirtualAddress);
 460     }
 461     else
 462     {
 463         /* Get the PTE using physical addressing */
 464         PageTable = (PMMPTE)(Pde->u.Hard.PageFrameNumber << PAGE_SHIFT);
 465         Pte = &PageTable[MiAddressToPteOffset(VirtualAddress)];
 466     }
 467
 468     /* Is the PTE valid? */
 469     if (!Pte->u.Hard.Valid)
 470     {
 471         return FALSE;
 472     }
 473
 474     /* Does caller want the physical address?  */
 475     if (PhysicalAddress)
 476     {
 477         /* Return it */
 478         PhysicalAddress->QuadPart = (Pte->u.Hard.PageFrameNumber << PAGE_SHIFT) +
 479                                      BYTE_OFFSET(VirtualAddress);
 480     }
 481
 482     /* Does caller want cache attributes? */
 483     if (CacheAttributes)
 484     {
 485         /* Not yet -- lie and say it's cached */
 486         EfiPrintf(L"Cache checking not yet enabled\r\n");
 487         *CacheAttributes = BlMemoryWriteBack;
 488     }
 489
 490     /* It exists! */
 491     return TRUE;
 492 }
 493
 494 NTSTATUS
 495 MmMapPhysicalAddress (
 496     _Inout_ PPHYSICAL_ADDRESS PhysicalAddressPtr,
 497     _Inout_ PVOID* VirtualAddressPtr,
 498     _Inout_ PULONGLONG SizePtr,
 499     _In_ ULONG CacheAttributes
 500     )
 501 {
 502     ULONGLONG Size;
 503     ULONGLONG PhysicalAddress;
 504     PVOID VirtualAddress;
 505     PHYSICAL_ADDRESS TranslatedAddress;
 506     ULONG_PTR CurrentAddress, VirtualAddressEnd;
 507     NTSTATUS Status;
 508
 509     /* Fail if any parameters are missing */
 510     if (!(PhysicalAddressPtr) || !(VirtualAddressPtr) || !(SizePtr))
 511     {
 512         return STATUS_INVALID_PARAMETER;
 513     }
 514
 515     /* Fail if the size is over 32-bits */
 516     Size = *SizePtr;
 517     if (Size > 0xFFFFFFFF)
 518     {
 519         return STATUS_INVALID_PARAMETER;
 520     }
 521
 522     /* Nothing to do if we're in physical mode */
 523     if (MmTranslationType == BlNone)
 524     {
 525         return STATUS_SUCCESS;
 526     }
 527
 528     /* Can't use virtual memory in real mode */
 529     if (CurrentExecutionContext->Mode == BlRealMode)
 530     {
 531         return STATUS_UNSUCCESSFUL;
 532     }
 533
 534     /* Capture the current virtual and physical addresses */
 535     VirtualAddress = *VirtualAddressPtr;
 536     PhysicalAddress = PhysicalAddressPtr->QuadPart;
 537
 538     /* Check if a physical address was requested */
 539     if (PhysicalAddress != 0xFFFFFFFF)
 540     {
 541         /* Round down the base addresses */
 542         PhysicalAddress = PAGE_ROUND_DOWN(PhysicalAddress);
 543         VirtualAddress = (PVOID)PAGE_ROUND_DOWN(VirtualAddress);
 544
 545         /* Round up the size */
 546         Size = ROUND_TO_PAGES(PhysicalAddressPtr->QuadPart -
 547                               PhysicalAddress +
 548                               Size);
 549
 550         /* Loop every virtual page */
 551         CurrentAddress = (ULONG_PTR)VirtualAddress;
 552         VirtualAddressEnd = CurrentAddress + Size - 1;
 553         while (CurrentAddress < VirtualAddressEnd)
 554         {
 555             /* Get the physical page of this virtual page */
 556             if (MmArchTranslateVirtualAddress((PVOID)CurrentAddress,
 557                                               &TranslatedAddress,
 558                                               &CacheAttributes))
 559             {
 560                 /* Make sure the physical page of the virtual page, matches our page */
 561                 if (TranslatedAddress.QuadPart !=
 562                     (PhysicalAddress +
 563                      (CurrentAddress - (ULONG_PTR)VirtualAddress)))
 564                 {
 565                     /* There is an existing virtual mapping for a different address */
 566                     EfiPrintf(L"Existing mapping exists: %lx vs %lx\r\n",
 567                               TranslatedAddress.QuadPart,
 568                               PhysicalAddress + (CurrentAddress - (ULONG_PTR)VirtualAddress));
 569                     EfiStall(10000000);
 570                     return STATUS_INVALID_PARAMETER;
 571                 }
 572             }
 573
 574             /* Try the next one */
 575             CurrentAddress += PAGE_SIZE;
 576         }
 577     }
 578
 579     /* Aactually do the mapping */
 580     TranslatedAddress.QuadPart = PhysicalAddress;
 581     Status = Mmx86MapPhysicalAddress(TranslatedAddress,
 582                                      VirtualAddress,
 583                                      Size,
 584                                      CacheAttributes);
 585     if (!NT_SUCCESS(Status))
 586     {
 587         EfiPrintf(L"Failed to map!: %lx\r\n", Status);
 588         EfiStall(1000000);
 589         return Status;
 590     }
 591
 592     /* Return aligned/fixed up output parameters */
 593     PhysicalAddressPtr->QuadPart = PhysicalAddress;
 594     *VirtualAddressPtr = VirtualAddress;
 595     *SizePtr = Size;
 596
 597     /* Flush the TLB if paging is enabled */
 598     if (BlMmIsTranslationEnabled())
 599     {
 600         Mmx86FlushTlb();
 601     }
 602
 603     /* All good! */
 604     return STATUS_SUCCESS;
 605 }
 606
 607 NTSTATUS
 608 Mmx86MapInitStructure (
 609     _In_ PVOID VirtualAddress,
 610     _In_ ULONGLONG Size,
 611     _In_ PHYSICAL_ADDRESS PhysicalAddress
 612     )
 613 {
 614     NTSTATUS Status;
 615
 616     /* Make a virtual mapping for this physical address */
 617     Status = MmMapPhysicalAddress(&PhysicalAddress, &VirtualAddress, &Size, 0);
 618     if (!NT_SUCCESS(Status))
 619     {
 620         return Status;
 621     }
 622
 623     /* Nothing else to do if we're not in paging mode */
 624     if (MmTranslationType == BlNone)
 625     {
 626         return STATUS_SUCCESS;
 627     }
 628
 629     /* Otherwise, remove this region from the list of free virtual ranges */
 630     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
 631                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
 632                                        (ULONG_PTR)VirtualAddress >> PAGE_SHIFT,
 633                                        Size >> PAGE_SHIFT,
 634                                        0);
 635     if (!NT_SUCCESS(Status))
 636     {
 637         /* Unmap the address if that failed */
 638         MmUnmapVirtualAddress(&VirtualAddress, &Size);
 639     }
 640
 641     /* Return back to caller */
 642     return Status;
 643 }
 644
 645 VOID
 646 MmMdDbgDumpList (
 647     _In_ PBL_MEMORY_DESCRIPTOR_LIST DescriptorList,
 648     _In_opt_ ULONG MaxCount
 649     )
 650 {
 651     ULONGLONG EndPage, VirtualEndPage;
 652     PBL_MEMORY_DESCRIPTOR MemoryDescriptor;
 653     PLIST_ENTRY NextEntry;
 654
 655     /* If no maximum was provided, use essentially infinite */
 656     if (MaxCount == 0)
 657     {
 658         MaxCount = 0xFFFFFFFF;
 659     }
 660
 661     /* Loop the list as long as there's entries and max isn't reached */
 662     NextEntry = DescriptorList->First->Flink;
 663     while ((NextEntry != DescriptorList->First) && (MaxCount--))
 664     {
 665         /* Get the descriptor */
 666         MemoryDescriptor = CONTAINING_RECORD(NextEntry,
 667                                              BL_MEMORY_DESCRIPTOR,
 668                                              ListEntry);
 669
 670         /* Get the descriptor end page, and see if it was virtually mapepd */
 671         EndPage = MemoryDescriptor->BasePage + MemoryDescriptor->PageCount;
 672         if (MemoryDescriptor->VirtualPage)
 673         {
 674             /* Get the virtual end page too, then */
 675             VirtualEndPage = MemoryDescriptor->VirtualPage +
 676                              MemoryDescriptor->PageCount;
 677         }
 678         else
 679         {
 680             VirtualEndPage = 0;
 681         }
 682
 683         /* Print out the descriptor, physical range, virtual range, and type */
 684         EfiPrintf(L"%p - [%08llx-%08llx @ %08llx-%08llx]:%x\r\n",
 685                     MemoryDescriptor,
 686                     MemoryDescriptor->BasePage << PAGE_SHIFT,
 687                     (EndPage << PAGE_SHIFT) - 1,
 688                     MemoryDescriptor->VirtualPage << PAGE_SHIFT,
 689                     VirtualEndPage ? (VirtualEndPage << PAGE_SHIFT) - 1 : 0,
 690                     (ULONG)MemoryDescriptor->Type);
 691
 692         /* Next entry */
 693         NextEntry = NextEntry->Flink;
 694     }
 695 }
 696
 697 NTSTATUS
 698 Mmx86pMapMemoryRegions (
 699     _In_ ULONG Phase,
 700     _In_ PBL_MEMORY_DATA MemoryData
 701     )
 702 {
 703     BOOLEAN DoDeferred;
 704     ULONG DescriptorCount;
 705     PBL_MEMORY_DESCRIPTOR Descriptor;
 706     ULONG FinalOffset;
 707     PHYSICAL_ADDRESS PhysicalAddress;
 708     ULONGLONG Size;
 709     NTSTATUS Status;
 710     PVOID VirtualAddress;
 711     BL_MEMORY_DESCRIPTOR_LIST FirmwareMdl;
 712     PLIST_ENTRY Head, NextEntry;
 713
 714     /* Check which phase this is */
 715     if (Phase == 1)
 716     {
 717         /* In phase 1 we don't initialize deferred mappings */
 718         DoDeferred = FALSE;
 719     }
 720     else
 721     {
 722         /* Don't do anything if there's nothing to initialize */
 723         if (!MmDeferredMappingCount)
 724         {
 725             return STATUS_SUCCESS;
 726         }
 727
 728         /* We'll do deferred descriptors in phase 2 */
 729         DoDeferred = TRUE;
 730     }
 731
 732     /*
 733     * Because BL supports cross x86-x64 application launches and a LIST_ENTRY
 734     * is of variable size, care must be taken here to ensure that we see a
 735     * consistent view of descriptors. BL uses some offset magic to figure out
 736     * where the data actually starts, since everything is ULONGLONG past the
 737     * LIST_ENTRY itself
 738     */
 739     FinalOffset = MemoryData->MdListOffset + MemoryData->DescriptorOffset;
 740     Descriptor = (PBL_MEMORY_DESCRIPTOR)((ULONG_PTR)MemoryData + FinalOffset -
 741                                          FIELD_OFFSET(BL_MEMORY_DESCRIPTOR, BasePage));
 742
 743     /* Scan all of them */
 744     DescriptorCount = MemoryData->DescriptorCount;
 745     while (DescriptorCount != 0)
 746     {
 747         /* Ignore application data */
 748         if (Descriptor->Type != BlApplicationData)
 749         {
 750             /* If this is a ramdisk, do it in phase 2 */
 751             if ((Descriptor->Type == BlLoaderRamDisk) == DoDeferred)
 752             {
 753                 /* Get the current physical address and size */
 754                 PhysicalAddress.QuadPart = Descriptor->BasePage << PAGE_SHIFT;
 755                 Size = Descriptor->PageCount << PAGE_SHIFT;
 756
 757                 /* Check if it was already mapped */
 758                 if (Descriptor->VirtualPage)
 759                 {
 760                     /* Use the existing address */
 761                     VirtualAddress = (PVOID)(ULONG_PTR)(Descriptor->VirtualPage << PAGE_SHIFT);
 762                 }
 763                 else
 764                 {
 765                     /* Use the physical address */
 766                     VirtualAddress = (PVOID)(ULONG_PTR)PhysicalAddress.QuadPart;
 767                 }
 768
 769                 /* Crete the mapping */
 770                 Status = Mmx86MapInitStructure(VirtualAddress,
 771                                                Size,
 772                                                PhysicalAddress);
 773                 if (!NT_SUCCESS(Status))
 774                 {
 775                     return Status;
 776                 }
 777             }
 778
 779             /* Check if we're in phase 1 and deferring RAM disk */
 780             if ((Phase == 1) && (Descriptor->Type == BlLoaderRamDisk))
 781             {
 782                 MmDeferredMappingCount++;
 783             }
 784         }
 785
 786         /* Move on to the next descriptor */
 787         DescriptorCount--;
 788         Descriptor = (PBL_MEMORY_DESCRIPTOR)((ULONG_PTR)Descriptor + MemoryData->DescriptorSize);
 789     }
 790
 791     /* In phase 1, also do UEFI mappings */
 792     if (Phase != 2)
 793     {
 794         /* Get the memory map */
 795         MmMdInitializeListHead(&FirmwareMdl);
 796         Status = MmFwGetMemoryMap(&FirmwareMdl, BL_MM_FLAG_REQUEST_COALESCING);
 797         if (!NT_SUCCESS(Status))
 798         {
 799             return Status;
 800         }
 801
 802         /* Iterate over it */
 803         Head = FirmwareMdl.First;
 804         NextEntry = Head->Flink;
 805         while (NextEntry != Head)
 806         {
 807             /* Check if this is a UEFI-related descriptor, unless it's the self-map page */
 808             Descriptor = CONTAINING_RECORD(NextEntry, BL_MEMORY_DESCRIPTOR, ListEntry);
 809             if (((Descriptor->Type == BlEfiBootMemory) ||
 810                  (Descriptor->Type == BlEfiRuntimeCodeMemory) ||
 811                  (Descriptor->Type == BlEfiRuntimeDataMemory) || // WINBUG?
 812                  (Descriptor->Type == BlLoaderMemory)) &&
 813                 ((Descriptor->BasePage << PAGE_SHIFT) != Mmx86SelfMapBase.QuadPart))
 814             {
 815                 /* Identity-map it */
 816                 PhysicalAddress.QuadPart = Descriptor->BasePage << PAGE_SHIFT;
 817                 Status = Mmx86MapInitStructure((PVOID)((ULONG_PTR)Descriptor->BasePage << PAGE_SHIFT),
 818                                                Descriptor->PageCount << PAGE_SHIFT,
 819                                                PhysicalAddress);
 820                 if (!NT_SUCCESS(Status))
 821                 {
 822                     return Status;
 823                 }
 824             }
 825
 826             /* Move to the next descriptor */
 827             NextEntry = NextEntry->Flink;
 828         }
 829
 830         /* Reset */
 831         NextEntry = Head->Flink;
 832         while (NextEntry != Head)
 833         {
 834             /* Get the descriptor */
 835             Descriptor = CONTAINING_RECORD(NextEntry, BL_MEMORY_DESCRIPTOR, ListEntry);
 836
 837             /* Skip to the next entry before we free */
 838             NextEntry = NextEntry->Flink;
 839
 840             /* Remove and free it */
 841             MmMdRemoveDescriptorFromList(&FirmwareMdl, Descriptor);
 842             MmMdFreeDescriptor(Descriptor);
 843         }
 844     }
 845
 846     /* All library mappings identity mapped now */
 847     return STATUS_SUCCESS;
 848 }
 849
 850 NTSTATUS
 851 Mmx86InitializeMemoryMap (
 852     _In_ ULONG Phase,
 853     _In_ PBL_MEMORY_DATA MemoryData
 854     )
 855 {
 856     ULONG ImageSize;
 857     PVOID ImageBase;
 858     KDESCRIPTOR Gdt, Idt;
 859     NTSTATUS Status;
 860     PHYSICAL_ADDRESS PhysicalAddress;
 861
 862     /* If this is phase 2, map the memory regions */
 863     if (Phase != 1)
 864     {
 865         return Mmx86pMapMemoryRegions(Phase, MemoryData);
 866     }
 867
 868     /* Get the application image base/size */
 869     Status = BlGetApplicationBaseAndSize(&ImageBase, &ImageSize);
 870     if (!NT_SUCCESS(Status))
 871     {
 872         return Status;
 873     }
 874
 875     /* Map the image back at the same place */
 876     PhysicalAddress.QuadPart = (ULONG_PTR)ImageBase;
 877     Status = Mmx86MapInitStructure(ImageBase, ImageSize, PhysicalAddress);
 878     if (!NT_SUCCESS(Status))
 879     {
 880         return Status;
 881     }
 882
 883     /* Map the first 4MB of memory */
 884     PhysicalAddress.QuadPart = 0;
 885     Status = Mmx86MapInitStructure(NULL, 4 * 1024 * 1024, PhysicalAddress);
 886     if (!NT_SUCCESS(Status))
 887     {
 888         return Status;
 889     }
 890
 891     /* Map the GDT */
 892     _sgdt(&Gdt.Limit);
 893     PhysicalAddress.QuadPart = Gdt.Base;
 894     Status = Mmx86MapInitStructure((PVOID)Gdt.Base, Gdt.Limit + 1, PhysicalAddress);
 895     if (!NT_SUCCESS(Status))
 896     {
 897         return Status;
 898     }
 899
 900     /* Map the IDT */
 901     __sidt(&Idt.Limit);
 902     PhysicalAddress.QuadPart = Idt.Base;
 903     Status = Mmx86MapInitStructure((PVOID)Idt.Base, Idt.Limit + 1, PhysicalAddress);
 904     if (!NT_SUCCESS(Status))
 905     {
 906         return Status;
 907     }
 908
 909     /* Map the reference page */
 910     PhysicalAddress.QuadPart = (ULONG_PTR)MmArchReferencePage;
 911     Status = Mmx86MapInitStructure(MmArchReferencePage,
 912                                    MmArchReferencePageSize,
 913                                    PhysicalAddress);
 914     if (!NT_SUCCESS(Status))
 915     {
 916         return Status;
 917     }
 918
 919     /* More to do */
 920     return Mmx86pMapMemoryRegions(Phase, MemoryData);
 921 }
 922
 923 NTSTATUS
 924 MmDefInitializeTranslation (
 925     _In_ PBL_MEMORY_DATA MemoryData,
 926     _In_ BL_TRANSLATION_TYPE TranslationType
 927     )
 928 {
 929     NTSTATUS Status;
 930     PHYSICAL_ADDRESS PhysicalAddress;
 931     ULONG PdeIndex;
 932
 933     /* Set the global function pointers for memory translation */
 934     Mmx86TranslateVirtualAddress = MmDefpTranslateVirtualAddress;
 935     Mmx86MapPhysicalAddress = MmDefpMapPhysicalAddress;
 936     Mmx86UnmapVirtualAddress = MmDefpUnmapVirtualAddress;
 937     Mmx86RemapVirtualAddress = MmDefpRemapVirtualAddress;
 938     Mmx86FlushTlb = MmDefpFlushTlb;
 939     Mmx86FlushTlbEntry = MmDefpFlushTlbEntry;
 940     Mmx86DestroySelfMap = MmDefpDestroySelfMap;
 941
 942     /* Check what mode we're currently in */
 943     if (TranslationType == BlVirtual)
 944     {
 945         EfiPrintf(L"Virtual->Virtual not yet supported\r\n");
 946         return STATUS_NOT_IMPLEMENTED;
 947     }
 948     else if (TranslationType != BlNone)
 949     {
 950         /* Not even Windows supports PAE->Virtual downgrade */
 951         return STATUS_NOT_IMPLEMENTED;
 952     }
 953
 954     /* The None->Virtual case */
 955     MmPdpt = NULL;
 956     Mmx86SelfMapBase.QuadPart = 0;
 957     MmArchReferencePage = NULL;
 958
 959     /* Truncate all memory above 4GB so that we don't use it */
 960     Status = MmPaTruncateMemory(0x100000);
 961     if (!NT_SUCCESS(Status))
 962     {
 963         goto Failure;
 964     }
 965
 966     /* Allocate a page directory */
 967     Status = MmPapAllocatePhysicalPagesInRange(&PhysicalAddress,
 968                                                BlLoaderPageDirectory,
 969                                                1,
 970                                                0,
 971                                                0,
 972                                                &MmMdlUnmappedAllocated,
 973                                                0,
 974                                                0);
 975     if (!NT_SUCCESS(Status))
 976     {
 977         goto Failure;
 978     }
 979
 980     /* Zero out the page directory */
 981     MmPdpt = (PVOID)PhysicalAddress.LowPart;
 982     RtlZeroMemory(MmPdpt, PAGE_SIZE);
 983
 984     /* Set the page size */
 985     MmArchReferencePageSize = PAGE_SIZE;
 986
 987     /* Allocate the self-map page */
 988     Status = MmPapAllocatePhysicalPagesInRange(&PhysicalAddress,
 989                                                BlLoaderReferencePage,
 990                                                1,
 991                                                0,
 992                                                0,
 993                                                &MmMdlUnmappedAllocated,
 994                                                0,
 995                                                0);
 996     if (!NT_SUCCESS(Status))
 997     {
 998         goto Failure;
 999     }
1000
1001     /* Set the reference page */
1002     MmArchReferencePage = (PVOID)PhysicalAddress.LowPart;
1003
1004     /* Zero it out */
1005     RtlZeroMemory(MmArchReferencePage, MmArchReferencePageSize);
1006
1007     /* Allocate 4MB worth of self-map pages */
1008     Status = MmPaReserveSelfMapPages(&Mmx86SelfMapBase,
1009                                      (4 * 1024 * 1024) >> PAGE_SHIFT,
1010                                      (4 * 1024 * 1024) >> PAGE_SHIFT);
1011     if (!NT_SUCCESS(Status))
1012     {
1013         goto Failure;
1014     }
1015
1016     /* Zero them out */
1017     RtlZeroMemory((PVOID)Mmx86SelfMapBase.LowPart, 4 * 1024 * 1024);
1018     EfiPrintf(L"PDPT at 0x%p Reference Page at 0x%p Self-map at 0x%p\r\n",
1019               MmPdpt, MmArchReferencePage, Mmx86SelfMapBase.LowPart);
1020
1021     /* Align PTE base to 4MB region */
1022     MmPteBase = (PVOID)(Mmx86SelfMapBase.LowPart & ~0x3FFFFF);
1023
1024     /* The PDE is the PTE of the PTE base */
1025     MmPdeBase = MiAddressToPte(MmPteBase);
1026     PdeIndex = MiAddressToPdeOffset(MmPdeBase);
1027     MmPdpt[PdeIndex].u.Hard.Valid = 1;
1028     MmPdpt[PdeIndex].u.Hard.Write = 1;
1029     MmPdpt[PdeIndex].u.Hard.PageFrameNumber = (ULONG_PTR)MmPdpt >> PAGE_SHIFT;
1030     MmArchReferencePage[PdeIndex]++;
1031
1032     /* Remove PTE_BASE from free virtual memory */
1033     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
1034                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
1035                                        PTE_BASE >> PAGE_SHIFT,
1036                                        (4 * 1024 * 1024) >> PAGE_SHIFT,
1037                                        0);
1038     if (!NT_SUCCESS(Status))
1039     {
1040         goto Failure;
1041     }
1042
1043     /* Remove HAL_HEAP from free virtual memory */
1044     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
1045                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
1046                                        MM_HAL_VA_START >> PAGE_SHIFT,
1047                                        (4 * 1024 * 1024) >> PAGE_SHIFT,
1048                                        0);
1049     if (!NT_SUCCESS(Status))
1050     {
1051         goto Failure;
1052     }
1053
1054     /* Initialize the virtual->physical memory mappings */
1055     Status = Mmx86InitializeMemoryMap(1, MemoryData);
1056     if (!NT_SUCCESS(Status))
1057     {
1058         goto Failure;
1059     }
1060
1061     /* Turn on paging with the new CR3 */
1062     __writecr3((ULONG_PTR)MmPdpt);
1063     BlpArchEnableTranslation();
1064     EfiPrintf(L"Paging... %d\r\n", BlMmIsTranslationEnabled());
1065
1066     /* Return success */
1067     return Status;
1068
1069 Failure:
1070     /* Free reference page if we allocated it */
1071     if (MmArchReferencePage)
1072     {
1073         PhysicalAddress.QuadPart = (ULONG_PTR)MmArchReferencePage;
1074         BlMmFreePhysicalPages(PhysicalAddress);
1075     }
1076
1077     /* Free page directory if we allocated it */
1078     if (MmPdpt)
1079     {
1080         PhysicalAddress.QuadPart = (ULONG_PTR)MmPdpt;
1081         BlMmFreePhysicalPages(PhysicalAddress);
1082     }
1083
1084     /* Free the self map if we allocated it */
1085     if (Mmx86SelfMapBase.QuadPart)
1086     {
1087         MmPaReleaseSelfMapPages(Mmx86SelfMapBase);
1088     }
1089
1090     /* All done */
1091     return Status;
1092 }
1093
1094 NTSTATUS
1095 MmArchInitialize (
1096     _In_ ULONG Phase,
1097     _In_ PBL_MEMORY_DATA MemoryData,
1098     _In_ BL_TRANSLATION_TYPE TranslationType,
1099     _In_ BL_TRANSLATION_TYPE RequestedTranslationType
1100     )
1101 {
1102     NTSTATUS Status;
1103     ULONGLONG IncreaseUserVa, PerfCounter, CpuRandom;
1104     CPU_INFO CpuInfo;
1105
1106     /* For phase 2, just map deferred regions */
1107     if (Phase != 1)
1108     {
1109         return Mmx86pMapMemoryRegions(2, MemoryData);
1110     }
1111
1112     /* What translation type are we switching to? */
1113     switch (RequestedTranslationType)
1114     {
1115         /* Physical memory */
1116         case BlNone:
1117
1118             /* Initialize everything to default/null values */
1119             MmArchLargePageSize = 1;
1120             MmArchKsegBase = 0;
1121             MmArchKsegBias = 0;
1122             MmArchKsegAddressRange.Minimum = 0;
1123             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1124             MmArchTopOfApplicationAddressSpace = 0;
1125             Mmx86SelfMapBase.QuadPart = 0;
1126
1127             /* Set stub functions */
1128             BlMmRelocateSelfMap = MmArchNullFunction;
1129             BlMmFlushTlb = MmArchNullFunction;
1130
1131             /* Set success */
1132             Status = STATUS_SUCCESS;
1133             break;
1134
1135         case BlVirtual:
1136
1137             /* Set the large page size to 1024 pages (4MB) */
1138             MmArchLargePageSize = (4 * 1024 * 1024) / PAGE_SIZE;
1139
1140             /* Check if /USERVA option was used */
1141             Status = BlGetBootOptionInteger(BlpApplicationEntry.BcdData,
1142                                             BcdOSLoaderInteger_IncreaseUserVa,
1143                                             &IncreaseUserVa);
1144             if (NT_SUCCESS(Status) && (IncreaseUserVa))
1145             {
1146                 /* Yes -- load the kernel at 0xE0000000 instead */
1147                 MmArchKsegBase = 0xE0000000;
1148             }
1149             else
1150             {
1151                 /* Nope, load at the standard 2GB split */
1152                 MmArchKsegBase = 0x80000000;
1153             }
1154
1155             /* Check if CPUID 01h is supported */
1156             CpuRandom = 0;
1157             if (BlArchIsCpuIdFunctionSupported(1))
1158             {
1159                 /* Call it */
1160                 BlArchCpuId(1, 0, &CpuInfo);
1161
1162                 /* Check if RDRAND is supported */
1163                 if (CpuInfo.Ecx & 0x40000000)
1164                 {
1165                     EfiPrintf(L"Your CPU can do RDRAND! Good for you!\r\n");
1166                     CpuRandom = 0;
1167                 }
1168             }
1169
1170             /* Read the TSC */
1171             PerfCounter = BlArchGetPerformanceCounter();
1172             PerfCounter >>= 4;
1173             _rotl16(PerfCounter, 5);
1174
1175             /* Set the address range */
1176             MmArchKsegAddressRange.Minimum = 0;
1177             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1178
1179             /* Set the KASLR bias */
1180             MmArchKsegBias = ((PerfCounter ^ CpuRandom) & 0xFFF) << 12;
1181             MmArchKsegBias = 0;
1182             MmArchKsegBase += MmArchKsegBias;
1183
1184             /* Set the kernel range */
1185             MmArchKsegAddressRange.Minimum = MmArchKsegBase;
1186             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1187
1188             /* Set the boot application top maximum */
1189             MmArchTopOfApplicationAddressSpace = 0x70000000 - 1; // Windows bug
1190
1191             /* Initialize virtual address space translation */
1192             Status = MmDefInitializeTranslation(MemoryData, TranslationType);
1193             if (NT_SUCCESS(Status))
1194             {
1195                 /* Set stub functions */
1196                 BlMmRelocateSelfMap = MmDefRelocateSelfMap;
1197                 BlMmFlushTlb = Mmx86FlushTlb;
1198                 BlMmMoveVirtualAddressRange = MmDefMoveVirtualAddressRange;
1199                 BlMmZeroVirtualAddressRange = MmDefZeroVirtualAddressRange;
1200             }
1201             break;
1202
1203         case BlPae:
1204
1205             /* We don't support PAE */
1206             Status = STATUS_NOT_SUPPORTED;
1207             break;
1208
1209         default:
1210
1211             /* Invalid architecture type*/
1212             Status = STATUS_INVALID_PARAMETER;
1213             break;
1214     }
1215
1216     /* Back to caller */
1217     return Status;
1218 }