reactos/boot/environ/lib/mm/i386/mmx86.c

   1 /*
   2 * COPYRIGHT:       See COPYING.ARM in the top level directory
   3 * PROJECT:         ReactOS UEFI Boot Library
   4 * FILE:            boot/environ/lib/mm/i386/mmx86.c
   5 * PURPOSE:         Boot Library Memory Manager x86-Specific Code
   6 * PROGRAMMER:      Alex Ionescu (alex.ionescu@reactos.org)
   7 */
   8
   9 /* INCLUDES ******************************************************************/
  10
  11 #include "bl.h"
  12 #include "bcd.h"
  13
  14 #define PTE_BASE                0xC0000000
  15
  16 //
  17 // Specific PDE/PTE macros to be used inside the boot library environment
  18 //
  19 #define MiAddressToPte(x)       ((PMMPTE)(((((ULONG)(x)) >> 12) << 2) + (ULONG_PTR)MmPteBase))
  20 #define MiAddressToPde(x)       ((PMMPDE)(((((ULONG)(x)) >> 22) << 2) + (ULONG_PTR)MmPdeBase))
  21 #define MiAddressToPteOffset(x) ((((ULONG)(x)) << 10) >> 22)
  22 #define MiAddressToPdeOffset(x) (((ULONG)(x)) / (1024 * PAGE_SIZE))
  23
  24 /* DATA VARIABLES ************************************************************/
  25
  26 ULONG_PTR MmArchKsegBase;
  27 ULONG_PTR MmArchKsegBias;
  28 ULONG MmArchLargePageSize;
  29 BL_ADDRESS_RANGE MmArchKsegAddressRange;
  30 ULONG_PTR MmArchTopOfApplicationAddressSpace;
  31 PHYSICAL_ADDRESS Mmx86SelfMapBase;
  32 ULONG MmDeferredMappingCount;
  33 PMMPTE MmPdpt;
  34 PULONG MmArchReferencePage;
  35 PVOID MmPteBase;
  36 PVOID MmPdeBase;
  37 ULONG MmArchReferencePageSize;
  38
  39 typedef VOID
  40 (*PBL_MM_FLUSH_TLB) (
  41     VOID
  42     );
  43
  44 typedef VOID
  45 (*PBL_MM_RELOCATE_SELF_MAP) (
  46     VOID
  47     );
  48
  49 typedef NTSTATUS
  50 (*PBL_MM_MOVE_VIRTUAL_ADDRESS_RANGE) (
  51     _In_ PVOID DestinationAddress,
  52     _In_ PVOID SourceAddress,
  53     _In_ ULONGLONG Size
  54     );
  55
  56 typedef NTSTATUS
  57 (*PBL_MM_ZERO_VIRTUAL_ADDRESS_RANGE) (
  58     _In_ PVOID DestinationAddress,
  59     _In_ ULONGLONG Size
  60     );
  61
  62 typedef VOID
  63 (*PBL_MM_DESTROY_SELF_MAP) (
  64     VOID
  65     );
  66
  67 typedef VOID
  68 (*PBL_MM_FLUSH_TLB_ENTRY) (
  69     _In_ PVOID VirtualAddress
  70     );
  71
  72 typedef VOID
  73 (*PBL_MM_FLUSH_TLB) (
  74     VOID
  75     );
  76
  77 typedef NTSTATUS
  78 (*PBL_MM_UNMAP_VIRTUAL_ADDRESS) (
  79     _In_ PVOID VirtualAddress,
  80     _In_ ULONG Size
  81     );
  82
  83 typedef NTSTATUS
  84 (*PBL_MM_REMAP_VIRTUAL_ADDRESS) (
  85     _In_ PPHYSICAL_ADDRESS PhysicalAddress,
  86     _Out_ PVOID VirtualAddress,
  87     _In_ ULONG Size,
  88     _In_ ULONG CacheAttributes
  89     );
  90
  91 typedef NTSTATUS
  92 (*PBL_MM_MAP_PHYSICAL_ADDRESS) (
  93     _In_ PHYSICAL_ADDRESS PhysicalAddress,
  94     _Out_ PVOID VirtualAddress,
  95     _In_ ULONG Size,
  96     _In_ ULONG CacheAttributes
  97     );
  98
  99 typedef BOOLEAN
 100 (*PBL_MM_TRANSLATE_VIRTUAL_ADDRESS) (
 101     _In_ PVOID VirtualAddress,
 102     _Out_ PPHYSICAL_ADDRESS PhysicalAddress,
 103     _Out_opt_ PULONG CacheAttributes
 104     );
 105
 106 PBL_MM_TRANSLATE_VIRTUAL_ADDRESS Mmx86TranslateVirtualAddress;
 107 PBL_MM_MAP_PHYSICAL_ADDRESS Mmx86MapPhysicalAddress;
 108 PBL_MM_REMAP_VIRTUAL_ADDRESS Mmx86RemapVirtualAddress;
 109 PBL_MM_UNMAP_VIRTUAL_ADDRESS Mmx86UnmapVirtualAddress;
 110 PBL_MM_FLUSH_TLB Mmx86FlushTlb;
 111 PBL_MM_FLUSH_TLB_ENTRY Mmx86FlushTlbEntry;
 112 PBL_MM_DESTROY_SELF_MAP Mmx86DestroySelfMap;
 113
 114 PBL_MM_RELOCATE_SELF_MAP BlMmRelocateSelfMap;
 115 PBL_MM_FLUSH_TLB BlMmFlushTlb;
 116 PBL_MM_MOVE_VIRTUAL_ADDRESS_RANGE BlMmMoveVirtualAddressRange;
 117 PBL_MM_ZERO_VIRTUAL_ADDRESS_RANGE BlMmZeroVirtualAddressRange;
 118
 119 PBL_MM_FLUSH_TLB Mmx86FlushTlb;
 120
 121 /* FUNCTIONS *****************************************************************/
 122
 123 BOOLEAN
 124 BlMmIsTranslationEnabled (
 125     VOID
 126     )
 127 {
 128     /* Return if paging is on */
 129     return ((CurrentExecutionContext) &&
 130             (CurrentExecutionContext->Mode & BL_CONTEXT_PAGING_ON));
 131 }
 132
 133 VOID
 134 MmArchNullFunction (
 135     VOID
 136     )
 137 {
 138     /* Nothing to do */
 139     return;
 140 }
 141
 142 VOID
 143 MmDefRelocateSelfMap (
 144     VOID
 145     )
 146 {
 147     if (MmPteBase != (PVOID)PTE_BASE)
 148     {
 149         EfiPrintf(L"Supposed to relocate CR3\r\n");
 150     }
 151 }
 152
 153 NTSTATUS
 154 MmDefMoveVirtualAddressRange (
 155     _In_ PVOID DestinationAddress,
 156     _In_ PVOID SourceAddress,
 157     _In_ ULONGLONG Size
 158     )
 159 {
 160     EfiPrintf(L"Supposed to move shit\r\n");
 161     return STATUS_NOT_IMPLEMENTED;
 162 }
 163
 164 NTSTATUS
 165 MmDefZeroVirtualAddressRange (
 166     _In_ PVOID DestinationAddress,
 167     _In_ ULONGLONG Size
 168     )
 169 {
 170     EfiPrintf(L"Supposed to zero shit\r\n");
 171     return STATUS_NOT_IMPLEMENTED;
 172 }
 173
 174 BOOLEAN
 175 MmArchTranslateVirtualAddress (
 176     _In_ PVOID VirtualAddress,
 177     _Out_opt_ PPHYSICAL_ADDRESS PhysicalAddress,
 178     _Out_opt_ PULONG CachingFlags
 179     )
 180 {
 181     PBL_MEMORY_DESCRIPTOR Descriptor;
 182
 183     /* Check if paging is on */
 184     if ((CurrentExecutionContext) &&
 185         (CurrentExecutionContext->ContextFlags & BL_CONTEXT_PAGING_ON))
 186     {
 187         /* Yes -- we have to translate this from virtual */
 188         return Mmx86TranslateVirtualAddress(VirtualAddress,
 189                                             PhysicalAddress,
 190                                             CachingFlags);
 191     }
 192
 193     /* Look in all descriptors except truncated and firmware ones */
 194     Descriptor = MmMdFindDescriptor(BL_MM_INCLUDE_NO_FIRMWARE_MEMORY &
 195                                     ~BL_MM_INCLUDE_TRUNCATED_MEMORY,
 196                                     BL_MM_REMOVE_PHYSICAL_REGION_FLAG,
 197                                     (ULONG_PTR)VirtualAddress >> PAGE_SHIFT);
 198
 199     /* Return the virtual address as the physical address */
 200     if (PhysicalAddress)
 201     {
 202         PhysicalAddress->HighPart = 0;
 203         PhysicalAddress->LowPart = (ULONG_PTR)VirtualAddress;
 204     }
 205
 206     /* There's no caching on physical memory */
 207     if (CachingFlags)
 208     {
 209         *CachingFlags = 0;
 210     }
 211
 212     /* Success is if we found a descriptor */
 213     return Descriptor != NULL;
 214 }
 215
 216 VOID
 217 MmDefpDestroySelfMap (
 218     VOID
 219     )
 220 {
 221     EfiPrintf(L"No destroy\r\n");
 222 }
 223
 224 VOID
 225 MmDefpFlushTlbEntry (
 226     _In_ PVOID VirtualAddress
 227     )
 228 {
 229     /* Flush the TLB */
 230     __invlpg(VirtualAddress);
 231 }
 232
 233 VOID
 234 MmDefpFlushTlb (
 235     VOID
 236     )
 237 {
 238     /* Flush the TLB */
 239     __writecr3(__readcr3());
 240 }
 241
 242 NTSTATUS
 243 MmDefpUnmapVirtualAddress (
 244     _In_ PVOID VirtualAddress,
 245     _In_ ULONG Size
 246     )
 247 {
 248     EfiPrintf(L"No unmap\r\n");
 249     return STATUS_NOT_IMPLEMENTED;
 250 }
 251
 252 NTSTATUS
 253 MmDefpRemapVirtualAddress (
 254     _In_ PPHYSICAL_ADDRESS PhysicalAddress,
 255     _Out_ PVOID VirtualAddress,
 256     _In_ ULONG Size,
 257     _In_ ULONG CacheAttributes
 258     )
 259 {
 260     EfiPrintf(L"No remap\r\n");
 261     return STATUS_NOT_IMPLEMENTED;
 262 }
 263
 264 NTSTATUS
 265 MmDefpMapPhysicalAddress (
 266     _In_ PHYSICAL_ADDRESS PhysicalAddress,
 267     _In_ PVOID VirtualAddress,
 268     _In_ ULONG Size,
 269     _In_ ULONG CacheAttributes
 270     )
 271 {
 272     BOOLEAN Enabled;
 273     ULONG i, PageCount, PdeOffset;
 274     ULONGLONG CurrentAddress;
 275     PMMPDE Pde;
 276     PMMPTE Pte;
 277     PMMPTE PageTable;
 278     PHYSICAL_ADDRESS PageTableAddress;
 279     NTSTATUS Status;
 280
 281     /* Check if paging is on yet */
 282     Enabled = BlMmIsTranslationEnabled();
 283
 284     /* Get the physical address aligned */
 285     CurrentAddress = (PhysicalAddress.QuadPart >> PAGE_SHIFT) << PAGE_SHIFT;
 286
 287     /* Get the number of pages and loop through each one */
 288     PageCount = Size >> PAGE_SHIFT;
 289     for (i = 0; i < PageCount; i++)
 290     {
 291         /* Check if translation already exists for this page */
 292         if (Mmx86TranslateVirtualAddress(VirtualAddress, NULL, NULL))
 293         {
 294             /* Ignore it and move to the next one */
 295             VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress + PAGE_SIZE);
 296             CurrentAddress += PAGE_SIZE;
 297             continue;
 298         }
 299
 300         /* Get the PDE offset */
 301         PdeOffset = MiAddressToPdeOffset(VirtualAddress);
 302
 303         /* Check if paging is actually turned on */
 304         if (Enabled)
 305         {
 306             /* Get the PDE entry using the self-map */
 307             Pde = MiAddressToPde(VirtualAddress);
 308         }
 309         else
 310         {
 311             /* Get it using our physical mappings */
 312             Pde = &MmPdpt[PdeOffset];
 313             PageTable = (PMMPDE)(Pde->u.Hard.PageFrameNumber << PAGE_SHIFT);
 314         }
 315
 316         /* Check if we don't yet have a PDE */
 317         if (!Pde->u.Hard.Valid)
 318         {
 319             /* Allocate a page table */
 320             Status = MmPapAllocatePhysicalPagesInRange(&PageTableAddress,
 321                                                        BlLoaderPageDirectory,
 322                                                        1,
 323                                                        0,
 324                                                        0,
 325                                                        &MmMdlUnmappedAllocated,
 326                                                        0,
 327                                                        0);
 328             if (!NT_SUCCESS(Status))
 329             {
 330                 return STATUS_NO_MEMORY;
 331             }
 332
 333             /* This is our page table */
 334             PageTable = (PVOID)(ULONG_PTR)PageTableAddress.QuadPart;
 335
 336             /* Build the PDE for it */
 337             Pde->u.Hard.PageFrameNumber = PageTableAddress.QuadPart >> PAGE_SHIFT;
 338             Pde->u.Hard.Write = 1;
 339             Pde->u.Hard.CacheDisable = 1;
 340             Pde->u.Hard.WriteThrough = 1;
 341             Pde->u.Hard.Valid = 1;
 342
 343             /* Check if paging is enabled */
 344             if (Enabled)
 345             {
 346                 /* Then actually, get the page table's virtual address */
 347                 PageTable = (PVOID)PAGE_ROUND_DOWN(MiAddressToPte(VirtualAddress));
 348
 349                 /* Flush the TLB */
 350                 Mmx86FlushTlb();
 351             }
 352
 353             /* Zero out the page table */
 354             RtlZeroMemory(PageTable, PAGE_SIZE);
 355
 356             /* Reset caching attributes now */
 357             Pde->u.Hard.CacheDisable = 0;
 358             Pde->u.Hard.WriteThrough = 0;
 359
 360             /* Check for paging again */
 361             if (Enabled)
 362             {
 363                 /* Flush the TLB entry for the page table only */
 364                 Mmx86FlushTlbEntry(PageTable);
 365             }
 366         }
 367
 368         /* Add a reference to this page table */
 369         MmArchReferencePage[PdeOffset]++;
 370
 371         /* Check if a physical address was given */
 372         if (PhysicalAddress.QuadPart != -1)
 373         {
 374             /* Check if paging is turned on */
 375             if (Enabled)
 376             {
 377                 /* Get the PTE using the self-map */
 378                 Pte = MiAddressToPte(VirtualAddress);
 379             }
 380             else
 381             {
 382                 /* Get the PTE using physical addressing */
 383                 Pte = &PageTable[MiAddressToPteOffset(VirtualAddress)];
 384             }
 385
 386             /* Build a valid PTE for it */
 387             Pte->u.Hard.PageFrameNumber = CurrentAddress >> PAGE_SHIFT;
 388             Pte->u.Hard.Write = 1;
 389             Pte->u.Hard.Valid = 1;
 390
 391             /* Check if this is uncached */
 392             if (CacheAttributes == BlMemoryUncached)
 393             {
 394                 /* Set the flags */
 395                 Pte->u.Hard.CacheDisable = 1;
 396                 Pte->u.Hard.WriteThrough = 1;
 397             }
 398             else if (CacheAttributes == BlMemoryWriteThrough)
 399             {
 400                 /* It's write-through, set the flag */
 401                 Pte->u.Hard.WriteThrough = 1;
 402             }
 403         }
 404
 405         /* Move to the next physical/virtual address */
 406         VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress + PAGE_SIZE);
 407         CurrentAddress += PAGE_SIZE;
 408     }
 409
 410     /* All done! */
 411     return STATUS_SUCCESS;
 412 }
 413
 414 BOOLEAN
 415 MmDefpTranslateVirtualAddress (
 416     _In_ PVOID VirtualAddress,
 417     _Out_ PPHYSICAL_ADDRESS PhysicalAddress,
 418     _Out_opt_ PULONG CacheAttributes
 419     )
 420 {
 421     PMMPDE Pde;
 422     PMMPTE Pte;
 423     PMMPTE PageTable;
 424     BOOLEAN Enabled;
 425
 426     /* Is there no page directory yet? */
 427     if (!MmPdpt)
 428     {
 429         return FALSE;
 430     }
 431
 432     /* Is paging enabled? */
 433     Enabled = BlMmIsTranslationEnabled();
 434
 435     /* Check if paging is actually turned on */
 436     if (Enabled)
 437     {
 438         /* Get the PDE entry using the self-map */
 439         Pde = MiAddressToPde(VirtualAddress);
 440     }
 441     else
 442     {
 443         /* Get it using our physical mappings */
 444         Pde = &MmPdpt[MiAddressToPdeOffset(VirtualAddress)];
 445     }
 446
 447     /* Is the PDE valid? */
 448     if (!Pde->u.Hard.Valid)
 449     {
 450         return FALSE;
 451     }
 452
 453     /* Check if paging is turned on */
 454     if (Enabled)
 455     {
 456         /* Get the PTE using the self-map */
 457         Pte = MiAddressToPte(VirtualAddress);
 458     }
 459     else
 460     {
 461         /* Get the PTE using physical addressing */
 462         PageTable = (PMMPTE)(Pde->u.Hard.PageFrameNumber << PAGE_SHIFT);
 463         Pte = &PageTable[MiAddressToPteOffset(VirtualAddress)];
 464     }
 465
 466     /* Is the PTE valid? */
 467     if (!Pte->u.Hard.Valid)
 468     {
 469         return FALSE;
 470     }
 471
 472     /* Does caller want the physical address?  */
 473     if (PhysicalAddress)
 474     {
 475         /* Return it */
 476         PhysicalAddress->QuadPart = (Pte->u.Hard.PageFrameNumber << PAGE_SHIFT) +
 477                                      BYTE_OFFSET(VirtualAddress);
 478     }
 479
 480     /* Does caller want cache attributes? */
 481     if (CacheAttributes)
 482     {
 483         /* Not yet -- lie and say it's cached */
 484         EfiPrintf(L"Cache checking not yet enabled\r\n");
 485         *CacheAttributes = BlMemoryWriteBack;
 486     }
 487
 488     /* It exists! */
 489     return TRUE;
 490 }
 491
 492 NTSTATUS
 493 MmMapPhysicalAddress (
 494     _Inout_ PPHYSICAL_ADDRESS PhysicalAddressPtr,
 495     _Inout_ PVOID* VirtualAddressPtr,
 496     _Inout_ PULONGLONG SizePtr,
 497     _In_ ULONG CacheAttributes
 498     )
 499 {
 500     ULONGLONG Size;
 501     ULONGLONG PhysicalAddress;
 502     PVOID VirtualAddress;
 503     PHYSICAL_ADDRESS TranslatedAddress;
 504     ULONG_PTR CurrentAddress, VirtualAddressEnd;
 505     NTSTATUS Status;
 506
 507     /* Fail if any parameters are missing */
 508     if (!(PhysicalAddressPtr) || !(VirtualAddressPtr) || !(SizePtr))
 509     {
 510         return STATUS_INVALID_PARAMETER;
 511     }
 512
 513     /* Fail if the size is over 32-bits */
 514     Size = *SizePtr;
 515     if (Size > 0xFFFFFFFF)
 516     {
 517         return STATUS_INVALID_PARAMETER;
 518     }
 519
 520     /* Nothing to do if we're in physical mode */
 521     if (MmTranslationType == BlNone)
 522     {
 523         return STATUS_SUCCESS;
 524     }
 525
 526     /* Can't use virtual memory in real mode */
 527     if (CurrentExecutionContext->Mode == BlRealMode)
 528     {
 529         return STATUS_UNSUCCESSFUL;
 530     }
 531
 532     /* Capture the current virtual and physical addresses */
 533     VirtualAddress = *VirtualAddressPtr;
 534     PhysicalAddress = PhysicalAddressPtr->QuadPart;
 535
 536     /* Check if a physical address was requested */
 537     if (PhysicalAddress != 0xFFFFFFFF)
 538     {
 539         /* Round down the base addresses */
 540         PhysicalAddress = PAGE_ROUND_DOWN(PhysicalAddress);
 541         VirtualAddress = (PVOID)PAGE_ROUND_DOWN(VirtualAddress);
 542
 543         /* Round up the size */
 544         Size = ROUND_TO_PAGES(PhysicalAddressPtr->QuadPart -
 545                               PhysicalAddress +
 546                               Size);
 547
 548         /* Loop every virtual page */
 549         CurrentAddress = (ULONG_PTR)VirtualAddress;
 550         VirtualAddressEnd = CurrentAddress + Size - 1;
 551         while (CurrentAddress < VirtualAddressEnd)
 552         {
 553             /* Get the physical page of this virtual page */
 554             if (MmArchTranslateVirtualAddress((PVOID)CurrentAddress,
 555                                               &TranslatedAddress,
 556                                               &CacheAttributes))
 557             {
 558                 /* Make sure the physical page of the virtual page, matches our page */
 559                 if (TranslatedAddress.QuadPart !=
 560                     (PhysicalAddress +
 561                      (CurrentAddress - (ULONG_PTR)VirtualAddress)))
 562                 {
 563                     /* There is an existing virtual mapping for a different address */
 564                     EfiPrintf(L"Existing mapping exists: %lx vs %lx\r\n",
 565                               TranslatedAddress.QuadPart,
 566                               PhysicalAddress + (CurrentAddress - (ULONG_PTR)VirtualAddress));
 567                     EfiStall(10000000);
 568                     return STATUS_INVALID_PARAMETER;
 569                 }
 570             }
 571
 572             /* Try the next one */
 573             CurrentAddress += PAGE_SIZE;
 574         }
 575     }
 576
 577     /* Aactually do the mapping */
 578     TranslatedAddress.QuadPart = PhysicalAddress;
 579     Status = Mmx86MapPhysicalAddress(TranslatedAddress,
 580                                      VirtualAddress,
 581                                      Size,
 582                                      CacheAttributes);
 583     if (!NT_SUCCESS(Status))
 584     {
 585         EfiPrintf(L"Failed to map!: %lx\r\n", Status);
 586         return Status;
 587     }
 588
 589     /* Return aligned/fixed up output parameters */
 590     PhysicalAddressPtr->QuadPart = PhysicalAddress;
 591     *VirtualAddressPtr = VirtualAddress;
 592     *SizePtr = Size;
 593
 594     /* Flush the TLB if paging is enabled */
 595     if (BlMmIsTranslationEnabled())
 596     {
 597         Mmx86FlushTlb();
 598     }
 599
 600     /* All good! */
 601     return STATUS_SUCCESS;
 602 }
 603
 604 NTSTATUS
 605 Mmx86MapInitStructure (
 606     _In_ PVOID VirtualAddress,
 607     _In_ ULONGLONG Size,
 608     _In_ PHYSICAL_ADDRESS PhysicalAddress
 609     )
 610 {
 611     NTSTATUS Status;
 612
 613     /* Make a virtual mapping for this physical address */
 614     Status = MmMapPhysicalAddress(&PhysicalAddress, &VirtualAddress, &Size, 0);
 615     if (!NT_SUCCESS(Status))
 616     {
 617         return Status;
 618     }
 619
 620     /* Nothing else to do if we're not in paging mode */
 621     if (MmTranslationType == BlNone)
 622     {
 623         return STATUS_SUCCESS;
 624     }
 625
 626     /* Otherwise, remove this region from the list of free virtual ranges */
 627     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
 628                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
 629                                        (ULONG_PTR)VirtualAddress >> PAGE_SHIFT,
 630                                        Size >> PAGE_SHIFT,
 631                                        0);
 632     if (!NT_SUCCESS(Status))
 633     {
 634         /* Unmap the address if that failed */
 635         MmUnmapVirtualAddress(&VirtualAddress, &Size);
 636     }
 637
 638     /* Return back to caller */
 639     return Status;
 640 }
 641
 642 VOID
 643 MmMdDbgDumpList (
 644     _In_ PBL_MEMORY_DESCRIPTOR_LIST DescriptorList
 645     )
 646 {
 647     ULONGLONG EndPage, VirtualEndPage;
 648     PBL_MEMORY_DESCRIPTOR MemoryDescriptor;
 649     PLIST_ENTRY NextEntry;
 650
 651     NextEntry = DescriptorList->First->Flink;
 652     while (NextEntry != DescriptorList->First)
 653     {
 654         MemoryDescriptor = CONTAINING_RECORD(NextEntry,
 655                                              BL_MEMORY_DESCRIPTOR,
 656                                              ListEntry);
 657
 658         EndPage = MemoryDescriptor->BasePage + MemoryDescriptor->PageCount;
 659         if (MemoryDescriptor->VirtualPage != 0)
 660         {
 661             VirtualEndPage = MemoryDescriptor->VirtualPage + MemoryDescriptor->PageCount;
 662         }
 663         else
 664         {
 665             VirtualEndPage = 0;
 666         }
 667
 668         EfiPrintf(L"%p - [%08llx-%08llx @ %08llx-%08llx]:%x\r\n",
 669                     MemoryDescriptor,
 670                     MemoryDescriptor->BasePage << PAGE_SHIFT,
 671                     (EndPage << PAGE_SHIFT) - 1,
 672                     MemoryDescriptor->VirtualPage << PAGE_SHIFT,
 673                     VirtualEndPage ? (VirtualEndPage << PAGE_SHIFT) - 1 : 0,
 674                     (ULONG)MemoryDescriptor->Type);
 675
 676         NextEntry = NextEntry->Flink;
 677     }
 678 }
 679
 680 NTSTATUS
 681 Mmx86pMapMemoryRegions (
 682     _In_ ULONG Phase,
 683     _In_ PBL_MEMORY_DATA MemoryData
 684     )
 685 {
 686     BOOLEAN DoDeferred;
 687     ULONG DescriptorCount;
 688     PBL_MEMORY_DESCRIPTOR Descriptor;
 689     ULONG FinalOffset;
 690     PHYSICAL_ADDRESS PhysicalAddress;
 691     ULONGLONG Size;
 692     NTSTATUS Status;
 693     PVOID VirtualAddress;
 694     BL_MEMORY_DESCRIPTOR_LIST FirmwareMdl;
 695     PLIST_ENTRY Head, NextEntry;
 696
 697     /* Check which phase this is */
 698     if (Phase == 1)
 699     {
 700         /* In phase 1 we don't initialize deferred mappings */
 701         DoDeferred = FALSE;
 702     }
 703     else
 704     {
 705         /* Don't do anything if there's nothing to initialize */
 706         if (!MmDeferredMappingCount)
 707         {
 708             return STATUS_SUCCESS;
 709         }
 710
 711         /* We'll do deferred descriptors in phase 2 */
 712         DoDeferred = TRUE;
 713     }
 714
 715     /*
 716     * Because BL supports cross x86-x64 application launches and a LIST_ENTRY
 717     * is of variable size, care must be taken here to ensure that we see a
 718     * consistent view of descriptors. BL uses some offset magic to figure out
 719     * where the data actually starts, since everything is ULONGLONG past the
 720     * LIST_ENTRY itself
 721     */
 722     FinalOffset = MemoryData->MdListOffset + MemoryData->DescriptorOffset;
 723     Descriptor = (PBL_MEMORY_DESCRIPTOR)((ULONG_PTR)MemoryData + FinalOffset -
 724                                          FIELD_OFFSET(BL_MEMORY_DESCRIPTOR, BasePage));
 725
 726     /* Scan all of them */
 727     DescriptorCount = MemoryData->DescriptorCount;
 728     while (DescriptorCount != 0)
 729     {
 730         /* Ignore application data */
 731         if (Descriptor->Type != BlApplicationData)
 732         {
 733             /* If this is a ramdisk, do it in phase 2 */
 734             if ((Descriptor->Type == BlLoaderRamDisk) == DoDeferred)
 735             {
 736                 /* Get the current physical address and size */
 737                 PhysicalAddress.QuadPart = Descriptor->BasePage << PAGE_SHIFT;
 738                 Size = Descriptor->PageCount << PAGE_SHIFT;
 739
 740                 /* Check if it was already mapped */
 741                 if (Descriptor->VirtualPage)
 742                 {
 743                     /* Use the existing address */
 744                     VirtualAddress = (PVOID)(ULONG_PTR)(Descriptor->VirtualPage << PAGE_SHIFT);
 745                 }
 746                 else
 747                 {
 748                     /* Use the physical address */
 749                     VirtualAddress = (PVOID)(ULONG_PTR)PhysicalAddress.QuadPart;
 750                 }
 751
 752                 /* Crete the mapping */
 753                 Status = Mmx86MapInitStructure(VirtualAddress,
 754                                                Size,
 755                                                PhysicalAddress);
 756                 if (!NT_SUCCESS(Status))
 757                 {
 758                     return Status;
 759                 }
 760             }
 761
 762             /* Check if we're in phase 1 and deferring RAM disk */
 763             if ((Phase == 1) && (Descriptor->Type == BlLoaderRamDisk))
 764             {
 765                 MmDeferredMappingCount++;
 766             }
 767         }
 768
 769         /* Move on to the next descriptor */
 770         DescriptorCount--;
 771         Descriptor = (PBL_MEMORY_DESCRIPTOR)((ULONG_PTR)Descriptor + MemoryData->DescriptorSize);
 772     }
 773
 774     /* In phase 1, also do UEFI mappings */
 775     if (Phase != 2)
 776     {
 777         /* Get the memory map */
 778         MmMdInitializeListHead(&FirmwareMdl);
 779         Status = MmFwGetMemoryMap(&FirmwareMdl, BL_MM_FLAG_REQUEST_COALESCING);
 780         if (!NT_SUCCESS(Status))
 781         {
 782             return Status;
 783         }
 784
 785         /* Iterate over it */
 786         Head = FirmwareMdl.First;
 787         NextEntry = Head->Flink;
 788         while (NextEntry != Head)
 789         {
 790             /* Check if this is a UEFI-related descriptor, unless it's the self-map page */
 791             Descriptor = CONTAINING_RECORD(NextEntry, BL_MEMORY_DESCRIPTOR, ListEntry);
 792             if (((Descriptor->Type == BlEfiBootMemory) ||
 793                  (Descriptor->Type == BlEfiRuntimeMemory) ||
 794                  (Descriptor->Type == BlLoaderMemory)) &&
 795                 ((Descriptor->BasePage << PAGE_SHIFT) != Mmx86SelfMapBase.QuadPart))
 796             {
 797                 /* Identity-map it */
 798                 PhysicalAddress.QuadPart = Descriptor->BasePage << PAGE_SHIFT;
 799                 Status = Mmx86MapInitStructure((PVOID)((ULONG_PTR)Descriptor->BasePage << PAGE_SHIFT),
 800                                                Descriptor->PageCount << PAGE_SHIFT,
 801                                                PhysicalAddress);
 802                 if (!NT_SUCCESS(Status))
 803                 {
 804                     return Status;
 805                 }
 806             }
 807
 808             /* Move to the next descriptor */
 809             NextEntry = NextEntry->Flink;
 810         }
 811
 812         /* Reset */
 813         NextEntry = Head->Flink;
 814         while (NextEntry != Head)
 815         {
 816             /* Get the descriptor */
 817             Descriptor = CONTAINING_RECORD(NextEntry, BL_MEMORY_DESCRIPTOR, ListEntry);
 818
 819             /* Skip to the next entry before we free */
 820             NextEntry = NextEntry->Flink;
 821
 822             /* Remove and free it */
 823             MmMdRemoveDescriptorFromList(&FirmwareMdl, Descriptor);
 824             MmMdFreeDescriptor(Descriptor);
 825         }
 826     }
 827
 828     /* All library mappings identity mapped now */
 829     return STATUS_SUCCESS;
 830 }
 831
 832 NTSTATUS
 833 Mmx86InitializeMemoryMap (
 834     _In_ ULONG Phase,
 835     _In_ PBL_MEMORY_DATA MemoryData
 836     )
 837 {
 838     ULONG ImageSize;
 839     PVOID ImageBase;
 840     KDESCRIPTOR Gdt, Idt;
 841     NTSTATUS Status;
 842     PHYSICAL_ADDRESS PhysicalAddress;
 843
 844     /* If this is phase 2, map the memory regions */
 845     if (Phase != 1)
 846     {
 847         return Mmx86pMapMemoryRegions(Phase, MemoryData);
 848     }
 849
 850     /* Get the application image base/size */
 851     Status = BlGetApplicationBaseAndSize(&ImageBase, &ImageSize);
 852     if (!NT_SUCCESS(Status))
 853     {
 854         return Status;
 855     }
 856
 857     /* Map the image back at the same place */
 858     PhysicalAddress.QuadPart = (ULONG_PTR)ImageBase;
 859     Status = Mmx86MapInitStructure(ImageBase, ImageSize, PhysicalAddress);
 860     if (!NT_SUCCESS(Status))
 861     {
 862         return Status;
 863     }
 864
 865     /* Map the first 4MB of memory */
 866     PhysicalAddress.QuadPart = 0;
 867     Status = Mmx86MapInitStructure(NULL, 4 * 1024 * 1024, PhysicalAddress);
 868     if (!NT_SUCCESS(Status))
 869     {
 870         return Status;
 871     }
 872
 873     /* Map the GDT */
 874     _sgdt(&Gdt.Limit);
 875     PhysicalAddress.QuadPart = Gdt.Base;
 876     Status = Mmx86MapInitStructure((PVOID)Gdt.Base, Gdt.Limit + 1, PhysicalAddress);
 877     if (!NT_SUCCESS(Status))
 878     {
 879         return Status;
 880     }
 881
 882     /* Map the IDT */
 883     __sidt(&Idt.Limit);
 884     PhysicalAddress.QuadPart = Idt.Base;
 885     Status = Mmx86MapInitStructure((PVOID)Idt.Base, Idt.Limit + 1, PhysicalAddress);
 886     if (!NT_SUCCESS(Status))
 887     {
 888         return Status;
 889     }
 890
 891     /* Map the reference page */
 892     PhysicalAddress.QuadPart = (ULONG_PTR)MmArchReferencePage;
 893     Status = Mmx86MapInitStructure(MmArchReferencePage,
 894                                    MmArchReferencePageSize,
 895                                    PhysicalAddress);
 896     if (!NT_SUCCESS(Status))
 897     {
 898         return Status;
 899     }
 900
 901     /* More to do */
 902     return Mmx86pMapMemoryRegions(Phase, MemoryData);
 903 }
 904
 905 NTSTATUS
 906 MmDefInitializeTranslation (
 907     _In_ PBL_MEMORY_DATA MemoryData,
 908     _In_ BL_TRANSLATION_TYPE TranslationType
 909     )
 910 {
 911     NTSTATUS Status;
 912     PHYSICAL_ADDRESS PhysicalAddress;
 913     ULONG PdeIndex;
 914
 915     /* Set the global function pointers for memory translation */
 916     Mmx86TranslateVirtualAddress = MmDefpTranslateVirtualAddress;
 917     Mmx86MapPhysicalAddress = MmDefpMapPhysicalAddress;
 918     Mmx86UnmapVirtualAddress = MmDefpUnmapVirtualAddress;
 919     Mmx86RemapVirtualAddress = MmDefpRemapVirtualAddress;
 920     Mmx86FlushTlb = MmDefpFlushTlb;
 921     Mmx86FlushTlbEntry = MmDefpFlushTlbEntry;
 922     Mmx86DestroySelfMap = MmDefpDestroySelfMap;
 923
 924     /* Check what mode we're currently in */
 925     if (TranslationType == BlVirtual)
 926     {
 927         EfiPrintf(L"Virtual->Virtual not yet supported\r\n");
 928         return STATUS_NOT_IMPLEMENTED;
 929     }
 930     else if (TranslationType != BlNone)
 931     {
 932         /* Not even Windows supports PAE->Virtual downgrade */
 933         return STATUS_NOT_IMPLEMENTED;
 934     }
 935
 936     /* The None->Virtual case */
 937     MmPdpt = NULL;
 938     Mmx86SelfMapBase.QuadPart = 0;
 939     MmArchReferencePage = NULL;
 940
 941     /* Truncate all memory above 4GB so that we don't use it @TODO: FIXME */
 942     EfiPrintf(L"Warning: not truncating > 4GB memory. Don't boot with more than 4GB of RAM!\r\n");
 943     //Status = MmPaTruncateMemory(0x100000);
 944     Status = STATUS_SUCCESS;
 945     if (!NT_SUCCESS(Status))
 946     {
 947         goto Quickie;
 948     }
 949
 950     /* Allocate a page directory */
 951     Status = MmPapAllocatePhysicalPagesInRange(&PhysicalAddress,
 952                                                BlLoaderPageDirectory,
 953                                                1,
 954                                                0,
 955                                                0,
 956                                                &MmMdlUnmappedAllocated,
 957                                                0,
 958                                                0);
 959     if (!NT_SUCCESS(Status))
 960     {
 961         goto Quickie;
 962     }
 963
 964     /* Zero out the page directory */
 965     MmPdpt = (PVOID)PhysicalAddress.LowPart;
 966     RtlZeroMemory(MmPdpt, PAGE_SIZE);
 967
 968     /* Set the page size */
 969     MmArchReferencePageSize = PAGE_SIZE;
 970
 971     /* Allocate the self-map page */
 972     Status = MmPapAllocatePhysicalPagesInRange(&PhysicalAddress,
 973                                                BlLoaderReferencePage,
 974                                                1,
 975                                                0,
 976                                                0,
 977                                                &MmMdlUnmappedAllocated,
 978                                                0,
 979                                                0);
 980     if (!NT_SUCCESS(Status))
 981     {
 982         goto Quickie;
 983     }
 984
 985     /* Set the reference page */
 986     MmArchReferencePage = (PVOID)PhysicalAddress.LowPart;
 987
 988     /* Zero it out */
 989     RtlZeroMemory(MmArchReferencePage, MmArchReferencePageSize);
 990
 991     /* Allocate 4MB worth of self-map pages */
 992     Status = MmPaReserveSelfMapPages(&Mmx86SelfMapBase,
 993                                      (4 * 1024 * 1024) >> PAGE_SHIFT,
 994                                      (4 * 1024 * 1024) >> PAGE_SHIFT);
 995     if (!NT_SUCCESS(Status))
 996     {
 997         goto Quickie;
 998     }
 999
1000     /* Zero them out */
1001     RtlZeroMemory((PVOID)Mmx86SelfMapBase.LowPart, 4 * 1024 * 1024);
1002     EfiPrintf(L"PDPT at 0x%p Reference Page at 0x%p Self-map at 0x%p\r\n",
1003               MmPdpt, MmArchReferencePage, Mmx86SelfMapBase.LowPart);
1004
1005     /* Align PTE base to 4MB region */
1006     MmPteBase = (PVOID)(Mmx86SelfMapBase.LowPart & ~0x3FFFFF);
1007
1008     /* The PDE is the PTE of the PTE base */
1009     MmPdeBase = MiAddressToPte(MmPteBase);
1010     PdeIndex = MiAddressToPdeOffset(MmPdeBase);
1011     MmPdpt[PdeIndex].u.Hard.Valid = 1;
1012     MmPdpt[PdeIndex].u.Hard.Write = 1;
1013     MmPdpt[PdeIndex].u.Hard.PageFrameNumber = (ULONG_PTR)MmPdpt >> PAGE_SHIFT;
1014     MmArchReferencePage[PdeIndex]++;
1015
1016     /* Remove PTE_BASE from free virtual memory */
1017     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
1018                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
1019                                        PTE_BASE >> PAGE_SHIFT,
1020                                        (4 * 1024 * 1024) >> PAGE_SHIFT,
1021                                        0);
1022     if (!NT_SUCCESS(Status))
1023     {
1024         goto Quickie;
1025     }
1026
1027     /* Remove HAL_HEAP from free virtual memory */
1028     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
1029                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
1030                                        MM_HAL_VA_START >> PAGE_SHIFT,
1031                                        (4 * 1024 * 1024) >> PAGE_SHIFT,
1032                                        0);
1033     if (!NT_SUCCESS(Status))
1034     {
1035         goto Quickie;
1036     }
1037
1038     /* Initialize the virtual->physical memory mappings */
1039     Status = Mmx86InitializeMemoryMap(1, MemoryData);
1040     if (!NT_SUCCESS(Status))
1041     {
1042         goto Quickie;
1043     }
1044
1045     EfiPrintf(L"Ready to turn on motherfucking paging, brah!\r\n");
1046     Status = STATUS_NOT_IMPLEMENTED;
1047
1048 Quickie:
1049     /* Free reference page if we allocated it */
1050     if (MmArchReferencePage)
1051     {
1052         PhysicalAddress.QuadPart = (ULONG_PTR)MmArchReferencePage;
1053         BlMmFreePhysicalPages(PhysicalAddress);
1054     }
1055
1056     /* Free page directory if we allocated it */
1057     if (MmPdpt)
1058     {
1059         PhysicalAddress.QuadPart = (ULONG_PTR)MmPdpt;
1060         BlMmFreePhysicalPages(PhysicalAddress);
1061     }
1062
1063     /* Free the self map if we allocated it */
1064     if (Mmx86SelfMapBase.QuadPart)
1065     {
1066         MmPaReleaseSelfMapPages(Mmx86SelfMapBase);
1067     }
1068
1069     /* All done */
1070     return Status;
1071 }
1072
1073 NTSTATUS
1074 MmArchInitialize (
1075     _In_ ULONG Phase,
1076     _In_ PBL_MEMORY_DATA MemoryData,
1077     _In_ BL_TRANSLATION_TYPE TranslationType,
1078     _In_ BL_TRANSLATION_TYPE RequestedTranslationType
1079     )
1080 {
1081     NTSTATUS Status;
1082     ULONGLONG IncreaseUserVa, PerfCounter, CpuRandom;
1083     INT CpuInfo[4];
1084
1085     /* For phase 2, just map deferred regions */
1086     if (Phase != 1)
1087     {
1088         return Mmx86pMapMemoryRegions(2, MemoryData);
1089     }
1090
1091     /* What translation type are we switching to? */
1092     switch (RequestedTranslationType)
1093     {
1094         /* Physical memory */
1095         case BlNone:
1096
1097             /* Initialize everything to default/null values */
1098             MmArchLargePageSize = 1;
1099             MmArchKsegBase = 0;
1100             MmArchKsegBias = 0;
1101             MmArchKsegAddressRange.Minimum = 0;
1102             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1103             MmArchTopOfApplicationAddressSpace = 0;
1104             Mmx86SelfMapBase.QuadPart = 0;
1105
1106             /* Set stub functions */
1107             BlMmRelocateSelfMap = MmArchNullFunction;
1108             BlMmFlushTlb = MmArchNullFunction;
1109
1110             /* Set success */
1111             Status = STATUS_SUCCESS;
1112             break;
1113
1114         case BlVirtual:
1115
1116             /* Set the large page size to 1024 pages (4MB) */
1117             MmArchLargePageSize = (4 * 1024 * 1024) / PAGE_SIZE;
1118
1119             /* Check if /USERVA option was used */
1120             Status = BlGetBootOptionInteger(BlpApplicationEntry.BcdData,
1121                                             BcdOSLoaderInteger_IncreaseUserVa,
1122                                             &IncreaseUserVa);
1123             if (NT_SUCCESS(Status) && (IncreaseUserVa))
1124             {
1125                 /* Yes -- load the kernel at 0xE0000000 instead */
1126                 MmArchKsegBase = 0xE0000000;
1127             }
1128             else
1129             {
1130                 /* Nope, load at the standard 2GB split */
1131                 MmArchKsegBase = 0x80000000;
1132             }
1133
1134             /* Check if CPUID 01h is supported */
1135             CpuRandom = 0;
1136             if (BlArchIsCpuIdFunctionSupported(1))
1137             {
1138                 /* Call it */
1139                 BlArchCpuId(1, 0, CpuInfo);
1140
1141                 /* Check if RDRAND is supported */
1142                 if (CpuInfo[2] & 0x40000000)
1143                 {
1144                     EfiPrintf(L"Your CPU can do RDRAND! Good for you!\r\n");
1145                     CpuRandom = 0;
1146                 }
1147             }
1148
1149             /* Read the TSC */
1150             PerfCounter = BlArchGetPerformanceCounter();
1151             PerfCounter >>= 4;
1152             _rotl16(PerfCounter, 5);
1153
1154             /* Set the address range */
1155             MmArchKsegAddressRange.Minimum = 0;
1156             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1157
1158             /* Set the KASLR bias */
1159             MmArchKsegBias = ((PerfCounter ^ CpuRandom) & 0xFFF) << 12;
1160             MmArchKsegBias = 0;
1161             MmArchKsegBase += MmArchKsegBias;
1162
1163             /* Set the kernel range */
1164             MmArchKsegAddressRange.Minimum = MmArchKsegBase;
1165             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1166
1167             /* Set the boot application top maximum */
1168             MmArchTopOfApplicationAddressSpace = 0x70000000 - 1; // Windows bug
1169
1170             /* Initialize virtual address space translation */
1171             Status = MmDefInitializeTranslation(MemoryData, TranslationType);
1172             if (NT_SUCCESS(Status))
1173             {
1174                 /* Set stub functions */
1175                 BlMmRelocateSelfMap = MmDefRelocateSelfMap;
1176                 BlMmFlushTlb = Mmx86FlushTlb;
1177                 BlMmMoveVirtualAddressRange = MmDefMoveVirtualAddressRange;
1178                 BlMmZeroVirtualAddressRange = MmDefZeroVirtualAddressRange;
1179             }
1180             break;
1181
1182         case BlPae:
1183
1184             /* We don't support PAE */
1185             Status = STATUS_NOT_SUPPORTED;
1186             break;
1187
1188         default:
1189
1190             /* Invalid architecture type*/
1191             Status = STATUS_INVALID_PARAMETER;
1192             break;
1193     }
1194
1195     /* Back to caller */
1196     return Status;
1197 }