reactos/boot/environ/lib/mm/i386/mmx86.c

   1 /*
   2 * COPYRIGHT:       See COPYING.ARM in the top level directory
   3 * PROJECT:         ReactOS UEFI Boot Library
   4 * FILE:            boot/environ/lib/mm/i386/mmx86.c
   5 * PURPOSE:         Boot Library Memory Manager x86-Specific Code
   6 * PROGRAMMER:      Alex Ionescu (alex.ionescu@reactos.org)
   7 */
   8
   9 /* INCLUDES ******************************************************************/
  10
  11 #include "bl.h"
  12 #include "bcd.h"
  13
  14 #define PTE_BASE                0xC0000000
  15
  16 //
  17 // Specific PDE/PTE macros to be used inside the boot library environment
  18 //
  19 #define MiAddressToPte(x)       ((PMMPTE)(((((ULONG)(x)) >> 12) << 2) + (ULONG_PTR)MmPteBase))
  20 #define MiAddressToPde(x)       ((PMMPDE)(((((ULONG)(x)) >> 22) << 2) + (ULONG_PTR)MmPdeBase))
  21 #define MiAddressToPteOffset(x) ((((ULONG)(x)) << 10) >> 22)
  22 #define MiAddressToPdeOffset(x) (((ULONG)(x)) / (1024 * PAGE_SIZE))
  23
  24 /* DATA VARIABLES ************************************************************/
  25
  26 ULONG_PTR MmArchKsegBase;
  27 ULONG_PTR MmArchKsegBias;
  28 ULONG MmArchLargePageSize;
  29 BL_ADDRESS_RANGE MmArchKsegAddressRange;
  30 ULONG_PTR MmArchTopOfApplicationAddressSpace;
  31 PHYSICAL_ADDRESS Mmx86SelfMapBase;
  32 ULONG MmDeferredMappingCount;
  33 PMMPTE MmPdpt;
  34 PULONG MmArchReferencePage;
  35 PVOID MmPteBase;
  36 PVOID MmPdeBase;
  37 ULONG MmArchReferencePageSize;
  38
  39 typedef VOID
  40 (*PBL_MM_FLUSH_TLB) (
  41     VOID
  42     );
  43
  44 typedef VOID
  45 (*PBL_MM_RELOCATE_SELF_MAP) (
  46     VOID
  47     );
  48
  49 typedef NTSTATUS
  50 (*PBL_MM_MOVE_VIRTUAL_ADDRESS_RANGE) (
  51     _In_ PVOID DestinationAddress,
  52     _In_ PVOID SourceAddress,
  53     _In_ ULONGLONG Size
  54     );
  55
  56 typedef NTSTATUS
  57 (*PBL_MM_ZERO_VIRTUAL_ADDRESS_RANGE) (
  58     _In_ PVOID DestinationAddress,
  59     _In_ ULONGLONG Size
  60     );
  61
  62 typedef VOID
  63 (*PBL_MM_DESTROY_SELF_MAP) (
  64     VOID
  65     );
  66
  67 typedef VOID
  68 (*PBL_MM_FLUSH_TLB_ENTRY) (
  69     _In_ PVOID VirtualAddress
  70     );
  71
  72 typedef VOID
  73 (*PBL_MM_FLUSH_TLB) (
  74     VOID
  75     );
  76
  77 typedef NTSTATUS
  78 (*PBL_MM_UNMAP_VIRTUAL_ADDRESS) (
  79     _In_ PVOID VirtualAddress,
  80     _In_ ULONG Size
  81     );
  82
  83 typedef NTSTATUS
  84 (*PBL_MM_REMAP_VIRTUAL_ADDRESS) (
  85     _In_ PPHYSICAL_ADDRESS PhysicalAddress,
  86     _Out_ PVOID VirtualAddress,
  87     _In_ ULONG Size,
  88     _In_ ULONG CacheAttributes
  89     );
  90
  91 typedef NTSTATUS
  92 (*PBL_MM_MAP_PHYSICAL_ADDRESS) (
  93     _In_ PHYSICAL_ADDRESS PhysicalAddress,
  94     _Out_ PVOID VirtualAddress,
  95     _In_ ULONG Size,
  96     _In_ ULONG CacheAttributes
  97     );
  98
  99 typedef BOOLEAN
 100 (*PBL_MM_TRANSLATE_VIRTUAL_ADDRESS) (
 101     _In_ PVOID VirtualAddress,
 102     _Out_ PPHYSICAL_ADDRESS PhysicalAddress,
 103     _Out_opt_ PULONG CacheAttributes
 104     );
 105
 106 PBL_MM_TRANSLATE_VIRTUAL_ADDRESS Mmx86TranslateVirtualAddress;
 107 PBL_MM_MAP_PHYSICAL_ADDRESS Mmx86MapPhysicalAddress;
 108 PBL_MM_REMAP_VIRTUAL_ADDRESS Mmx86RemapVirtualAddress;
 109 PBL_MM_UNMAP_VIRTUAL_ADDRESS Mmx86UnmapVirtualAddress;
 110 PBL_MM_FLUSH_TLB Mmx86FlushTlb;
 111 PBL_MM_FLUSH_TLB_ENTRY Mmx86FlushTlbEntry;
 112 PBL_MM_DESTROY_SELF_MAP Mmx86DestroySelfMap;
 113
 114 PBL_MM_RELOCATE_SELF_MAP BlMmRelocateSelfMap;
 115 PBL_MM_FLUSH_TLB BlMmFlushTlb;
 116 PBL_MM_MOVE_VIRTUAL_ADDRESS_RANGE BlMmMoveVirtualAddressRange;
 117 PBL_MM_ZERO_VIRTUAL_ADDRESS_RANGE BlMmZeroVirtualAddressRange;
 118
 119 PBL_MM_FLUSH_TLB Mmx86FlushTlb;
 120
 121 /* FUNCTIONS *****************************************************************/
 122
 123 BOOLEAN
 124 BlMmIsTranslationEnabled (
 125     VOID
 126     )
 127 {
 128     /* Return if paging is on */
 129     return ((CurrentExecutionContext) &&
 130             (CurrentExecutionContext->Mode & BL_CONTEXT_PAGING_ON));
 131 }
 132
 133 VOID
 134 MmArchNullFunction (
 135     VOID
 136     )
 137 {
 138     /* Nothing to do */
 139     return;
 140 }
 141
 142 VOID
 143 MmDefRelocateSelfMap (
 144     VOID
 145     )
 146 {
 147     if (MmPteBase != (PVOID)PTE_BASE)
 148     {
 149         EfiPrintf(L"Supposed to relocate CR3\r\n");
 150     }
 151 }
 152
 153 NTSTATUS
 154 MmDefMoveVirtualAddressRange (
 155     _In_ PVOID DestinationAddress,
 156     _In_ PVOID SourceAddress,
 157     _In_ ULONGLONG Size
 158     )
 159 {
 160     EfiPrintf(L"Supposed to move shit\r\n");
 161     return STATUS_NOT_IMPLEMENTED;
 162 }
 163
 164 NTSTATUS
 165 MmDefZeroVirtualAddressRange (
 166     _In_ PVOID DestinationAddress,
 167     _In_ ULONGLONG Size
 168     )
 169 {
 170     EfiPrintf(L"Supposed to zero shit\r\n");
 171     return STATUS_NOT_IMPLEMENTED;
 172 }
 173
 174 BOOLEAN
 175 MmArchTranslateVirtualAddress (
 176     _In_ PVOID VirtualAddress,
 177     _Out_opt_ PPHYSICAL_ADDRESS PhysicalAddress,
 178     _Out_opt_ PULONG CachingFlags
 179     )
 180 {
 181     PBL_MEMORY_DESCRIPTOR Descriptor;
 182
 183     /* Check if paging is on */
 184     if ((CurrentExecutionContext) &&
 185         (CurrentExecutionContext->ContextFlags & BL_CONTEXT_PAGING_ON))
 186     {
 187         /* Yes -- we have to translate this from virtual */
 188         return Mmx86TranslateVirtualAddress(VirtualAddress,
 189                                             PhysicalAddress,
 190                                             CachingFlags);
 191     }
 192
 193     /* Look in all descriptors except truncated and firmware ones */
 194     Descriptor = MmMdFindDescriptor(BL_MM_INCLUDE_NO_FIRMWARE_MEMORY &
 195                                     ~BL_MM_INCLUDE_TRUNCATED_MEMORY,
 196                                     BL_MM_REMOVE_PHYSICAL_REGION_FLAG,
 197                                     (ULONG_PTR)VirtualAddress >> PAGE_SHIFT);
 198
 199     /* Return the virtual address as the physical address */
 200     if (PhysicalAddress)
 201     {
 202         PhysicalAddress->HighPart = 0;
 203         PhysicalAddress->LowPart = (ULONG_PTR)VirtualAddress;
 204     }
 205
 206     /* There's no caching on physical memory */
 207     if (CachingFlags)
 208     {
 209         *CachingFlags = 0;
 210     }
 211
 212     /* Success is if we found a descriptor */
 213     return Descriptor != NULL;
 214 }
 215
 216 VOID
 217 MmDefpDestroySelfMap (
 218     VOID
 219     )
 220 {
 221     EfiPrintf(L"No destroy\r\n");
 222 }
 223
 224 VOID
 225 MmDefpFlushTlbEntry (
 226     _In_ PVOID VirtualAddress
 227     )
 228 {
 229     /* Flush the TLB */
 230     __invlpg(VirtualAddress);
 231 }
 232
 233 VOID
 234 MmDefpFlushTlb (
 235     VOID
 236     )
 237 {
 238     /* Flush the TLB */
 239     __writecr3(__readcr3());
 240 }
 241
 242 NTSTATUS
 243 MmDefpUnmapVirtualAddress (
 244     _In_ PVOID VirtualAddress,
 245     _In_ ULONG Size
 246     )
 247 {
 248     EfiPrintf(L"No unmap\r\n");
 249     return STATUS_NOT_IMPLEMENTED;
 250 }
 251
 252 NTSTATUS
 253 MmDefpRemapVirtualAddress (
 254     _In_ PPHYSICAL_ADDRESS PhysicalAddress,
 255     _Out_ PVOID VirtualAddress,
 256     _In_ ULONG Size,
 257     _In_ ULONG CacheAttributes
 258     )
 259 {
 260     EfiPrintf(L"No remap\r\n");
 261     return STATUS_NOT_IMPLEMENTED;
 262 }
 263
 264 NTSTATUS
 265 MmDefpMapPhysicalAddress (
 266     _In_ PHYSICAL_ADDRESS PhysicalAddress,
 267     _In_ PVOID VirtualAddress,
 268     _In_ ULONG Size,
 269     _In_ ULONG CacheAttributes
 270     )
 271 {
 272     BOOLEAN Enabled;
 273     ULONG i, PageCount, PdeOffset;
 274     ULONGLONG CurrentAddress;
 275     PMMPDE Pde;
 276     PMMPTE Pte;
 277     PMMPTE PageTable;
 278     PHYSICAL_ADDRESS PageTableAddress;
 279     NTSTATUS Status;
 280
 281     /* Check if paging is on yet */
 282     Enabled = BlMmIsTranslationEnabled();
 283
 284     /* Get the physical address aligned */
 285     CurrentAddress = (PhysicalAddress.QuadPart >> PAGE_SHIFT) << PAGE_SHIFT;
 286
 287     /* Get the number of pages and loop through each one */
 288     PageCount = Size >> PAGE_SHIFT;
 289     for (i = 0; i < PageCount; i++)
 290     {
 291         /* Check if translation already exists for this page */
 292         if (Mmx86TranslateVirtualAddress(VirtualAddress, NULL, NULL))
 293         {
 294             /* Ignore it and move to the next one */
 295             VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress + PAGE_SIZE);
 296             CurrentAddress += PAGE_SIZE;
 297             continue;
 298         }
 299
 300         /* Get the PDE offset */
 301         PdeOffset = MiAddressToPdeOffset(VirtualAddress);
 302
 303         /* Check if paging is actually turned on */
 304         if (Enabled)
 305         {
 306             /* Get the PDE entry using the self-map */
 307             Pde = MiAddressToPde(VirtualAddress);
 308         }
 309         else
 310         {
 311             /* Get it using our physical mappings */
 312             Pde = &MmPdpt[PdeOffset];
 313             PageTable = (PMMPDE)(Pde->u.Hard.PageFrameNumber << PAGE_SHIFT);
 314         }
 315
 316         /* Check if we don't yet have a PDE */
 317         if (!Pde->u.Hard.Valid)
 318         {
 319             /* Allocate a page table */
 320             Status = MmPapAllocatePhysicalPagesInRange(&PageTableAddress,
 321                                                        BlLoaderPageDirectory,
 322                                                        1,
 323                                                        0,
 324                                                        0,
 325                                                        &MmMdlUnmappedAllocated,
 326                                                        0,
 327                                                        0);
 328             if (!NT_SUCCESS(Status))
 329             {
 330                 return STATUS_NO_MEMORY;
 331             }
 332
 333             /* This is our page table */
 334             PageTable = (PVOID)(ULONG_PTR)PageTableAddress.QuadPart;
 335
 336             /* Build the PDE for it */
 337             Pde->u.Hard.PageFrameNumber = PageTableAddress.QuadPart >> PAGE_SHIFT;
 338             Pde->u.Hard.Write = 1;
 339             Pde->u.Hard.CacheDisable = 1;
 340             Pde->u.Hard.WriteThrough = 1;
 341             Pde->u.Hard.Valid = 1;
 342
 343             /* Check if paging is enabled */
 344             if (Enabled)
 345             {
 346                 /* Then actually, get the page table's virtual address */
 347                 PageTable = (PVOID)PAGE_ROUND_DOWN(MiAddressToPte(VirtualAddress));
 348
 349                 /* Flush the TLB */
 350                 Mmx86FlushTlb();
 351             }
 352
 353             /* Zero out the page table */
 354             RtlZeroMemory(PageTable, PAGE_SIZE);
 355
 356             /* Reset caching attributes now */
 357             Pde->u.Hard.CacheDisable = 0;
 358             Pde->u.Hard.WriteThrough = 0;
 359
 360             /* Check for paging again */
 361             if (Enabled)
 362             {
 363                 /* Flush the TLB entry for the page table only */
 364                 Mmx86FlushTlbEntry(PageTable);
 365             }
 366         }
 367
 368         /* Add a reference to this page table */
 369         MmArchReferencePage[PdeOffset]++;
 370
 371         /* Check if a physical address was given */
 372         if (PhysicalAddress.QuadPart != -1)
 373         {
 374             /* Check if paging is turned on */
 375             if (Enabled)
 376             {
 377                 /* Get the PTE using the self-map */
 378                 Pte = MiAddressToPte(VirtualAddress);
 379             }
 380             else
 381             {
 382                 /* Get the PTE using physical addressing */
 383                 Pte = &PageTable[MiAddressToPteOffset(VirtualAddress)];
 384             }
 385
 386             /* Build a valid PTE for it */
 387             Pte->u.Hard.PageFrameNumber = CurrentAddress >> PAGE_SHIFT;
 388             Pte->u.Hard.Write = 1;
 389             Pte->u.Hard.Valid = 1;
 390
 391             /* Check if this is uncached */
 392             if (CacheAttributes == BlMemoryUncached)
 393             {
 394                 /* Set the flags */
 395                 Pte->u.Hard.CacheDisable = 1;
 396                 Pte->u.Hard.WriteThrough = 1;
 397             }
 398             else if (CacheAttributes == BlMemoryWriteThrough)
 399             {
 400                 /* It's write-through, set the flag */
 401                 Pte->u.Hard.WriteThrough = 1;
 402             }
 403         }
 404
 405         /* Move to the next physical/virtual address */
 406         VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress + PAGE_SIZE);
 407         CurrentAddress += PAGE_SIZE;
 408     }
 409
 410     /* All done! */
 411     return STATUS_SUCCESS;
 412 }
 413
 414 BOOLEAN
 415 MmDefpTranslateVirtualAddress (
 416     _In_ PVOID VirtualAddress,
 417     _Out_ PPHYSICAL_ADDRESS PhysicalAddress,
 418     _Out_opt_ PULONG CacheAttributes
 419     )
 420 {
 421     PMMPDE Pde;
 422     PMMPTE Pte;
 423     PMMPTE PageTable;
 424     BOOLEAN Enabled;
 425
 426     /* Is there no page directory yet? */
 427     if (!MmPdpt)
 428     {
 429         return FALSE;
 430     }
 431
 432     /* Is paging enabled? */
 433     Enabled = BlMmIsTranslationEnabled();
 434
 435     /* Check if paging is actually turned on */
 436     if (Enabled)
 437     {
 438         /* Get the PDE entry using the self-map */
 439         Pde = MiAddressToPde(VirtualAddress);
 440     }
 441     else
 442     {
 443         /* Get it using our physical mappings */
 444         Pde = &MmPdpt[MiAddressToPdeOffset(VirtualAddress)];
 445     }
 446
 447     /* Is the PDE valid? */
 448     if (!Pde->u.Hard.Valid)
 449     {
 450         return FALSE;
 451     }
 452
 453     /* Check if paging is turned on */
 454     if (Enabled)
 455     {
 456         /* Get the PTE using the self-map */
 457         Pte = MiAddressToPte(VirtualAddress);
 458     }
 459     else
 460     {
 461         /* Get the PTE using physical addressing */
 462         PageTable = (PMMPTE)(Pde->u.Hard.PageFrameNumber << PAGE_SHIFT);
 463         Pte = &PageTable[MiAddressToPteOffset(VirtualAddress)];
 464     }
 465
 466     /* Is the PTE valid? */
 467     if (!Pte->u.Hard.Valid)
 468     {
 469         return FALSE;
 470     }
 471
 472     /* Does caller want the physical address?  */
 473     if (PhysicalAddress)
 474     {
 475         /* Return it */
 476         PhysicalAddress->QuadPart = (Pte->u.Hard.PageFrameNumber << PAGE_SHIFT) +
 477                                      BYTE_OFFSET(VirtualAddress);
 478     }
 479
 480     /* Does caller want cache attributes? */
 481     if (CacheAttributes)
 482     {
 483         /* Not yet -- lie and say it's cached */
 484         EfiPrintf(L"Cache checking not yet enabled\r\n");
 485         *CacheAttributes = BlMemoryWriteBack;
 486     }
 487
 488     /* It exists! */
 489     return TRUE;
 490 }
 491
 492 NTSTATUS
 493 MmSelectMappingAddress (
 494     _Out_ PVOID* MappingAddress,
 495     _In_ ULONGLONG Size,
 496     _In_ ULONG AllocationAttributes,
 497     _In_ ULONG Flags,
 498     _In_ PHYSICAL_ADDRESS PhysicalAddress
 499     )
 500 {
 501     /* Are we in physical mode? */
 502     if (MmTranslationType == BlNone)
 503     {
 504         /* Just return the physical address as the mapping address */
 505         *MappingAddress = (PVOID)PhysicalAddress.LowPart;
 506         return STATUS_SUCCESS;
 507     }
 508
 509     /* We don't support virtual memory yet @TODO */
 510 #ifdef _MSC_VER // Fuck gcc.
 511     EfiPrintf(L"not yet implemented in " __FUNCTION__ "\r\n");
 512     EfiStall(1000000);
 513 #endif
 514     return STATUS_NOT_IMPLEMENTED;
 515 }
 516
 517 NTSTATUS
 518 MmMapPhysicalAddress (
 519     _Inout_ PPHYSICAL_ADDRESS PhysicalAddressPtr,
 520     _Inout_ PVOID* VirtualAddressPtr,
 521     _Inout_ PULONGLONG SizePtr,
 522     _In_ ULONG CacheAttributes
 523     )
 524 {
 525     ULONGLONG Size, TotalSize;
 526     ULONGLONG PhysicalAddress;
 527     PVOID VirtualAddress;
 528     PHYSICAL_ADDRESS TranslatedAddress;
 529     ULONG_PTR CurrentAddress, VirtualAddressEnd;
 530     NTSTATUS Status;
 531
 532     /* Fail if any parameters are missing */
 533     if (!(PhysicalAddressPtr) || !(VirtualAddressPtr) || !(SizePtr))
 534     {
 535         return STATUS_INVALID_PARAMETER;
 536     }
 537
 538     /* Fail if the size is over 32-bits */
 539     Size = *SizePtr;
 540     if (Size > 0xFFFFFFFF)
 541     {
 542         return STATUS_INVALID_PARAMETER;
 543     }
 544
 545     /* Nothing to do if we're in physical mode */
 546     if (MmTranslationType == BlNone)
 547     {
 548         return STATUS_SUCCESS;
 549     }
 550
 551     /* Can't use virtual memory in real mode */
 552     if (CurrentExecutionContext->Mode == BlRealMode)
 553     {
 554         return STATUS_UNSUCCESSFUL;
 555     }
 556
 557     /* Capture the current virtual and physical addresses */
 558     VirtualAddress = *VirtualAddressPtr;
 559     PhysicalAddress = PhysicalAddressPtr->QuadPart;
 560
 561     /* Check if a physical address was requested */
 562     if (PhysicalAddress != 0xFFFFFFFF)
 563     {
 564         /* Round down the base addresses */
 565         PhysicalAddress = PAGE_ROUND_DOWN(PhysicalAddress);
 566         VirtualAddress = (PVOID)PAGE_ROUND_DOWN(VirtualAddress);
 567
 568         /* Round up the size */
 569         TotalSize = ROUND_TO_PAGES(PhysicalAddressPtr->QuadPart -
 570                                    PhysicalAddress +
 571                                    Size);
 572
 573         /* Loop every virtual page */
 574         CurrentAddress = (ULONG_PTR)VirtualAddress;
 575         VirtualAddressEnd = CurrentAddress + TotalSize - 1;
 576         while (CurrentAddress < VirtualAddressEnd)
 577         {
 578             /* Get the physical page of this virtual page */
 579             if (MmArchTranslateVirtualAddress((PVOID)CurrentAddress,
 580                                               &TranslatedAddress,
 581                                               &CacheAttributes))
 582             {
 583                 /* Make sure the physical page of the virtual page, matches our page */
 584                 if (TranslatedAddress.QuadPart !=
 585                     (PhysicalAddress +
 586                      (CurrentAddress - (ULONG_PTR)VirtualAddress)))
 587                 {
 588                     /* There is an existing virtual mapping for a different address */
 589                     EfiPrintf(L"Existing mapping exists: %lx vs %lx\r\n",
 590                               TranslatedAddress.QuadPart,
 591                               PhysicalAddress + (CurrentAddress - (ULONG_PTR)VirtualAddress));
 592                     return STATUS_INVALID_PARAMETER;
 593                 }
 594             }
 595
 596             /* Try the next one */
 597             CurrentAddress += PAGE_SIZE;
 598         }
 599     }
 600
 601     /* Aactually do the mapping */
 602     TranslatedAddress.QuadPart = PhysicalAddress;
 603     Status = Mmx86MapPhysicalAddress(TranslatedAddress,
 604                                      VirtualAddress,
 605                                      Size,
 606                                      CacheAttributes);
 607     if (!NT_SUCCESS(Status))
 608     {
 609         EfiPrintf(L"Failed to map!: %lx\r\n", Status);
 610         return Status;
 611     }
 612
 613     /* Return aligned/fixed up output parameters */
 614     PhysicalAddressPtr->QuadPart = PhysicalAddress;
 615     *VirtualAddressPtr = VirtualAddress;
 616     *SizePtr = Size;
 617
 618     /* Flush the TLB if paging is enabled */
 619     if (BlMmIsTranslationEnabled())
 620     {
 621         Mmx86FlushTlb();
 622     }
 623
 624     /* All good! */
 625     return STATUS_SUCCESS;
 626 }
 627
 628 NTSTATUS
 629 Mmx86MapInitStructure (
 630     _In_ PVOID VirtualAddress,
 631     _In_ ULONGLONG Size,
 632     _In_ PHYSICAL_ADDRESS PhysicalAddress
 633     )
 634 {
 635     NTSTATUS Status;
 636
 637     /* Make a virtual mapping for this physical address */
 638     Status = MmMapPhysicalAddress(&PhysicalAddress, &VirtualAddress, &Size, 0);
 639     if (!NT_SUCCESS(Status))
 640     {
 641         return Status;
 642     }
 643
 644     /* Nothing else to do if we're not in paging mode */
 645     if (MmTranslationType == BlNone)
 646     {
 647         return STATUS_SUCCESS;
 648     }
 649
 650     /* Otherwise, remove this region from the list of free virtual ranges */
 651     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
 652                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
 653                                        (ULONG_PTR)VirtualAddress >> PAGE_SHIFT,
 654                                        Size >> PAGE_SHIFT,
 655                                        0);
 656     if (!NT_SUCCESS(Status))
 657     {
 658         /* Unmap the address if that failed */
 659         MmUnmapVirtualAddress(&VirtualAddress, &Size);
 660     }
 661
 662     /* Return back to caller */
 663     return Status;
 664 }
 665
 666 NTSTATUS
 667 Mmx86pMapMemoryRegions (
 668     _In_ ULONG Phase,
 669     _In_ PBL_MEMORY_DATA MemoryData
 670     )
 671 {
 672     BOOLEAN DoDeferred;
 673     ULONG DescriptorCount;
 674     PBL_MEMORY_DESCRIPTOR Descriptor;
 675     ULONG FinalOffset;
 676     PHYSICAL_ADDRESS PhysicalAddress;
 677     ULONGLONG Size;
 678     NTSTATUS Status;
 679     PVOID VirtualAddress;
 680     BL_MEMORY_DESCRIPTOR_LIST FirmwareMdl;
 681     PLIST_ENTRY Head, NextEntry;
 682
 683     /* In phase 1 we don't initialize deferred mappings*/
 684     if (Phase == 1)
 685     {
 686         DoDeferred = FALSE;
 687     }
 688     else
 689     {
 690         /* Don't do anything if there's nothing to initialize */
 691         if (!MmDeferredMappingCount)
 692         {
 693             return STATUS_SUCCESS;
 694         }
 695
 696         DoDeferred = TRUE;
 697     }
 698
 699     /*
 700     * Because BL supports cross x86-x64 application launches and a LIST_ENTRY
 701     * is of variable size, care must be taken here to ensure that we see a
 702     * consistent view of descriptors. BL uses some offset magic to figure out
 703     * where the data actually starts, since everything is ULONGLONG past the
 704     * LIST_ENTRY itself
 705     */
 706     FinalOffset = MemoryData->MdListOffset + MemoryData->DescriptorOffset;
 707     Descriptor = (PBL_MEMORY_DESCRIPTOR)((ULONG_PTR)MemoryData + FinalOffset -
 708                                          FIELD_OFFSET(BL_MEMORY_DESCRIPTOR, BasePage));
 709
 710     /* Scan all of them */
 711     DescriptorCount = MemoryData->DescriptorCount;
 712     while (DescriptorCount != 0)
 713     {
 714         /* Ignore application data */
 715         if (Descriptor->Type != BlApplicationData)
 716         {
 717             /* If this is a ramdisk, do it in phase 2 */
 718             if ((Descriptor->Type == BlLoaderRamDisk) == DoDeferred)
 719             {
 720                 /* Get the current physical address and size */
 721                 PhysicalAddress.QuadPart = Descriptor->BasePage << PAGE_SHIFT;
 722                 Size = Descriptor->PageCount << PAGE_SHIFT;
 723
 724                 /* Check if it was already mapped */
 725                 if (Descriptor->VirtualPage)
 726                 {
 727                     /* Use the existing address */
 728                     VirtualAddress = (PVOID)(ULONG_PTR)(Descriptor->VirtualPage << PAGE_SHIFT);
 729                 }
 730                 else
 731                 {
 732                     /* Use the physical address */
 733                     VirtualAddress = (PVOID)(ULONG_PTR)PhysicalAddress.QuadPart;
 734                 }
 735
 736                 /* Crete the mapping */
 737                 Status = Mmx86MapInitStructure(VirtualAddress,
 738                                                Size,
 739                                                PhysicalAddress);
 740                 if (!NT_SUCCESS(Status))
 741                 {
 742                     return Status;
 743                 }
 744             }
 745
 746             /* Check if we're in phase 1 and deferring RAM disk */
 747             if ((Phase == 1) && (Descriptor->Type == BlLoaderRamDisk))
 748             {
 749                 MmDeferredMappingCount++;
 750             }
 751         }
 752
 753         /* Move on to the next descriptor */
 754         DescriptorCount--;
 755         Descriptor = (PBL_MEMORY_DESCRIPTOR)((ULONG_PTR)Descriptor + MemoryData->DescriptorSize);
 756     }
 757
 758     /* In phase 1, also do UEFI mappings */
 759     if (Phase != 2)
 760     {
 761         /* Get the memory map */
 762         MmMdInitializeListHead(&FirmwareMdl);
 763         Status = MmFwGetMemoryMap(&FirmwareMdl, BL_MM_FLAG_REQUEST_COALESCING);
 764         if (!NT_SUCCESS(Status))
 765         {
 766             return Status;
 767         }
 768
 769         /* Iterate over it */
 770         Head = FirmwareMdl.First;
 771         NextEntry = Head->Flink;
 772         while (NextEntry != Head)
 773         {
 774             /* Check if this is a UEFI-related descriptor, unless it's the self-map page */
 775             Descriptor = CONTAINING_RECORD(NextEntry, BL_MEMORY_DESCRIPTOR, ListEntry);
 776             if (((Descriptor->Type == BlEfiBootMemory) ||
 777                  (Descriptor->Type == BlEfiRuntimeMemory) ||
 778                  (Descriptor->Type == BlLoaderMemory)) &&
 779                 ((Descriptor->BasePage << PAGE_SHIFT) != Mmx86SelfMapBase.QuadPart))
 780             {
 781                 /* Identity-map it */
 782                 PhysicalAddress.QuadPart = Descriptor->BasePage << PAGE_SHIFT;
 783                 Status = Mmx86MapInitStructure((PVOID)((ULONG_PTR)Descriptor->BasePage << PAGE_SHIFT),
 784                                                Descriptor->PageCount << PAGE_SHIFT,
 785                                                PhysicalAddress);
 786                 if (!NT_SUCCESS(Status))
 787                 {
 788                     return Status;
 789                 }
 790             }
 791
 792             /* Move to the next descriptor */
 793             NextEntry = NextEntry->Flink;
 794         }
 795
 796         /* Reset */
 797         NextEntry = Head->Flink;
 798         while (NextEntry != Head)
 799         {
 800             /* Get the descriptor */
 801             Descriptor = CONTAINING_RECORD(NextEntry, BL_MEMORY_DESCRIPTOR, ListEntry);
 802
 803             /* Skip to the next entry before we free */
 804             NextEntry = NextEntry->Flink;
 805
 806             /* Remove and free it */
 807             MmMdRemoveDescriptorFromList(&FirmwareMdl, Descriptor);
 808             MmMdFreeDescriptor(Descriptor);
 809         }
 810     }
 811
 812     /* All library mappings identity mapped now */
 813     return STATUS_SUCCESS;
 814 }
 815
 816 NTSTATUS
 817 Mmx86InitializeMemoryMap (
 818     _In_ ULONG Phase,
 819     _In_ PBL_MEMORY_DATA MemoryData
 820     )
 821 {
 822     ULONG ImageSize;
 823     PVOID ImageBase;
 824     KDESCRIPTOR Gdt, Idt;
 825     NTSTATUS Status;
 826     PHYSICAL_ADDRESS PhysicalAddress;
 827
 828     /* If this is phase 2, map the memory regions */
 829     if (Phase != 1)
 830     {
 831         return Mmx86pMapMemoryRegions(Phase, MemoryData);
 832     }
 833
 834     /* Get the application image base/size */
 835     Status = BlGetApplicationBaseAndSize(&ImageBase, &ImageSize);
 836     if (!NT_SUCCESS(Status))
 837     {
 838         return Status;
 839     }
 840
 841     /* Map the image back at the same place */
 842     PhysicalAddress.QuadPart = (ULONG_PTR)ImageBase;
 843     Status = Mmx86MapInitStructure(ImageBase, ImageSize, PhysicalAddress);
 844     if (!NT_SUCCESS(Status))
 845     {
 846         return Status;
 847     }
 848
 849     /* Map the first 4MB of memory */
 850     PhysicalAddress.QuadPart = 0;
 851     Status = Mmx86MapInitStructure(NULL, 4 * 1024 * 1024, PhysicalAddress);
 852     if (!NT_SUCCESS(Status))
 853     {
 854         return Status;
 855     }
 856
 857     /* Map the GDT */
 858     _sgdt(&Gdt.Limit);
 859     PhysicalAddress.QuadPart = Gdt.Base;
 860     Status = Mmx86MapInitStructure((PVOID)Gdt.Base, Gdt.Limit + 1, PhysicalAddress);
 861     if (!NT_SUCCESS(Status))
 862     {
 863         return Status;
 864     }
 865
 866     /* Map the IDT */
 867     __sidt(&Idt.Limit);
 868     PhysicalAddress.QuadPart = Idt.Base;
 869     Status = Mmx86MapInitStructure((PVOID)Idt.Base, Idt.Limit + 1, PhysicalAddress);
 870     if (!NT_SUCCESS(Status))
 871     {
 872         return Status;
 873     }
 874
 875     /* Map the reference page */
 876     PhysicalAddress.QuadPart = (ULONG_PTR)MmArchReferencePage;
 877     Status = Mmx86MapInitStructure(MmArchReferencePage,
 878                                    MmArchReferencePageSize,
 879                                    PhysicalAddress);
 880     if (!NT_SUCCESS(Status))
 881     {
 882         return Status;
 883     }
 884
 885     /* More to do */
 886     return Mmx86pMapMemoryRegions(Phase, MemoryData);
 887 }
 888
 889 NTSTATUS
 890 MmDefInitializeTranslation (
 891     _In_ PBL_MEMORY_DATA MemoryData,
 892     _In_ BL_TRANSLATION_TYPE TranslationType
 893     )
 894 {
 895     NTSTATUS Status;
 896     PHYSICAL_ADDRESS PhysicalAddress;
 897     ULONG PdeIndex;
 898
 899     /* Set the global function pointers for memory translation */
 900     Mmx86TranslateVirtualAddress = MmDefpTranslateVirtualAddress;
 901     Mmx86MapPhysicalAddress = MmDefpMapPhysicalAddress;
 902     Mmx86UnmapVirtualAddress = MmDefpUnmapVirtualAddress;
 903     Mmx86RemapVirtualAddress = MmDefpRemapVirtualAddress;
 904     Mmx86FlushTlb = MmDefpFlushTlb;
 905     Mmx86FlushTlbEntry = MmDefpFlushTlbEntry;
 906     Mmx86DestroySelfMap = MmDefpDestroySelfMap;
 907
 908     /* Check what mode we're currently in */
 909     if (TranslationType == BlVirtual)
 910     {
 911         EfiPrintf(L"Virtual->Virtual not yet supported\r\n");
 912         return STATUS_NOT_IMPLEMENTED;
 913     }
 914     else if (TranslationType != BlNone)
 915     {
 916         /* Not even Windows supports PAE->Virtual downgrade */
 917         return STATUS_NOT_IMPLEMENTED;
 918     }
 919
 920     /* The None->Virtual case */
 921     MmPdpt = NULL;
 922     Mmx86SelfMapBase.QuadPart = 0;
 923     MmArchReferencePage = NULL;
 924
 925     /* Truncate all memory above 4GB so that we don't use it @TODO: FIXME */
 926     EfiPrintf(L"Warning: not truncating > 4GB memory. Don't boot with more than 4GB of RAM!\r\n");
 927     //Status = MmPaTruncateMemory(0x100000);
 928     Status = STATUS_SUCCESS;
 929     if (!NT_SUCCESS(Status))
 930     {
 931         goto Quickie;
 932     }
 933
 934     /* Allocate a page directory */
 935     Status = MmPapAllocatePhysicalPagesInRange(&PhysicalAddress,
 936                                                BlLoaderPageDirectory,
 937                                                1,
 938                                                0,
 939                                                0,
 940                                                &MmMdlUnmappedAllocated,
 941                                                0,
 942                                                0);
 943     if (!NT_SUCCESS(Status))
 944     {
 945         goto Quickie;
 946     }
 947
 948     /* Zero out the page directory */
 949     MmPdpt = (PVOID)PhysicalAddress.LowPart;
 950     RtlZeroMemory(MmPdpt, PAGE_SIZE);
 951
 952     /* Set the page size */
 953     MmArchReferencePageSize = PAGE_SIZE;
 954
 955     /* Allocate the self-map page */
 956     Status = MmPapAllocatePhysicalPagesInRange(&PhysicalAddress,
 957                                                BlLoaderReferencePage,
 958                                                1,
 959                                                0,
 960                                                0,
 961                                                &MmMdlUnmappedAllocated,
 962                                                0,
 963                                                0);
 964     if (!NT_SUCCESS(Status))
 965     {
 966         goto Quickie;
 967     }
 968
 969     /* Set the reference page */
 970     MmArchReferencePage = (PVOID)PhysicalAddress.LowPart;
 971
 972     /* Zero it out */
 973     RtlZeroMemory(MmArchReferencePage, MmArchReferencePageSize);
 974
 975     /* Allocate 4MB worth of self-map pages */
 976     Status = MmPaReserveSelfMapPages(&Mmx86SelfMapBase,
 977                                      (4 * 1024 * 1024) >> PAGE_SHIFT,
 978                                      (4 * 1024 * 1024) >> PAGE_SHIFT);
 979     if (!NT_SUCCESS(Status))
 980     {
 981         goto Quickie;
 982     }
 983
 984     /* Zero them out */
 985     RtlZeroMemory((PVOID)Mmx86SelfMapBase.LowPart, 4 * 1024 * 1024);
 986     EfiPrintf(L"PDPT at 0x%p Reference Page at 0x%p Self-map at 0x%p\r\n",
 987               MmPdpt, MmArchReferencePage, Mmx86SelfMapBase.LowPart);
 988
 989     /* Align PTE base to 4MB region */
 990     MmPteBase = (PVOID)(Mmx86SelfMapBase.LowPart & ~0x3FFFFF);
 991
 992     /* The PDE is the PTE of the PTE base */
 993     MmPdeBase = MiAddressToPte(MmPteBase);
 994     PdeIndex = MiAddressToPdeOffset(MmPdeBase);
 995     MmPdpt[PdeIndex].u.Hard.Valid = 1;
 996     MmPdpt[PdeIndex].u.Hard.Write = 1;
 997     MmPdpt[PdeIndex].u.Hard.PageFrameNumber = (ULONG_PTR)MmPdpt >> PAGE_SHIFT;
 998     MmArchReferencePage[PdeIndex]++;
 999
1000     /* Remove PTE_BASE from free virtual memory */
1001     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
1002                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
1003                                        PTE_BASE >> PAGE_SHIFT,
1004                                        (4 * 1024 * 1024) >> PAGE_SHIFT,
1005                                        0);
1006     if (!NT_SUCCESS(Status))
1007     {
1008         goto Quickie;
1009     }
1010
1011     /* Remove HAL_HEAP from free virtual memory */
1012     Status = MmMdRemoveRegionFromMdlEx(&MmMdlFreeVirtual,
1013                                        BL_MM_REMOVE_VIRTUAL_REGION_FLAG,
1014                                        MM_HAL_VA_START >> PAGE_SHIFT,
1015                                        (4 * 1024 * 1024) >> PAGE_SHIFT,
1016                                        0);
1017     if (!NT_SUCCESS(Status))
1018     {
1019         goto Quickie;
1020     }
1021
1022     /* Initialize the virtual->physical memory mappings */
1023     Status = Mmx86InitializeMemoryMap(1, MemoryData);
1024     if (!NT_SUCCESS(Status))
1025     {
1026         goto Quickie;
1027     }
1028
1029     EfiPrintf(L"Ready to turn on motherfucking paging, brah!\r\n");
1030     Status = STATUS_NOT_IMPLEMENTED;
1031
1032 Quickie:
1033     /* Free reference page if we allocated it */
1034     if (MmArchReferencePage)
1035     {
1036         PhysicalAddress.QuadPart = (ULONG_PTR)MmArchReferencePage;
1037         BlMmFreePhysicalPages(PhysicalAddress);
1038     }
1039
1040     /* Free page directory if we allocated it */
1041     if (MmPdpt)
1042     {
1043         PhysicalAddress.QuadPart = (ULONG_PTR)MmPdpt;
1044         BlMmFreePhysicalPages(PhysicalAddress);
1045     }
1046
1047     /* Free the self map if we allocated it */
1048     if (Mmx86SelfMapBase.QuadPart)
1049     {
1050         MmPaReleaseSelfMapPages(Mmx86SelfMapBase);
1051     }
1052
1053     /* All done */
1054     return Status;
1055 }
1056
1057 NTSTATUS
1058 MmArchInitialize (
1059     _In_ ULONG Phase,
1060     _In_ PBL_MEMORY_DATA MemoryData,
1061     _In_ BL_TRANSLATION_TYPE TranslationType,
1062     _In_ BL_TRANSLATION_TYPE RequestedTranslationType
1063     )
1064 {
1065     NTSTATUS Status;
1066     ULONGLONG IncreaseUserVa, PerfCounter, CpuRandom;
1067     INT CpuInfo[4];
1068
1069     /* For phase 2, just map deferred regions */
1070     if (Phase != 1)
1071     {
1072         return Mmx86pMapMemoryRegions(2, MemoryData);
1073     }
1074
1075     /* What translation type are we switching to? */
1076     switch (RequestedTranslationType)
1077     {
1078         /* Physical memory */
1079         case BlNone:
1080
1081             /* Initialize everything to default/null values */
1082             MmArchLargePageSize = 1;
1083             MmArchKsegBase = 0;
1084             MmArchKsegBias = 0;
1085             MmArchKsegAddressRange.Minimum = 0;
1086             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1087             MmArchTopOfApplicationAddressSpace = 0;
1088             Mmx86SelfMapBase.QuadPart = 0;
1089
1090             /* Set stub functions */
1091             BlMmRelocateSelfMap = MmArchNullFunction;
1092             BlMmFlushTlb = MmArchNullFunction;
1093
1094             /* Set success */
1095             Status = STATUS_SUCCESS;
1096             break;
1097
1098         case BlVirtual:
1099
1100             /* Set the large page size to 1024 pages (4MB) */
1101             MmArchLargePageSize = (4 * 1024 * 1024) / PAGE_SIZE;
1102
1103             /* Check if /USERVA option was used */
1104             Status = BlGetBootOptionInteger(BlpApplicationEntry.BcdData,
1105                                             BcdOSLoaderInteger_IncreaseUserVa,
1106                                             &IncreaseUserVa);
1107             if (NT_SUCCESS(Status) && (IncreaseUserVa))
1108             {
1109                 /* Yes -- load the kernel at 0xE0000000 instead */
1110                 MmArchKsegBase = 0xE0000000;
1111             }
1112             else
1113             {
1114                 /* Nope, load at the standard 2GB split */
1115                 MmArchKsegBase = 0x80000000;
1116             }
1117
1118             /* Check if CPUID 01h is supported */
1119             CpuRandom = 0;
1120             if (BlArchIsCpuIdFunctionSupported(1))
1121             {
1122                 /* Call it */
1123                 BlArchCpuId(1, 0, CpuInfo);
1124
1125                 /* Check if RDRAND is supported */
1126                 if (CpuInfo[2] & 0x40000000)
1127                 {
1128                     EfiPrintf(L"Your CPU can do RDRAND! Good for you!\r\n");
1129                     CpuRandom = 0;
1130                 }
1131             }
1132
1133             /* Read the TSC */
1134             PerfCounter = BlArchGetPerformanceCounter();
1135             PerfCounter >>= 4;
1136             _rotl16(PerfCounter, 5);
1137
1138             /* Set the address range */
1139             MmArchKsegAddressRange.Minimum = 0;
1140             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1141
1142             /* Set the KASLR bias */
1143             MmArchKsegBias = ((PerfCounter ^ CpuRandom) & 0xFFF) << 12;
1144             MmArchKsegBias = 0;
1145             MmArchKsegBase += MmArchKsegBias;
1146
1147             /* Set the kernel range */
1148             MmArchKsegAddressRange.Minimum = MmArchKsegBase;
1149             MmArchKsegAddressRange.Maximum = (ULONGLONG)~0;
1150
1151             /* Set the boot application top maximum */
1152             MmArchTopOfApplicationAddressSpace = 0x70000000;
1153
1154             /* Initialize virtual address space translation */
1155             Status = MmDefInitializeTranslation(MemoryData, TranslationType);
1156             if (NT_SUCCESS(Status))
1157             {
1158                 /* Set stub functions */
1159                 BlMmRelocateSelfMap = MmDefRelocateSelfMap;
1160                 BlMmFlushTlb = Mmx86FlushTlb;
1161                 BlMmMoveVirtualAddressRange = MmDefMoveVirtualAddressRange;
1162                 BlMmZeroVirtualAddressRange = MmDefZeroVirtualAddressRange;
1163             }
1164             break;
1165
1166         case BlPae:
1167
1168             /* We don't support PAE */
1169             Status = STATUS_NOT_SUPPORTED;
1170             break;
1171
1172         default:
1173
1174             /* Invalid architecture type*/
1175             Status = STATUS_INVALID_PARAMETER;
1176             break;
1177     }
1178
1179     /* Back to caller */
1180     return Status;
1181 }