ntoskrnl/mm/amd64/page.c

   1 /*
   2  * COPYRIGHT:       GPL, See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/amd64/page.c
   5  * PURPOSE:         Low level memory managment manipulation
   6  *
   7  * PROGRAMMER:      Timo Kreuzer (timo.kreuzer@reactos.org)
   8  *                  ReactOS Portable Systems Group
   9  */
  10
  11 /* INCLUDES ***************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16 #include "../ARM3/miarm.h"
  17
  18 #undef InterlockedExchangePte
  19 #define InterlockedExchangePte(pte1, pte2) \
  20     InterlockedExchange64((LONG64*)&pte1->u.Long, pte2.u.Long)
  21
  22 #define PAGE_EXECUTE_ANY (PAGE_EXECUTE|PAGE_EXECUTE_READ|PAGE_EXECUTE_READWRITE|PAGE_EXECUTE_WRITECOPY)
  23 #define PAGE_WRITE_ANY (PAGE_EXECUTE_READWRITE|PAGE_READWRITE|PAGE_EXECUTE_WRITECOPY|PAGE_WRITECOPY)
  24 #define PAGE_WRITECOPY_ANY (PAGE_EXECUTE_WRITECOPY|PAGE_WRITECOPY)
  25
  26 extern MMPTE HyperTemplatePte;
  27
  28 /* GLOBALS *****************************************************************/
  29
  30 const
  31 ULONG
  32 MmProtectToPteMask[32] =
  33 {
  34     //
  35     // These are the base MM_ protection flags
  36     //
  37     0,
  38     PTE_READONLY            | PTE_ENABLE_CACHE,
  39     PTE_EXECUTE             | PTE_ENABLE_CACHE,
  40     PTE_EXECUTE_READ        | PTE_ENABLE_CACHE,
  41     PTE_READWRITE           | PTE_ENABLE_CACHE,
  42     PTE_WRITECOPY           | PTE_ENABLE_CACHE,
  43     PTE_EXECUTE_READWRITE   | PTE_ENABLE_CACHE,
  44     PTE_EXECUTE_WRITECOPY   | PTE_ENABLE_CACHE,
  45     //
  46     // These OR in the MM_NOCACHE flag
  47     //
  48     0,
  49     PTE_READONLY            | PTE_DISABLE_CACHE,
  50     PTE_EXECUTE             | PTE_DISABLE_CACHE,
  51     PTE_EXECUTE_READ        | PTE_DISABLE_CACHE,
  52     PTE_READWRITE           | PTE_DISABLE_CACHE,
  53     PTE_WRITECOPY           | PTE_DISABLE_CACHE,
  54     PTE_EXECUTE_READWRITE   | PTE_DISABLE_CACHE,
  55     PTE_EXECUTE_WRITECOPY   | PTE_DISABLE_CACHE,
  56     //
  57     // These OR in the MM_DECOMMIT flag, which doesn't seem supported on x86/64/ARM
  58     //
  59     0,
  60     PTE_READONLY            | PTE_ENABLE_CACHE,
  61     PTE_EXECUTE             | PTE_ENABLE_CACHE,
  62     PTE_EXECUTE_READ        | PTE_ENABLE_CACHE,
  63     PTE_READWRITE           | PTE_ENABLE_CACHE,
  64     PTE_WRITECOPY           | PTE_ENABLE_CACHE,
  65     PTE_EXECUTE_READWRITE   | PTE_ENABLE_CACHE,
  66     PTE_EXECUTE_WRITECOPY   | PTE_ENABLE_CACHE,
  67     //
  68     // These OR in the MM_NOACCESS flag, which seems to enable WriteCombining?
  69     //
  70     0,
  71     PTE_READONLY            | PTE_WRITECOMBINED_CACHE,
  72     PTE_EXECUTE             | PTE_WRITECOMBINED_CACHE,
  73     PTE_EXECUTE_READ        | PTE_WRITECOMBINED_CACHE,
  74     PTE_READWRITE           | PTE_WRITECOMBINED_CACHE,
  75     PTE_WRITECOPY           | PTE_WRITECOMBINED_CACHE,
  76     PTE_EXECUTE_READWRITE   | PTE_WRITECOMBINED_CACHE,
  77     PTE_EXECUTE_WRITECOPY   | PTE_WRITECOMBINED_CACHE,
  78 };
  79
  80 /* PRIVATE FUNCTIONS *******************************************************/
  81
  82 BOOLEAN
  83 FORCEINLINE
  84 MiIsHyperspaceAddress(PVOID Address)
  85 {
  86     return ((ULONG64)Address >= HYPER_SPACE &&
  87             (ULONG64)Address <= HYPER_SPACE_END);
  88 }
  89
  90 VOID
  91 MiFlushTlb(PMMPTE Pte, PVOID Address)
  92 {
  93     if (MiIsHyperspaceAddress(Pte))
  94     {
  95         MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pte));
  96     }
  97     else
  98     {
  99         __invlpg(Address);
 100     }
 101 }
 102
 103 static
 104 PMMPTE
 105 MiGetPteForProcess(
 106     PEPROCESS Process,
 107     PVOID Address,
 108     BOOLEAN Create)
 109 {
 110     MMPTE TmplPte, *Pte;
 111
 112     /* Check if we need hypersapce mapping */
 113     if (Address < MmSystemRangeStart &&
 114         Process && Process != PsGetCurrentProcess())
 115     {
 116         UNIMPLEMENTED;
 117         return NULL;
 118     }
 119     else if (Create)
 120     {
 121         TmplPte.u.Long = 0;
 122         TmplPte.u.Flush.Valid = 1;
 123         TmplPte.u.Flush.Write = 1;
 124
 125         /* Get the PXE */
 126         Pte = MiAddressToPxe(Address);
 127         if (!Pte->u.Hard.Valid)
 128         {
 129 //            TmplPte.u.Hard.PageFrameNumber = MiAllocPage(TRUE);
 130             InterlockedExchangePte(Pte, TmplPte);
 131         }
 132
 133         /* Get the PPE */
 134         Pte = MiAddressToPpe(Address);
 135         if (!Pte->u.Hard.Valid)
 136         {
 137 //            TmplPte.u.Hard.PageFrameNumber = MiAllocPage(TRUE);
 138             InterlockedExchangePte(Pte, TmplPte);
 139         }
 140
 141         /* Get the PDE */
 142         Pte = MiAddressToPde(Address);
 143         if (!Pte->u.Hard.Valid)
 144         {
 145 //            TmplPte.u.Hard.PageFrameNumber = MiAllocPage(TRUE);
 146             InterlockedExchangePte(Pte, TmplPte);
 147         }
 148     }
 149     else
 150     {
 151         /* Get the PXE */
 152         Pte = MiAddressToPxe(Address);
 153         if (!Pte->u.Hard.Valid)
 154             return NULL;
 155
 156         /* Get the PPE */
 157         Pte = MiAddressToPpe(Address);
 158         if (!Pte->u.Hard.Valid)
 159             return NULL;
 160
 161         /* Get the PDE */
 162         Pte = MiAddressToPde(Address);
 163         if (!Pte->u.Hard.Valid)
 164             return NULL;
 165     }
 166
 167     return MiAddressToPte(Address);
 168 }
 169
 170 static
 171 ULONG64
 172 MiGetPteValueForProcess(
 173     PEPROCESS Process,
 174     PVOID Address)
 175 {
 176     PMMPTE Pte;
 177     ULONG64 PteValue;
 178
 179     Pte = MiGetPteForProcess(Process, Address, FALSE);
 180     PteValue = Pte ? Pte->u.Long : 0;
 181
 182     if (MiIsHyperspaceAddress(Pte))
 183         MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pte));
 184
 185     return PteValue;
 186 }
 187
 188 ULONG
 189 NTAPI
 190 MiGetPteProtection(MMPTE Pte)
 191 {
 192     ULONG Protect;
 193
 194     if (!Pte.u.Flush.Valid)
 195     {
 196         Protect = PAGE_NOACCESS;
 197     }
 198     else if (Pte.u.Flush.NoExecute)
 199     {
 200         if (Pte.u.Flush.CopyOnWrite)
 201             Protect = PAGE_WRITECOPY;
 202         else if (Pte.u.Flush.Write)
 203             Protect = PAGE_READWRITE;
 204         else
 205             Protect = PAGE_READONLY;
 206     }
 207     else
 208     {
 209         if (Pte.u.Flush.CopyOnWrite)
 210             Protect = PAGE_EXECUTE_WRITECOPY;
 211         else if (Pte.u.Flush.Write)
 212             Protect = PAGE_EXECUTE_READWRITE;
 213         else
 214             Protect = PAGE_EXECUTE_READ;
 215     }
 216
 217     if (Pte.u.Flush.CacheDisable)
 218         Protect |= PAGE_NOCACHE;
 219
 220     if (Pte.u.Flush.WriteThrough)
 221         Protect |= PAGE_WRITETHROUGH;
 222
 223     // PAGE_GUARD ?
 224     return Protect;
 225 }
 226
 227 VOID
 228 NTAPI
 229 MiSetPteProtection(PMMPTE Pte, ULONG Protection)
 230 {
 231     Pte->u.Flush.CopyOnWrite = (Protection & PAGE_WRITECOPY_ANY) ? 1 : 0;
 232     Pte->u.Flush.Write = (Protection & PAGE_WRITE_ANY) ? 1 : 0;
 233     Pte->u.Flush.CacheDisable = (Protection & PAGE_NOCACHE) ? 1 : 0;
 234     Pte->u.Flush.WriteThrough = (Protection & PAGE_WRITETHROUGH) ? 1 : 0;
 235
 236     // FIXME: This doesn't work. Why?
 237 //    Pte->u.Flush.NoExecute = (Protection & PAGE_EXECUTE_ANY) ? 0 : 1;
 238 }
 239
 240 /* FUNCTIONS ***************************************************************/
 241
 242 PFN_NUMBER
 243 NTAPI
 244 MmGetPfnForProcess(PEPROCESS Process,
 245                    PVOID Address)
 246 {
 247     MMPTE Pte;
 248     Pte.u.Long = MiGetPteValueForProcess(Process, Address);
 249     return Pte.u.Hard.Valid ? Pte.u.Hard.PageFrameNumber : 0;
 250 }
 251
 252 PHYSICAL_ADDRESS
 253 NTAPI
 254 MmGetPhysicalAddress(PVOID Address)
 255 {
 256     PHYSICAL_ADDRESS p;
 257     MMPTE Pte;
 258
 259     Pte.u.Long = MiGetPteValueForProcess(NULL, Address);
 260     if (Pte.u.Hard.Valid)
 261     {
 262         p.QuadPart = Pte.u.Hard.PageFrameNumber * PAGE_SIZE;
 263         p.u.LowPart |= (ULONG_PTR)Address & (PAGE_SIZE - 1);
 264     }
 265     else
 266     {
 267         p.QuadPart = 0;
 268     }
 269
 270     return p;
 271 }
 272
 273 BOOLEAN
 274 NTAPI
 275 MmIsPagePresent(PEPROCESS Process, PVOID Address)
 276 {
 277     MMPTE Pte;
 278     Pte.u.Long = MiGetPteValueForProcess(Process, Address);
 279     return Pte.u.Hard.Valid;
 280 }
 281
 282 BOOLEAN
 283 NTAPI
 284 MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
 285 {
 286     MMPTE Pte;
 287     Pte.u.Long = MiGetPteValueForProcess(Process, Address);
 288     return Pte.u.Hard.Valid && Pte.u.Soft.Transition;
 289 }
 290
 291 BOOLEAN
 292 NTAPI
 293 MmIsDirtyPage(PEPROCESS Process, PVOID Address)
 294 {
 295     MMPTE Pte;
 296     Pte.u.Long = MiGetPteValueForProcess(Process, Address);
 297     return Pte.u.Hard.Valid && Pte.u.Hard.Dirty;
 298 }
 299
 300 ULONG
 301 NTAPI
 302 MmGetPageProtect(PEPROCESS Process, PVOID Address)
 303 {
 304     MMPTE Pte;
 305
 306     Pte.u.Long = MiGetPteValueForProcess(Process, Address);
 307
 308     return MiGetPteProtection(Pte);
 309 }
 310
 311 VOID
 312 NTAPI
 313 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
 314 {
 315     PMMPTE Pte;
 316     MMPTE NewPte;
 317
 318     Pte = MiGetPteForProcess(Process, Address, FALSE);
 319     ASSERT(Pte != NULL);
 320
 321     NewPte = *Pte;
 322
 323     MiSetPteProtection(&NewPte, flProtect);
 324
 325     InterlockedExchangePte(Pte, NewPte);
 326
 327     MiFlushTlb(Pte, Address);
 328 }
 329
 330 VOID
 331 NTAPI
 332 MmSetCleanPage(PEPROCESS Process, PVOID Address)
 333 {
 334     PMMPTE Pte;
 335
 336     Pte = MiGetPteForProcess(Process, Address, FALSE);
 337     if (!Pte)
 338     {
 339         KeBugCheckEx(MEMORY_MANAGEMENT, 0x1234, (ULONG64)Address, 0, 0);
 340     }
 341
 342     /* Ckear the dirty bit */
 343     if (InterlockedBitTestAndReset64((PVOID)Pte, 6))
 344     {
 345         if (!MiIsHyperspaceAddress(Pte))
 346             __invlpg(Address);
 347     }
 348
 349     MiFlushTlb(Pte, Address);
 350 }
 351
 352 VOID
 353 NTAPI
 354 MmSetDirtyPage(PEPROCESS Process, PVOID Address)
 355 {
 356     PMMPTE Pte;
 357
 358     Pte = MiGetPteForProcess(Process, Address, FALSE);
 359     if (!Pte)
 360     {
 361         KeBugCheckEx(MEMORY_MANAGEMENT, 0x1234, (ULONG64)Address, 0, 0);
 362     }
 363
 364     /* Ckear the dirty bit */
 365     if (InterlockedBitTestAndSet64((PVOID)Pte, 6))
 366     {
 367         if (!MiIsHyperspaceAddress(Pte))
 368             __invlpg(Address);
 369     }
 370
 371     MiFlushTlb(Pte, Address);
 372 }
 373
 374
 375 NTSTATUS
 376 NTAPI
 377 Mmi386ReleaseMmInfo(PEPROCESS Process)
 378 {
 379     UNIMPLEMENTED;
 380     return STATUS_UNSUCCESSFUL;
 381 }
 382
 383 VOID
 384 NTAPI
 385 MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPFN_NUMBER Page)
 386 {
 387     UNIMPLEMENTED;
 388 }
 389
 390 VOID
 391 NTAPI
 392 MmRawDeleteVirtualMapping(PVOID Address)
 393 {
 394     UNIMPLEMENTED;
 395 }
 396
 397 VOID
 398 NTAPI
 399 MmDeleteVirtualMapping(
 400     PEPROCESS Process,
 401     PVOID Address,
 402     BOOLEAN FreePage,
 403     BOOLEAN* WasDirty,
 404     PPFN_NUMBER Page)
 405 {
 406     PFN_NUMBER Pfn;
 407     PMMPTE Pte;
 408     MMPTE OldPte;
 409
 410     Pte = MiGetPteForProcess(Process, Address, FALSE);
 411
 412     if (Pte)
 413     {
 414         /* Atomically set the entry to zero and get the old value. */
 415         OldPte.u.Long = InterlockedExchange64((LONG64*)&Pte->u.Long, 0);
 416
 417         if (OldPte.u.Hard.Valid)
 418         {
 419             Pfn = OldPte.u.Hard.PageFrameNumber;
 420
 421             if (FreePage)
 422                 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 423         }
 424         else
 425             Pfn = 0;
 426     }
 427     else
 428     {
 429         OldPte.u.Long = 0;
 430         Pfn = 0;
 431     }
 432
 433     /* Return information to the caller */
 434     if (WasDirty)
 435         *WasDirty = OldPte.u.Hard.Dirty;;
 436
 437     if (Page)
 438         *Page = Pfn;
 439
 440     MiFlushTlb(Pte, Address);
 441 }
 442
 443 VOID
 444 NTAPI
 445 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
 446                         SWAPENTRY* SwapEntry)
 447 {
 448     UNIMPLEMENTED;
 449 }
 450
 451
 452 VOID
 453 NTAPI
 454 MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
 455 {
 456     UNIMPLEMENTED;
 457 }
 458
 459
 460 NTSTATUS
 461 NTAPI
 462 MmCreatePageFileMapping(PEPROCESS Process,
 463                         PVOID Address,
 464                         SWAPENTRY SwapEntry)
 465 {
 466     UNIMPLEMENTED;
 467     return STATUS_UNSUCCESSFUL;
 468 }
 469
 470
 471 NTSTATUS
 472 NTAPI
 473 MmCreateVirtualMappingUnsafe(
 474     PEPROCESS Process,
 475     PVOID Address,
 476     ULONG PageProtection,
 477     PPFN_NUMBER Pages,
 478     ULONG PageCount)
 479 {
 480     ULONG i;
 481     MMPTE TmplPte, *Pte;
 482
 483     /* Check if the range is valid */
 484     if ((Process == NULL && Address < MmSystemRangeStart) ||
 485         (Process != NULL && Address > MmHighestUserAddress))
 486     {
 487         DPRINT1("Address 0x%p is invalid for process %p\n", Address, Process);
 488         ASSERT(FALSE);
 489     }
 490
 491     TmplPte.u.Long = 0;
 492     TmplPte.u.Hard.Valid = 1;
 493     MiSetPteProtection(&TmplPte, PageProtection);
 494
 495 //__debugbreak();
 496
 497     for (i = 0; i < PageCount; i++)
 498     {
 499         TmplPte.u.Hard.PageFrameNumber = Pages[i];
 500
 501         Pte = MiGetPteForProcess(Process, Address, TRUE);
 502
 503 DPRINT1("MmCreateVirtualMappingUnsafe, Address=%p, TmplPte=%p, Pte=%p\n",
 504         Address, TmplPte.u.Long, Pte);
 505
 506         if (InterlockedExchangePte(Pte, TmplPte))
 507         {
 508             KeInvalidateTlbEntry(Address);
 509         }
 510
 511         if (MiIsHyperspaceAddress(Pte))
 512             MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pte));
 513
 514         Address = (PVOID)((ULONG64)Address + PAGE_SIZE);
 515     }
 516
 517
 518     return STATUS_SUCCESS;
 519 }
 520
 521 NTSTATUS
 522 NTAPI
 523 MmCreateVirtualMapping(PEPROCESS Process,
 524                        PVOID Address,
 525                        ULONG Protect,
 526                        PPFN_NUMBER Pages,
 527                        ULONG PageCount)
 528 {
 529     ULONG i;
 530
 531     for (i = 0; i < PageCount; i++)
 532     {
 533         if (!MmIsPageInUse(Pages[i]))
 534         {
 535             DPRINT1("Page %x not in use\n", Pages[i]);
 536             KeBugCheck(MEMORY_MANAGEMENT);
 537         }
 538     }
 539
 540     return MmCreateVirtualMappingUnsafe(Process, Address, Protect, Pages, PageCount);
 541 }
 542
 543 BOOLEAN
 544 NTAPI
 545 MmCreateProcessAddressSpace(IN ULONG MinWs,
 546                             IN PEPROCESS Process,
 547                             OUT PULONG_PTR DirectoryTableBase)
 548 {
 549     KIRQL OldIrql;
 550     PFN_NUMBER TableBasePfn, HyperPfn;
 551     PMMPTE PointerPte;
 552     MMPTE TempPte, PdePte;
 553     ULONG TableIndex;
 554     PMMPTE SystemTable;
 555
 556     /* No page colors yet */
 557     Process->NextPageColor = 0;
 558
 559     /* Setup the hyperspace lock */
 560     KeInitializeSpinLock(&Process->HyperSpaceLock);
 561
 562     /* Lock PFN database */
 563     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 564
 565     /* Get a page for the table base and for hyperspace */
 566     TableBasePfn = MiRemoveAnyPage(0);
 567     HyperPfn = MiRemoveAnyPage(0);
 568
 569     /* Release PFN lock */
 570     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 571
 572     /* Zero both pages */
 573     MiZeroPhysicalPage(TableBasePfn);
 574     MiZeroPhysicalPage(HyperPfn);
 575
 576     /* Set the base directory pointers */
 577     DirectoryTableBase[0] = TableBasePfn << PAGE_SHIFT;
 578     DirectoryTableBase[1] = HyperPfn << PAGE_SHIFT;
 579
 580     /* Make sure we don't already have a page directory setup */
 581     ASSERT(Process->Pcb.DirectoryTableBase[0] == 0);
 582
 583     /* Insert us into the Mm process list */
 584     InsertTailList(&MmProcessList, &Process->MmProcessLinks);
 585
 586     /* Get a PTE to map the page directory */
 587     PointerPte = MiReserveSystemPtes(1, SystemPteSpace);
 588     ASSERT(PointerPte != NULL);
 589
 590     /* Build it */
 591     MI_MAKE_HARDWARE_PTE_KERNEL(&PdePte,
 592                                 PointerPte,
 593                                 MM_READWRITE,
 594                                 TableBasePfn);
 595
 596     /* Set it dirty and map it */
 597     PdePte.u.Hard.Dirty = TRUE;
 598     MI_WRITE_VALID_PTE(PointerPte, PdePte);
 599
 600     /* Now get the page directory (which we'll double map, so call it a page table */
 601     SystemTable = MiPteToAddress(PointerPte);
 602
 603     /* Copy all the kernel mappings */
 604     TableIndex = MiAddressToPxi(MmSystemRangeStart);
 605
 606     RtlCopyMemory(&SystemTable[TableIndex],
 607                   MiAddressToPxe(MmSystemRangeStart),
 608                   PAGE_SIZE - TableIndex * sizeof(MMPTE));
 609
 610     /* Now write the PTE/PDE entry for hyperspace itself */
 611     TempPte = ValidKernelPte;
 612     TempPte.u.Hard.PageFrameNumber = HyperPfn;
 613     TableIndex = MiAddressToPxi(HYPER_SPACE);
 614     SystemTable[TableIndex] = TempPte;
 615
 616     /* Sanity check */
 617     ASSERT(MiAddressToPxi(MmHyperSpaceEnd) > TableIndex);
 618
 619     /* Now do the x86 trick of making the PDE a page table itself */
 620     TableIndex = MiAddressToPxi(PTE_BASE);
 621     TempPte.u.Hard.PageFrameNumber = TableBasePfn;
 622     SystemTable[TableIndex] = TempPte;
 623
 624     /* Let go of the system PTE */
 625     MiReleaseSystemPtes(PointerPte, 1, SystemPteSpace);
 626
 627     /* Switch to phase 1 initialization */
 628     ASSERT(Process->AddressSpaceInitialized == 0);
 629     Process->AddressSpaceInitialized = 1;
 630
 631     return TRUE;
 632 }
 633
 634 /* EOF */