reactos/ntoskrnl/mm/i386/pagepae.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/i386/page.c
   5  * PURPOSE:         Low level memory managment manipulation
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES ***************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
  18 #endif
  19
  20
  21 /* GLOBALS *****************************************************************/
  22
  23 #define PA_BIT_PRESENT   (0)
  24 #define PA_BIT_READWRITE (1)
  25 #define PA_BIT_USER      (2)
  26 #define PA_BIT_WT        (3)
  27 #define PA_BIT_CD        (4)
  28 #define PA_BIT_ACCESSED  (5)
  29 #define PA_BIT_DIRTY     (6)
  30 #define PA_BIT_GLOBAL    (8)
  31
  32 #define PA_PRESENT   (1 << PA_BIT_PRESENT)
  33 #define PA_READWRITE (1 << PA_BIT_READWRITE)
  34 #define PA_USER      (1 << PA_BIT_USER)
  35 #define PA_DIRTY     (1 << PA_BIT_DIRTY)
  36 #define PA_WT        (1 << PA_BIT_WT)
  37 #define PA_CD        (1 << PA_BIT_CD)
  38 #define PA_ACCESSED  (1 << PA_BIT_ACCESSED)
  39 #define PA_GLOBAL    (1 << PA_BIT_GLOBAL)
  40
  41 #define PAGETABLE_MAP           (0xc0000000)
  42 #define PAGEDIRECTORY_MAP       (0xc0000000 + (PAGETABLE_MAP / (1024)))
  43
  44 #define PAE_PAGEDIRECTORY_MAP   (0xc0000000 + (PAGETABLE_MAP / (512)))
  45
  46 #define HYPERSPACE              (Ke386Pae ? 0xc0800000 : 0xc0400000)
  47 #define IS_HYPERSPACE(v)        (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
  48
  49 ULONG MmGlobalKernelPageDirectory[1024];
  50 ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];
  51
  52 #define PTE_TO_PFN(X)  ((X) >> PAGE_SHIFT)
  53 #define PFN_TO_PTE(X)  ((X) << PAGE_SHIFT)
  54
  55 #define PAE_PTE_TO_PFN(X)   (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
  56 #define PAE_PFN_TO_PTE(X)   ((X) << PAGE_SHIFT)
  57
  58 extern BOOLEAN Ke386Pae;
  59 extern BOOLEAN Ke386NoExecute;
  60
  61 /* FUNCTIONS ***************************************************************/
  62
  63 BOOLEAN MmUnmapPageTable(PULONG Pt);
  64
  65 ULONG_PTR
  66 NTAPI
  67 MiFlushTlbIpiRoutine(ULONG_PTR Address)
  68 {
  69    if (Address == (ULONGLONG)-1)
  70    {
  71       KeFlushCurrentTb();
  72    }
  73    else if (Address == (ULONGLONG)-2)
  74    {
  75       KeFlushCurrentTb();
  76    }
  77    else
  78    {
  79        __invlpg((PVOID)Address);
  80    }
  81    return 0;
  82 }
  83
  84 VOID
  85 MiFlushTlb(PULONG Pt, PVOID Address)
  86 {
  87 #ifdef CONFIG_SMP
  88    if (Pt)
  89    {
  90       MmUnmapPageTable(Pt);
  91    }
  92    if (KeNumberProcessors > 1)
  93    {
  94       KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address);
  95    }
  96    else
  97    {
  98       MiFlushTlbIpiRoutine((ULONG_PTR)Address);
  99    }
 100 #else
 101    if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart)
 102    {
 103       __invlpg(Address);
 104    }
 105 #endif
 106 }
 107
 108
 109
 110 PULONG
 111 MmGetPageDirectory(VOID)
 112 {
 113    return (PULONG)__readcr3();
 114 }
 115
 116 static ULONG
 117 ProtectToPTE(ULONG flProtect)
 118 {
 119    ULONG Attributes = 0;
 120
 121    if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
 122    {
 123       Attributes = 0;
 124    }
 125    else if (flProtect & PAGE_IS_WRITABLE)
 126    {
 127       Attributes = PA_PRESENT | PA_READWRITE;
 128    }
 129    else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
 130    {
 131       Attributes = PA_PRESENT;
 132    }
 133    else
 134    {
 135       DPRINT1("Unknown main protection type.\n");
 136       ASSERT(FALSE);
 137    }
 138    if (Ke386NoExecute &&
 139        !(flProtect & PAGE_IS_EXECUTABLE))
 140    {
 141       Attributes = Attributes | 0x80000000;
 142    }
 143
 144    if (flProtect & PAGE_SYSTEM)
 145    {
 146    }
 147    else
 148    {
 149       Attributes = Attributes | PA_USER;
 150    }
 151    if (flProtect & PAGE_NOCACHE)
 152    {
 153       Attributes = Attributes | PA_CD;
 154    }
 155    if (flProtect & PAGE_WRITETHROUGH)
 156    {
 157       Attributes = Attributes | PA_WT;
 158    }
 159    return(Attributes);
 160 }
 161
 162 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
 163
 164 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
 165                                 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
 166 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
 167
 168 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
 169
 170 #define ADDR_TO_PTE_OFFSET(v)  ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
 171
 172
 173 #define PAE_ADDR_TO_PAGE_TABLE(v)   (((ULONG)(v)) / (512 * PAGE_SIZE))
 174
 175 #define PAE_ADDR_TO_PDE(v)          (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
 176                                                   ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
 177 #define PAE_ADDR_TO_PTE(v)          (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
 178
 179
 180 #define PAE_ADDR_TO_PDTE_OFFSET(v)  (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
 181
 182 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v)   ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
 183
 184 #define PAE_ADDR_TO_PDE_OFFSET(v)   (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
 185
 186 #define PAE_ADDR_TO_PTE_OFFSET(v)   ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
 187
 188 BOOLEAN
 189 NTAPI
 190 MmCreateProcessAddressSpace(IN ULONG MinWs,
 191                             IN PEPROCESS Process,
 192                             IN PLARGE_INTEGER DirectoryTableBase)
 193 {
 194    NTSTATUS Status;
 195    ULONG i, j;
 196    PFN_NUMBER Pfn[7];
 197    ULONG Count;
 198
 199    DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process);
 200
 201    Count = Ke386Pae ? 7 : 2;
 202
 203    for (i = 0; i < Count; i++)
 204    {
 205       Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
 206       if (!NT_SUCCESS(Status))
 207       {
 208           for (j = 0; j < i; j++)
 209           {
 210               MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
 211           }
 212
 213           return FALSE;
 214       }
 215    }
 216
 217    if (Ke386Pae)
 218    {
 219       PULONGLONG PageDirTable;
 220       PULONGLONG PageDir;
 221
 222       PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
 223       for (i = 0; i < 4; i++)
 224       {
 225          PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
 226       }
 227       MmDeleteHyperspaceMapping(PageDirTable);
 228       for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++)
 229       {
 230          PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
 231          memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
 232          if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
 233          {
 234             for (j = 0; j < 4; j++)
 235             {
 236                PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
 237             }
 238          }
 239          if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
 240          {
 241             PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
 242             PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
 243          }
 244          MmDeleteHyperspaceMapping(PageDir);
 245       }
 246    }
 247    else
 248    {
 249       PULONG PageDirectory;
 250       PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);
 251
 252       memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
 253              MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
 254              (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG));
 255
 256       DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
 257       PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
 258       PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;
 259
 260       MmDeleteHyperspaceMapping(PageDirectory);
 261    }
 262
 263    DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]);
 264    DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart);
 265    return TRUE;
 266 }
 267
 268 VOID
 269 NTAPI
 270 MmFreePageTable(PEPROCESS Process, PVOID Address)
 271 {
 272    PEPROCESS CurrentProcess = PsGetCurrentProcess();
 273    ULONG i;
 274    PFN_NUMBER Pfn;
 275
 276    DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
 277    if (Process != NULL && Process != CurrentProcess)
 278    {
 279       KeAttachProcess(&Process->Pcb);
 280    }
 281    if (Ke386Pae)
 282    {
 283       PULONGLONG PageTable;
 284       ULONGLONG ZeroPte = 0LL;
 285       PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
 286       for (i = 0; i < 512; i++)
 287       {
 288          if (PageTable[i] != 0LL)
 289          {
 290             DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
 291                      ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
 292             ASSERT(FALSE);
 293          }
 294       }
 295       Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
 296       (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
 297       MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
 298    }
 299    else
 300    {
 301       PULONG PageTable;
 302       PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
 303       for (i = 0; i < 1024; i++)
 304       {
 305          if (PageTable[i] != 0)
 306          {
 307             DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
 308                      ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
 309             ASSERT(FALSE);
 310          }
 311       }
 312       Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
 313       *(ADDR_TO_PDE(Address)) = 0;
 314       MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
 315    }
 316
 317    if (Address >= MmSystemRangeStart)
 318    {
 319       //    MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
 320       ASSERT(FALSE);
 321    }
 322    else
 323    {
 324       MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 325    }
 326    if (Process != NULL && Process != CurrentProcess)
 327    {
 328       KeDetachProcess();
 329    }
 330 }
 331
 332 static PULONGLONG
 333 MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
 334 {
 335    NTSTATUS Status;
 336    PFN_NUMBER Pfn;
 337    ULONGLONG Entry;
 338    ULONGLONG ZeroEntry = 0LL;
 339    PULONGLONG Pt;
 340    PULONGLONG PageDir;
 341    PULONGLONG PageDirTable;
 342
 343    DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
 344           Process, Address, Create);
 345    if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000))
 346    {
 347       ASSERT(FALSE);
 348    }
 349    if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
 350    {
 351       PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
 352       if (PageDirTable == NULL)
 353       {
 354          ASSERT(FALSE);
 355       }
 356       PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
 357       MmDeleteHyperspaceMapping(PageDirTable);
 358       if (PageDir == NULL)
 359       {
 360          ASSERT(FALSE);
 361       }
 362       PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
 363       Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
 364       if (Entry == 0LL)
 365       {
 366          if (Create == FALSE)
 367          {
 368             MmDeleteHyperspaceMapping(PageDir);
 369             return NULL;
 370          }
 371          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 372          if (!NT_SUCCESS(Status))
 373          {
 374             ASSERT(FALSE);
 375          }
 376          Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
 377          Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
 378          if (Entry != 0LL)
 379          {
 380             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 381             Pfn = PAE_PTE_TO_PFN(Entry);
 382          }
 383       }
 384       else
 385       {
 386          Pfn = PAE_PTE_TO_PFN(Entry);
 387       }
 388       MmDeleteHyperspaceMapping(PageDir);
 389       Pt = MmCreateHyperspaceMapping(Pfn);
 390       if (Pt == NULL)
 391       {
 392          ASSERT(FALSE);
 393       }
 394       return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
 395    }
 396    PageDir = PAE_ADDR_TO_PDE(Address);
 397    if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
 398    {
 399       if (Address >= MmSystemRangeStart)
 400       {
 401          if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
 402          {
 403             if (Create == FALSE)
 404             {
 405                return NULL;
 406             }
 407             Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 408             if (!NT_SUCCESS(Status))
 409             {
 410                ASSERT(FALSE);
 411             }
 412             Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
 413             if (Ke386GlobalPagesEnabled)
 414             {
 415                Entry |= PA_GLOBAL;
 416             }
 417             if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
 418             {
 419                MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 420             }
 421          }
 422          (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
 423       }
 424       else
 425       {
 426          if (Create == FALSE)
 427          {
 428             return NULL;
 429          }
 430          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 431          if (!NT_SUCCESS(Status))
 432          {
 433             ASSERT(FALSE);
 434          }
 435          Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
 436          Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
 437          if (Entry != 0LL)
 438          {
 439             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 440          }
 441       }
 442    }
 443    return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
 444 }
 445
 446 static PULONG
 447 MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
 448 {
 449    ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
 450    NTSTATUS Status;
 451    PFN_NUMBER Pfn;
 452    ULONG Entry;
 453    PULONG Pt, PageDir;
 454
 455    if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
 456    {
 457       PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.LowPart));
 458       if (PageDir == NULL)
 459       {
 460          ASSERT(FALSE);
 461       }
 462       if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
 463       {
 464          if (Create == FALSE)
 465          {
 466             MmDeleteHyperspaceMapping(PageDir);
 467             return NULL;
 468          }
 469          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 470          if (!NT_SUCCESS(Status) || Pfn == 0)
 471          {
 472             ASSERT(FALSE);
 473          }
 474          Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
 475          if (Entry != 0)
 476          {
 477             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 478             Pfn = PTE_TO_PFN(Entry);
 479          }
 480       }
 481       else
 482       {
 483          Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
 484       }
 485       MmDeleteHyperspaceMapping(PageDir);
 486       Pt = MmCreateHyperspaceMapping(Pfn);
 487       if (Pt == NULL)
 488       {
 489          ASSERT(FALSE);
 490       }
 491       return Pt + ADDR_TO_PTE_OFFSET(Address);
 492    }
 493    PageDir = ADDR_TO_PDE(Address);
 494    if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
 495    {
 496       if (Address >= MmSystemRangeStart)
 497       {
 498          if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
 499          {
 500             if (Create == FALSE)
 501             {
 502                return NULL;
 503             }
 504             Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 505             if (!NT_SUCCESS(Status) || Pfn == 0)
 506             {
 507                ASSERT(FALSE);
 508             }
 509             Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
 510             if (Ke386GlobalPagesEnabled)
 511             {
 512                Entry |= PA_GLOBAL;
 513             }
 514             if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
 515             {
 516                MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 517             }
 518          }
 519          (void)InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
 520       }
 521       else
 522       {
 523          if (Create == FALSE)
 524          {
 525             return NULL;
 526          }
 527          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 528          if (!NT_SUCCESS(Status) || Pfn == 0)
 529          {
 530             ASSERT(FALSE);
 531          }
 532          Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
 533          if (Entry != 0)
 534          {
 535             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 536          }
 537       }
 538    }
 539    return (PULONG)ADDR_TO_PTE(Address);
 540 }
 541
 542 BOOLEAN MmUnmapPageTable(PULONG Pt)
 543 {
 544    if (Ke386Pae)
 545    {
 546       if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
 547       {
 548          return TRUE;
 549       }
 550    }
 551    else
 552    {
 553       if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
 554       {
 555          return TRUE;
 556       }
 557    }
 558    if (Pt)
 559    {
 560       MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
 561    }
 562    return FALSE;
 563 }
 564
 565 static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
 566 {
 567    ULONGLONG Pte;
 568    PULONGLONG Pt;
 569
 570    Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 571    if (Pt)
 572    {
 573       Pte = *Pt;
 574       MmUnmapPageTable((PULONG)Pt);
 575       return Pte;
 576    }
 577    return 0;
 578 }
 579
 580 static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
 581 {
 582    ULONG Pte;
 583    PULONG Pt;
 584
 585    Pt = MmGetPageTableForProcess(Process, Address, FALSE);
 586    if (Pt)
 587    {
 588       Pte = *Pt;
 589       MmUnmapPageTable(Pt);
 590       return Pte;
 591    }
 592    return 0;
 593 }
 594
 595 PFN_NUMBER
 596 NTAPI
 597 MmGetPfnForProcess(PEPROCESS Process,
 598                    PVOID Address)
 599 {
 600
 601    if (Ke386Pae)
 602    {
 603       ULONGLONG Entry;
 604       Entry = MmGetPageEntryForProcessForPAE(Process, Address);
 605       if (!(Entry & PA_PRESENT))
 606       {
 607          return 0;
 608       }
 609       return(PAE_PTE_TO_PFN(Entry));
 610    }
 611    else
 612    {
 613       ULONG Entry;
 614       Entry = MmGetPageEntryForProcess(Process, Address);
 615       if (!(Entry & PA_PRESENT))
 616       {
 617          return 0;
 618       }
 619       return(PTE_TO_PFN(Entry));
 620    }
 621 }
 622
 623 VOID
 624 NTAPI
 625 MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address,
 626                        BOOLEAN* WasDirty, PPFN_NUMBER Page)
 627 /*
 628  * FUNCTION: Delete a virtual mapping
 629  */
 630 {
 631    BOOLEAN WasValid = FALSE;
 632    PFN_NUMBER Pfn;
 633
 634    DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
 635           Process, Address, WasDirty, Page);
 636    if (Ke386Pae)
 637    {
 638       ULONGLONG Pte;
 639       PULONGLONG Pt;
 640
 641       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 642       if (Pt == NULL)
 643       {
 644          if (WasDirty != NULL)
 645          {
 646             *WasDirty = FALSE;
 647          }
 648          if (Page != NULL)
 649          {
 650             *Page = 0;
 651          }
 652          return;
 653       }
 654
 655       /*
 656        * Atomically set the entry to zero and get the old value.
 657        */
 658       Pte = 0LL;
 659       Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
 660
 661       MiFlushTlb((PULONG)Pt, Address);
 662
 663       WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
 664       if (WasValid)
 665       {
 666          Pfn = PAE_PTE_TO_PFN(Pte);
 667          MmMarkPageUnmapped(Pfn);
 668       }
 669       else
 670       {
 671          Pfn = 0;
 672       }
 673
 674       /*
 675        * Return some information to the caller
 676        */
 677       if (WasDirty != NULL)
 678       {
 679          *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
 680       }
 681       if (Page != NULL)
 682       {
 683          *Page = Pfn;
 684       }
 685    }
 686    else
 687    {
 688       ULONG Pte;
 689       PULONG Pt;
 690
 691       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
 692
 693       if (Pt == NULL)
 694       {
 695          if (WasDirty != NULL)
 696          {
 697             *WasDirty = FALSE;
 698          }
 699          if (Page != NULL)
 700          {
 701             *Page = 0;
 702          }
 703          return;
 704       }
 705
 706       /*
 707        * Atomically set the entry to zero and get the old value.
 708        */
 709       Pte = InterlockedExchangeUL(Pt, 0);
 710
 711       MiFlushTlb(Pt, Address);
 712
 713       WasValid = (PAGE_MASK(Pte) != 0);
 714       if (WasValid)
 715       {
 716          Pfn = PTE_TO_PFN(Pte);
 717          MmMarkPageUnmapped(Pfn);
 718       }
 719       else
 720       {
 721          Pfn = 0;
 722       }
 723
 724       /*
 725        * Return some information to the caller
 726        */
 727       if (WasDirty != NULL)
 728       {
 729          *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
 730       }
 731       if (Page != NULL)
 732       {
 733          *Page = Pfn;
 734       }
 735    }
 736    /*
 737     * Decrement the reference count for this page table.
 738     */
 739    if (Process != NULL && WasValid &&
 740        ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
 741        Address < MmSystemRangeStart)
 742    {
 743       PUSHORT Ptrc;
 744       ULONG Idx;
 745
 746       Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
 747       Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
 748
 749       Ptrc[Idx]--;
 750       if (Ptrc[Idx] == 0)
 751       {
 752          MmFreePageTable(Process, Address);
 753       }
 754    }
 755 }
 756
 757 VOID
 758 NTAPI
 759 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
 760                         SWAPENTRY* SwapEntry)
 761 /*
 762  * FUNCTION: Delete a virtual mapping
 763  */
 764 {
 765    if (Ke386Pae)
 766    {
 767       ULONGLONG Pte;
 768       PULONGLONG Pt;
 769
 770       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 771       if (Pt == NULL)
 772       {
 773          *SwapEntry = 0;
 774          return;
 775       }
 776
 777       /*
 778        * Atomically set the entry to zero and get the old value.
 779        */
 780       Pte = 0LL;
 781       Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
 782
 783       MiFlushTlb((PULONG)Pt, Address);
 784
 785       /*
 786        * Decrement the reference count for this page table.
 787        */
 788       if (Process != NULL && Pte &&
 789           ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
 790           Address < MmSystemRangeStart)
 791       {
 792          PUSHORT Ptrc;
 793
 794          Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
 795
 796          Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
 797          if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
 798          {
 799             MmFreePageTable(Process, Address);
 800          }
 801       }
 802
 803
 804       /*
 805        * Return some information to the caller
 806        */
 807       *SwapEntry = Pte >> 1;
 808    }
 809    else
 810    {
 811       ULONG Pte;
 812       PULONG Pt;
 813
 814       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
 815
 816       if (Pt == NULL)
 817       {
 818          *SwapEntry = 0;
 819          return;
 820       }
 821
 822       /*
 823        * Atomically set the entry to zero and get the old value.
 824        */
 825       Pte = InterlockedExchangeUL(Pt, 0);
 826
 827       MiFlushTlb(Pt, Address);
 828
 829       /*
 830        * Decrement the reference count for this page table.
 831        */
 832       if (Process != NULL && Pte &&
 833           ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
 834           Address < MmSystemRangeStart)
 835       {
 836          PUSHORT Ptrc;
 837
 838          Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
 839
 840          Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
 841          if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
 842          {
 843             MmFreePageTable(Process, Address);
 844          }
 845       }
 846
 847
 848       /*
 849        * Return some information to the caller
 850        */
 851       *SwapEntry = Pte >> 1;
 852    }
 853 }
 854
 855 BOOLEAN
 856 Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
 857 {
 858    if (Ke386Pae)
 859    {
 860       PULONGLONG Pt;
 861       PULONGLONG Pde;
 862       Pde = PAE_ADDR_TO_PDE(PAddress);
 863       if (*Pde == 0LL)
 864       {
 865          Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
 866 #if 0
 867          /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
 868          FLASH_TLB_ONE(PAddress);
 869 #endif
 870          if (Pt != NULL)
 871          {
 872             return TRUE;
 873          }
 874       }
 875    }
 876    else
 877    {
 878       PULONG Pt, Pde;
 879       Pde = ADDR_TO_PDE(PAddress);
 880       if (*Pde == 0)
 881       {
 882          Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
 883 #if 0
 884          /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
 885          FLASH_TLB_ONE(PAddress);
 886 #endif
 887          if (Pt != NULL)
 888          {
 889             return TRUE;
 890          }
 891       }
 892    }
 893    return(FALSE);
 894 }
 895
 896 BOOLEAN
 897 NTAPI
 898 MmIsDirtyPage(PEPROCESS Process, PVOID Address)
 899 {
 900    if (Ke386Pae)
 901    {
 902       return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
 903    }
 904    else
 905    {
 906       return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
 907    }
 908 }
 909
 910 VOID
 911 NTAPI
 912 MmSetCleanPage(PEPROCESS Process, PVOID Address)
 913 {
 914    if (Address < MmSystemRangeStart && Process == NULL)
 915    {
 916       DPRINT1("MmSetCleanPage is called for user space without a process.\n");
 917       ASSERT(FALSE);
 918    }
 919    if (Ke386Pae)
 920    {
 921       PULONGLONG Pt;
 922       ULONGLONG Pte;
 923       ULONGLONG tmpPte;
 924
 925       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 926
 927       if (Pt == NULL)
 928       {
 929          ASSERT(FALSE);
 930       }
 931
 932       do
 933       {
 934          Pte = *Pt;
 935          tmpPte = Pte & ~PA_DIRTY;
 936       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
 937
 938       if (Pte & PA_DIRTY)
 939       {
 940          MiFlushTlb((PULONG)Pt, Address);
 941       }
 942       else
 943       {
 944          MmUnmapPageTable((PULONG)Pt);
 945       }
 946    }
 947    else
 948    {
 949       PULONG Pt;
 950       ULONG Pte;
 951
 952       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
 953
 954       if (Pt == NULL)
 955       {
 956          ASSERT(FALSE);
 957       }
 958
 959       do
 960       {
 961          Pte = *Pt;
 962       } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));
 963
 964       if (Pte & PA_DIRTY)
 965       {
 966          MiFlushTlb(Pt, Address);
 967       }
 968       else
 969       {
 970          MmUnmapPageTable(Pt);
 971       }
 972    }
 973 }
 974
 975 VOID
 976 NTAPI
 977 MmSetDirtyPage(PEPROCESS Process, PVOID Address)
 978 {
 979    if (Address < MmSystemRangeStart && Process == NULL)
 980    {
 981       DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
 982       ASSERT(FALSE);
 983    }
 984    if (Ke386Pae)
 985    {
 986       PULONGLONG Pt;
 987       ULONGLONG Pte;
 988       ULONGLONG tmpPte;
 989
 990       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 991       if (Pt == NULL)
 992       {
 993          ASSERT(FALSE);
 994       }
 995
 996       do
 997       {
 998          Pte = *Pt;
 999          tmpPte = Pte | PA_DIRTY;
1000       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1001       if (!(Pte & PA_DIRTY))
1002       {
1003          MiFlushTlb((PULONG)Pt, Address);
1004       }
1005       else
1006       {
1007          MmUnmapPageTable((PULONG)Pt);
1008       }
1009    }
1010    else
1011    {
1012       PULONG Pt;
1013       ULONG Pte;
1014
1015       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1016       if (Pt == NULL)
1017       {
1018          ASSERT(FALSE);
1019       }
1020
1021       do
1022       {
1023          Pte = *Pt;
1024       } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
1025       if (!(Pte & PA_DIRTY))
1026       {
1027          MiFlushTlb(Pt, Address);
1028       }
1029       else
1030       {
1031          MmUnmapPageTable(Pt);
1032       }
1033    }
1034 }
1035
1036 BOOLEAN
1037 NTAPI
1038 MmIsPagePresent(PEPROCESS Process, PVOID Address)
1039 {
1040    if (Ke386Pae)
1041    {
1042       return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1043    }
1044    else
1045    {
1046       return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1047    }
1048 }
1049
1050 BOOLEAN
1051 NTAPI
1052 MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
1053 {
1054    if (Ke386Pae)
1055    {
1056       ULONGLONG Entry;
1057       Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1058       return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1059    }
1060    else
1061    {
1062       ULONG Entry;
1063       Entry = MmGetPageEntryForProcess(Process, Address);
1064       return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1065    }
1066 }
1067
1068 NTSTATUS
1069 NTAPI
1070 MmCreatePageFileMapping(PEPROCESS Process,
1071                         PVOID Address,
1072                         SWAPENTRY SwapEntry)
1073 {
1074    if (Process == NULL && Address < MmSystemRangeStart)
1075    {
1076       DPRINT1("No process\n");
1077       ASSERT(FALSE);
1078    }
1079    if (Process != NULL && Address >= MmSystemRangeStart)
1080    {
1081       DPRINT1("Setting kernel address with process context\n");
1082       ASSERT(FALSE);
1083    }
1084    if (SwapEntry & (1 << 31))
1085    {
1086       ASSERT(FALSE);
1087    }
1088
1089    if (Ke386Pae)
1090    {
1091       PULONGLONG Pt;
1092       ULONGLONG Pte;
1093       ULONGLONG tmpPte;
1094
1095       Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
1096       if (Pt == NULL)
1097       {
1098          ASSERT(FALSE);
1099       }
1100       tmpPte = SwapEntry << 1;
1101       Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1102       if (PAE_PAGE_MASK((Pte)) != 0)
1103       {
1104          MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1105       }
1106
1107       if (Pte != 0)
1108       {
1109          MiFlushTlb((PULONG)Pt, Address);
1110       }
1111       else
1112       {
1113          MmUnmapPageTable((PULONG)Pt);
1114       }
1115    }
1116    else
1117    {
1118       PULONG Pt;
1119       ULONG Pte;
1120
1121       Pt = MmGetPageTableForProcess(Process, Address, TRUE);
1122       if (Pt == NULL)
1123       {
1124          ASSERT(FALSE);
1125       }
1126       Pte = *Pt;
1127       if (PAGE_MASK((Pte)) != 0)
1128       {
1129          MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1130       }
1131       (void)InterlockedExchangeUL(Pt, SwapEntry << 1);
1132       if (Pte != 0)
1133       {
1134          MiFlushTlb(Pt, Address);
1135       }
1136       else
1137       {
1138          MmUnmapPageTable(Pt);
1139       }
1140    }
1141    if (Process != NULL &&
1142        ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1143        Address < MmSystemRangeStart)
1144    {
1145      PUSHORT Ptrc;
1146      ULONG Idx;
1147
1148      Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1149      Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1150      Ptrc[Idx]++;
1151    }
1152    return(STATUS_SUCCESS);
1153 }
1154
1155
1156 NTSTATUS
1157 NTAPI
1158 MmCreateVirtualMappingUnsafe(PEPROCESS Process,
1159                              PVOID Address,
1160                              ULONG flProtect,
1161                              PPFN_NUMBER Pages,
1162                              ULONG PageCount)
1163 {
1164    ULONG Attributes;
1165    PVOID Addr;
1166    ULONG i;
1167    ULONG oldPdeOffset, PdeOffset;
1168    BOOLEAN NoExecute = FALSE;
1169
1170    DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1171           Process, Address, flProtect, Pages, *Pages, PageCount);
1172
1173    if (Process == NULL)
1174    {
1175       if (Address < MmSystemRangeStart)
1176       {
1177          DPRINT1("No process\n");
1178          ASSERT(FALSE);
1179       }
1180       if (PageCount > 0x10000 ||
1181           (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
1182       {
1183          DPRINT1("Page count to large\n");
1184          ASSERT(FALSE);
1185       }
1186    }
1187    else
1188    {
1189       if (Address >= MmSystemRangeStart)
1190       {
1191          DPRINT1("Setting kernel address with process context\n");
1192          ASSERT(FALSE);
1193       }
1194       if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE ||
1195           (ULONG_PTR) Address / PAGE_SIZE + PageCount >
1196           (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE)
1197       {
1198          DPRINT1("Page Count to large\n");
1199          ASSERT(FALSE);
1200       }
1201    }
1202
1203    Attributes = ProtectToPTE(flProtect);
1204    if (Attributes & 0x80000000)
1205    {
1206       NoExecute = TRUE;
1207    }
1208    Attributes &= 0xfff;
1209    if (Address >= MmSystemRangeStart)
1210    {
1211       Attributes &= ~PA_USER;
1212       if (Ke386GlobalPagesEnabled)
1213       {
1214          Attributes |= PA_GLOBAL;
1215       }
1216    }
1217    else
1218    {
1219       Attributes |= PA_USER;
1220    }
1221
1222    Addr = Address;
1223
1224    if (Ke386Pae)
1225    {
1226       ULONGLONG Pte, tmpPte;
1227       PULONGLONG Pt = NULL;
1228
1229       oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1230       for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1231       {
1232          if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1233          {
1234             DPRINT1("Setting physical address but not allowing access at address "
1235                     "0x%.8X with attributes %x/%x.\n",
1236                     Addr, Attributes, flProtect);
1237             ASSERT(FALSE);
1238          }
1239          PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1240          if (oldPdeOffset != PdeOffset)
1241          {
1242             MmUnmapPageTable((PULONG)Pt);
1243             Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
1244             if (Pt == NULL)
1245             {
1246                ASSERT(FALSE);
1247             }
1248          }
1249          else
1250          {
1251             Pt++;
1252          }
1253          oldPdeOffset = PdeOffset;
1254
1255          MmMarkPageMapped(Pages[i]);
1256          tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
1257          if (NoExecute)
1258          {
1259             tmpPte |= 0x8000000000000000LL;
1260          }
1261          Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1262          if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
1263          {
1264             ASSERT(FALSE);
1265          }
1266          if (PAE_PAGE_MASK((Pte)) != 0LL)
1267          {
1268             MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1269          }
1270          if (Address < MmSystemRangeStart &&
1271              ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1272              Attributes & PA_PRESENT)
1273          {
1274             PUSHORT Ptrc;
1275
1276             Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1277
1278             Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
1279          }
1280          if (Pte != 0LL)
1281          {
1282             if (Address > MmSystemRangeStart ||
1283                 (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
1284             {
1285               MiFlushTlb((PULONG)Pt, Address);
1286             }
1287          }
1288       }
1289       if (Addr > Address)
1290       {
1291          MmUnmapPageTable((PULONG)Pt);
1292       }
1293    }
1294    else
1295    {
1296       PULONG Pt = NULL;
1297       ULONG Pte;
1298       oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
1299       for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1300       {
1301          if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1302          {
1303             DPRINT1("Setting physical address but not allowing access at address "
1304                     "0x%.8X with attributes %x/%x.\n",
1305                     Addr, Attributes, flProtect);
1306             ASSERT(FALSE);
1307          }
1308          PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1309          if (oldPdeOffset != PdeOffset)
1310          {
1311             MmUnmapPageTable(Pt);
1312             Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
1313             if (Pt == NULL)
1314             {
1315                ASSERT(FALSE);
1316             }
1317          }
1318          else
1319          {
1320             Pt++;
1321          }
1322          oldPdeOffset = PdeOffset;
1323
1324          Pte = *Pt;
1325          MmMarkPageMapped(Pages[i]);
1326          if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
1327          {
1328             ASSERT(FALSE);
1329          }
1330          if (PAGE_MASK((Pte)) != 0)
1331          {
1332             MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1333          }
1334          (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1335          if (Address < MmSystemRangeStart &&
1336              ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1337              Attributes & PA_PRESENT)
1338          {
1339             PUSHORT Ptrc;
1340
1341             Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1342
1343             Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
1344          }
1345          if (Pte != 0)
1346          {
1347             if (Address > MmSystemRangeStart ||
1348                 (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
1349             {
1350                MiFlushTlb(Pt, Address);
1351             }
1352          }
1353       }
1354       if (Addr > Address)
1355       {
1356          MmUnmapPageTable(Pt);
1357       }
1358    }
1359    return(STATUS_SUCCESS);
1360 }
1361
1362 NTSTATUS
1363 NTAPI
1364 MmCreateVirtualMapping(PEPROCESS Process,
1365                        PVOID Address,
1366                        ULONG flProtect,
1367                        PPFN_NUMBER Pages,
1368                        ULONG PageCount)
1369 {
1370    ULONG i;
1371
1372    for (i = 0; i < PageCount; i++)
1373    {
1374       if (!MmIsPageInUse(Pages[i]))
1375       {
1376          DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages[i]));
1377          ASSERT(FALSE);
1378       }
1379    }
1380
1381    return(MmCreateVirtualMappingUnsafe(Process,
1382                                        Address,
1383                                        flProtect,
1384                                        Pages,
1385                                        PageCount));
1386 }
1387
1388 ULONG
1389 NTAPI
1390 MmGetPageProtect(PEPROCESS Process, PVOID Address)
1391 {
1392    ULONG Entry;
1393    ULONG Protect;
1394    if (Ke386Pae)
1395    {
1396       Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1397    }
1398    else
1399    {
1400       Entry = MmGetPageEntryForProcess(Process, Address);
1401    }
1402
1403    if (!(Entry & PA_PRESENT))
1404    {
1405       Protect = PAGE_NOACCESS;
1406    }
1407    else
1408    {
1409       if (Entry & PA_READWRITE)
1410       {
1411          Protect = PAGE_READWRITE;
1412       }
1413       else
1414       {
1415          Protect = PAGE_EXECUTE_READ;
1416       }
1417       if (Entry & PA_CD)
1418       {
1419          Protect |= PAGE_NOCACHE;
1420       }
1421       if (Entry & PA_WT)
1422       {
1423          Protect |= PAGE_WRITETHROUGH;
1424       }
1425       if (!(Entry & PA_USER))
1426       {
1427          Protect |= PAGE_SYSTEM;
1428       }
1429
1430    }
1431    return(Protect);
1432 }
1433
1434 VOID
1435 NTAPI
1436 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
1437 {
1438    ULONG Attributes = 0;
1439    BOOLEAN NoExecute = FALSE;
1440
1441    DPRINT("MmSetPageProtect(Process %x  Address %x  flProtect %x)\n",
1442           Process, Address, flProtect);
1443
1444    Attributes = ProtectToPTE(flProtect);
1445    if (Attributes & 0x80000000)
1446    {
1447       NoExecute = TRUE;
1448    }
1449    Attributes &= 0xfff;
1450    if (Address >= MmSystemRangeStart)
1451    {
1452       Attributes &= ~PA_USER;
1453       if (Ke386GlobalPagesEnabled)
1454       {
1455          Attributes |= PA_GLOBAL;
1456       }
1457    }
1458    else
1459    {
1460       Attributes |= PA_USER;
1461    }
1462    if (Ke386Pae)
1463    {
1464       PULONGLONG Pt;
1465       ULONGLONG tmpPte, Pte;
1466
1467       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1468       if (Pt == NULL)
1469       {
1470          DPRINT1("Address %x\n", Address);
1471          ASSERT(FALSE);
1472       }
1473       do
1474       {
1475         Pte = *Pt;
1476         tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
1477         if (NoExecute)
1478         {
1479            tmpPte |= 0x8000000000000000LL;
1480         }
1481         else
1482         {
1483            tmpPte &= ~0x8000000000000000LL;
1484         }
1485       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1486
1487       MiFlushTlb((PULONG)Pt, Address);
1488    }
1489    else
1490    {
1491       PULONG Pt;
1492
1493       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1494       if (Pt == NULL)
1495       {
1496          ASSERT(FALSE);
1497       }
1498       InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
1499       MiFlushTlb(Pt, Address);
1500    }
1501 }
1502
1503 PVOID
1504 NTAPI
1505 MmCreateHyperspaceMapping(PFN_NUMBER Page)
1506 {
1507    PVOID Address;
1508    ULONG i;
1509
1510    if (Ke386Pae)
1511    {
1512       ULONGLONG Entry;
1513       ULONGLONG ZeroEntry = 0LL;
1514       PULONGLONG Pte;
1515
1516       Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
1517       Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
1518
1519       if (Page & 1024)
1520       {
1521          for (i = Page %1024; i < 1024; i++, Pte++)
1522          {
1523             if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
1524             {
1525                break;
1526             }
1527          }
1528          if (i >= 1024)
1529          {
1530             Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
1531             for (i = 0; i < Page % 1024; i++, Pte++)
1532             {
1533                if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
1534                {
1535                   break;
1536                }
1537             }
1538             if (i >= Page % 1024)
1539             {
1540                ASSERT(FALSE);
1541             }
1542          }
1543       }
1544       else
1545       {
1546          for (i = Page %1024; (LONG)i >= 0; i--, Pte--)
1547          {
1548             if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
1549             {
1550                break;
1551             }
1552          }
1553          if ((LONG)i < 0)
1554          {
1555             Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
1556             for (i = 1023; i > Page % 1024; i--, Pte--)
1557             {
1558                if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
1559                {
1560                   break;
1561                }
1562             }
1563             if (i <= Page % 1024)
1564             {
1565                ASSERT(FALSE);
1566             }
1567          }
1568       }
1569    }
1570    else
1571    {
1572       ULONG Entry;
1573       PULONG Pte;
1574       Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
1575       Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
1576       if (Page & 1024)
1577       {
1578          for (i = Page % 1024; i < 1024; i++, Pte++)
1579          {
1580             if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
1581             {
1582                break;
1583             }
1584          }
1585          if (i >= 1024)
1586          {
1587             Pte = ADDR_TO_PTE(HYPERSPACE);
1588             for (i = 0; i < Page % 1024; i++, Pte++)
1589             {
1590                if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
1591                {
1592                   break;
1593                }
1594             }
1595             if (i >= Page % 1024)
1596             {
1597                ASSERT(FALSE);
1598             }
1599          }
1600       }
1601       else
1602       {
1603          for (i = Page % 1024; (LONG)i >= 0; i--, Pte--)
1604          {
1605             if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
1606             {
1607                break;
1608             }
1609          }
1610          if ((LONG)i < 0)
1611          {
1612             Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
1613             for (i = 1023; i > Page % 1024; i--, Pte--)
1614             {
1615                if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
1616                {
1617                   break;
1618                }
1619             }
1620             if (i <= Page % 1024)
1621             {
1622                ASSERT(FALSE);
1623             }
1624          }
1625       }
1626    }
1627    Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE);
1628    __invlpg(Address);
1629    return Address;
1630 }
1631
1632 PFN_NUMBER
1633 NTAPI
1634 MmDeleteHyperspaceMapping(PVOID Address)
1635 {
1636    PFN_NUMBER Pfn;
1637    ASSERT (IS_HYPERSPACE(Address));
1638    if (Ke386Pae)
1639    {
1640       ULONGLONG Entry = 0LL;
1641       Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
1642       Pfn = PAE_PTE_TO_PFN(Entry);
1643    }
1644    else
1645    {
1646       ULONG Entry;
1647       Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0);
1648       Pfn = PTE_TO_PFN(Entry);
1649    }
1650    __invlpg(Address);
1651    return Pfn;
1652 }
1653
1654 VOID
1655 INIT_FUNCTION
1656 NTAPI
1657 MmInitGlobalKernelPageDirectory(VOID)
1658 {
1659    ULONG i;
1660
1661    DPRINT("MmInitGlobalKernelPageDirectory()\n");
1662
1663    if (Ke386Pae)
1664    {
1665       PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
1666       for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++)
1667       {
1668          if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
1669              !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
1670              0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
1671          {
1672             (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
1673             if (Ke386GlobalPagesEnabled)
1674             {
1675                MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
1676                CurrentPageDirectory[i] |= PA_GLOBAL;
1677             }
1678          }
1679       }
1680    }
1681    else
1682    {
1683       PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
1684       for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++)
1685       {
1686          if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
1687              i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
1688              0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
1689          {
1690             MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
1691             if (Ke386GlobalPagesEnabled)
1692             {
1693                MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
1694                CurrentPageDirectory[i] |= PA_GLOBAL;
1695             }
1696          }
1697       }
1698    }
1699 }
1700
1701 /* EOF */