reactos/ntoskrnl/mm/i386/page.c

   1 /* $Id$
   2  *
   3  * COPYRIGHT:       See COPYING in the top level directory
   4  * PROJECT:         ReactOS kernel
   5  * FILE:            ntoskrnl/mm/i386/page.c
   6  * PURPOSE:         Low level memory managment manipulation
   7  *
   8  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   9  */
  10
  11 /* INCLUDES ***************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <internal/debug.h>
  16
  17 /* GLOBALS *****************************************************************/
  18
  19 #define PA_BIT_PRESENT   (0)
  20 #define PA_BIT_READWRITE (1)
  21 #define PA_BIT_USER      (2)
  22 #define PA_BIT_WT        (3)
  23 #define PA_BIT_CD        (4)
  24 #define PA_BIT_ACCESSED  (5)
  25 #define PA_BIT_DIRTY     (6)
  26 #define PA_BIT_GLOBAL    (8)
  27
  28 #define PA_PRESENT   (1 << PA_BIT_PRESENT)
  29 #define PA_READWRITE (1 << PA_BIT_READWRITE)
  30 #define PA_USER      (1 << PA_BIT_USER)
  31 #define PA_DIRTY     (1 << PA_BIT_DIRTY)
  32 #define PA_WT        (1 << PA_BIT_WT)
  33 #define PA_CD        (1 << PA_BIT_CD)
  34 #define PA_ACCESSED  (1 << PA_BIT_ACCESSED)
  35 #define PA_GLOBAL    (1 << PA_BIT_GLOBAL)
  36
  37 #define PAGETABLE_MAP           (0xf0000000)
  38 #define PAGEDIRECTORY_MAP       (0xf0000000 + (PAGETABLE_MAP / (1024)))
  39
  40 #define PAE_PAGEDIRECTORY_MAP   (0xf0000000 + (PAGETABLE_MAP / (512)))
  41
  42 #define HYPERSPACE              (0xf0800000)
  43 #define IS_HYPERSPACE(v)        (((ULONG)(v) >= 0xF0800000 && (ULONG)(v) < 0xF0C00000))
  44
  45 ULONG MmGlobalKernelPageDirectory[1024];
  46 ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];
  47
  48 #define PTE_TO_PFN(X)  ((X) >> PAGE_SHIFT)
  49 #define PFN_TO_PTE(X)  ((X) << PAGE_SHIFT)
  50
  51 #define PAE_PTE_TO_PFN(X)   (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
  52 #define PAE_PFN_TO_PTE(X)   ((X) << PAGE_SHIFT)
  53
  54 #if defined(__GNUC__)
  55 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
  56 #else
  57 __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage)
  58 {
  59    LARGE_INTEGER dummy;
  60    dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage));
  61    return dummy;
  62 }
  63 #endif
  64
  65 extern BOOLEAN Ke386Pae;
  66 extern BOOLEAN Ke386NoExecute;
  67 extern BOOLEAN Ke386GlobalPagesEnabled;
  68
  69 /* FUNCTIONS ***************************************************************/
  70
  71 BOOLEAN MmUnmapPageTable(PULONG Pt);
  72
  73 VOID
  74 STDCALL
  75 MiFlushTlbIpiRoutine(PVOID Address)
  76 {
  77    if (Address == (PVOID)0xffffffff)
  78    {
  79       KeFlushCurrentTb();
  80    }
  81    else if (Address == (PVOID)0xfffffffe)
  82    {
  83       FLUSH_TLB;
  84    }
  85    else
  86    {
  87       FLUSH_TLB_ONE(Address);
  88    }
  89 }
  90
  91 VOID
  92 MiFlushTlb(PULONG Pt, PVOID Address)
  93 {
  94 #ifdef CONFIG_SMP
  95    if (Pt)
  96    {
  97       MmUnmapPageTable(Pt);
  98    }
  99    if (KeNumberProcessors>1)
 100    {
 101       KeIpiGenericCall(MiFlushTlbIpiRoutine, Address);
 102    }
 103    else
 104    {
 105       MiFlushTlbIpiRoutine(Address);
 106    }
 107 #else
 108    if ((Pt && MmUnmapPageTable(Pt)) || Address >= (PVOID)KERNEL_BASE)
 109    {
 110       FLUSH_TLB_ONE(Address);
 111    }
 112 #endif
 113 }
 114
 115
 116
 117 PULONG
 118 MmGetPageDirectory(VOID)
 119 {
 120    unsigned int page_dir=0;
 121    Ke386GetPageTableDirectory(page_dir);
 122    return((PULONG)page_dir);
 123 }
 124
 125 static ULONG
 126 ProtectToPTE(ULONG flProtect)
 127 {
 128    ULONG Attributes = 0;
 129
 130    if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
 131    {
 132       Attributes = 0;
 133    }
 134    else if (flProtect & PAGE_IS_WRITABLE)
 135    {
 136       Attributes = PA_PRESENT | PA_READWRITE;
 137    }
 138    else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
 139    {
 140       Attributes = PA_PRESENT;
 141    }
 142    else
 143    {
 144       DPRINT1("Unknown main protection type.\n");
 145       KEBUGCHECK(0);
 146    }
 147    if (Ke386NoExecute &&
 148        !(flProtect & PAGE_IS_EXECUTABLE))
 149    {
 150       Attributes = Attributes | 0x80000000;
 151    }
 152
 153    if (flProtect & PAGE_SYSTEM)
 154    {
 155    }
 156    else
 157    {
 158       Attributes = Attributes | PA_USER;
 159    }
 160    if (flProtect & PAGE_NOCACHE)
 161    {
 162       Attributes = Attributes | PA_CD;
 163    }
 164    if (flProtect & PAGE_WRITETHROUGH)
 165    {
 166       Attributes = Attributes | PA_WT;
 167    }
 168    return(Attributes);
 169 }
 170
 171 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
 172
 173 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
 174                                 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
 175 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
 176
 177 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
 178
 179 #define ADDR_TO_PTE_OFFSET(v)  ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
 180
 181
 182 #define PAE_ADDR_TO_PAGE_TABLE(v)   (((ULONG)(v)) / (512 * PAGE_SIZE))
 183
 184 #define PAE_ADDR_TO_PDE(v)          (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
 185                                                   ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
 186 #define PAE_ADDR_TO_PTE(v)          (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
 187
 188
 189 #define PAE_ADDR_TO_PDTE_OFFSET(v)  (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
 190
 191 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v)   ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
 192
 193 #define PAE_ADDR_TO_PDE_OFFSET(v)   (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
 194
 195 #define PAE_ADDR_TO_PTE_OFFSET(v)   ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
 196
 197
 198 NTSTATUS Mmi386ReleaseMmInfo(PEPROCESS Process)
 199 {
 200    PUSHORT LdtDescriptor;
 201    ULONG LdtBase;
 202    ULONG i, j;
 203
 204    DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process);
 205
 206    LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor[0];
 207    LdtBase = LdtDescriptor[1] |
 208              ((LdtDescriptor[2] & 0xff) << 16) |
 209              ((LdtDescriptor[3] & ~0xff) << 16);
 210
 211    DPRINT("LdtBase: %x\n", LdtBase);
 212
 213    if (LdtBase)
 214    {
 215       ExFreePool((PVOID) LdtBase);
 216    }
 217
 218    if (Ke386Pae)
 219    {
 220       PULONGLONG PageDirTable;
 221       PULONGLONG PageDir;
 222       PULONGLONG Pde;
 223       ULONG k;
 224
 225       PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
 226       for (i = 0; i < 4; i++)
 227       {
 228          PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i]));
 229          if (i < PAE_ADDR_TO_PDTE_OFFSET(KERNEL_BASE))
 230          {
 231             for (j = 0; j < 512; j++)
 232             {
 233                if (PageDir[j] != 0LL)
 234                {
 235                   DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
 236                           Process->UniqueProcessId,
 237                           (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1,
 238                           Process->AddressSpace.PageTableRefCountTable[i*512 + j]);
 239                   Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j]));
 240                   for (k = 0; k < 512; k++)
 241                   {
 242                      if(Pde[k] != 0)
 243                      {
 244                         if (Pde[k] & PA_PRESENT)
 245                         {
 246                            DPRINT1("Page at %08x is not freed\n",
 247                                    (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
 248                         }
 249                         else
 250                         {
 251                            DPRINT1("Swapentry %x at %x is not freed\n",
 252                                    (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
 253                         }
 254                      }
 255                   }
 256                   MmDeleteHyperspaceMapping(Pde);
 257                   MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j]));
 258                }
 259             }
 260          }
 261          if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE))
 262          {
 263             MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)]));
 264             MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1]));
 265          }
 266          MmDeleteHyperspaceMapping(PageDir);
 267          MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i]));
 268       }
 269       MmDeleteHyperspaceMapping((PVOID)PageDirTable);
 270       MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
 271    }
 272    else
 273    {
 274       PULONG Pde;
 275       PULONG PageDir;
 276       PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
 277       for (i = 0; i < ADDR_TO_PDE_OFFSET(KERNEL_BASE); i++)
 278       {
 279          if (PageDir[i] != 0)
 280          {
 281             DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
 282                     i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1,
 283                     Process->AddressSpace.PageTableRefCountTable[i]);
 284             Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i]));
 285             for (j = 0; j < 1024; j++)
 286             {
 287                if(Pde[j] != 0)
 288                {
 289                   if (Pde[j] & PA_PRESENT)
 290                   {
 291                      DPRINT1("Page at %08x is not freed\n",
 292                              i * 4 * 1024 * 1024 + j * PAGE_SIZE);
 293                   }
 294                   else
 295                   {
 296                      DPRINT1("Swapentry %x at %x is not freed\n",
 297                              Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE);
 298                   }
 299                }
 300             }
 301             MmDeleteHyperspaceMapping(Pde);
 302             MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i]));
 303          }
 304       }
 305       MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)]));
 306       MmDeleteHyperspaceMapping(PageDir);
 307       MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
 308    }
 309
 310 #if defined(__GNUC__)
 311
 312    Process->Pcb.DirectoryTableBase.QuadPart = 0LL;
 313 #else
 314
 315    Process->Pcb.DirectoryTableBase.QuadPart = 0;
 316 #endif
 317
 318    DPRINT("Finished Mmi386ReleaseMmInfo()\n");
 319    return(STATUS_SUCCESS);
 320 }
 321
 322 NTSTATUS MmCopyMmInfo(PEPROCESS Src, PEPROCESS Dest)
 323 {
 324    PKPROCESS KProcess = &Dest->Pcb;
 325    NTSTATUS Status;
 326    ULONG i, j;
 327    PFN_TYPE Pfn[7];
 328    ULONG Count;
 329
 330    DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", Src, Dest);
 331
 332    Count = Ke386Pae ? 7 : 2;
 333
 334    for (i = 0; i < Count; i++)
 335    {
 336       Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
 337       if (!NT_SUCCESS(Status))
 338       {
 339          for (j = 0; j < i; j++)
 340          {
 341             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
 342          }
 343          return Status;
 344       }
 345    }
 346
 347    if (Ke386Pae)
 348    {
 349       PULONGLONG PageDirTable;
 350       PULONGLONG PageDir;
 351
 352       PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
 353       for (i = 0; i < 4; i++)
 354       {
 355          PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
 356       }
 357       MmDeleteHyperspaceMapping(PageDirTable);
 358       for (i = PAE_ADDR_TO_PDTE_OFFSET(KERNEL_BASE); i < 4; i++)
 359       {
 360          PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
 361          memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
 362          if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
 363          {
 364             for (j = 0; j < 4; j++)
 365             {
 366                PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
 367             }
 368          }
 369          if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
 370          {
 371             PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
 372             PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
 373          }
 374          MmDeleteHyperspaceMapping(PageDir);
 375       }
 376    }
 377    else
 378    {
 379       PULONG PageDirectory;
 380       PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);
 381
 382       memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(KERNEL_BASE),
 383              MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(KERNEL_BASE),
 384              (1024 - ADDR_TO_PDE_OFFSET(KERNEL_BASE)) * sizeof(ULONG));
 385
 386       DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
 387       PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
 388       PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;
 389
 390       MmDeleteHyperspaceMapping(PageDirectory);
 391    }
 392    KProcess->DirectoryTableBase.QuadPart = PFN_TO_PTE(Pfn[0]);
 393    DPRINT("Finished MmCopyMmInfo()\n");
 394    return(STATUS_SUCCESS);
 395 }
 396
 397 VOID MmDeletePageTable(PEPROCESS Process, PVOID Address)
 398 {
 399    PEPROCESS CurrentProcess = PsGetCurrentProcess();
 400
 401    if (Process != NULL && Process != CurrentProcess)
 402    {
 403       KeAttachProcess(&Process->Pcb);
 404    }
 405
 406    if (Ke386Pae)
 407    {
 408       ULONGLONG ZeroPde = 0LL;
 409       ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde);
 410    }
 411    else
 412    {
 413       *(ADDR_TO_PDE(Address)) = 0;
 414    }
 415    if (Address >= (PVOID)KERNEL_BASE)
 416    {
 417       KEBUGCHECK(0);
 418       //       MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
 419    }
 420    MiFlushTlb(NULL, Address);
 421    if (Process != NULL && Process != CurrentProcess)
 422    {
 423       KeDetachProcess();
 424    }
 425 }
 426
 427 VOID MmFreePageTable(PEPROCESS Process, PVOID Address)
 428 {
 429    PEPROCESS CurrentProcess = PsGetCurrentProcess();
 430    ULONG i;
 431    PFN_TYPE Pfn;
 432
 433    DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
 434    if (Process != NULL && Process != CurrentProcess)
 435    {
 436       KeAttachProcess(&Process->Pcb);
 437    }
 438    if (Ke386Pae)
 439    {
 440       PULONGLONG PageTable;
 441       ULONGLONG ZeroPte = 0LL;
 442       PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
 443       for (i = 0; i < 512; i++)
 444       {
 445          if (PageTable[i] != 0LL)
 446          {
 447             DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
 448                      ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
 449             KEBUGCHECK(0);
 450          }
 451       }
 452       Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
 453       ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
 454    }
 455    else
 456    {
 457       PULONG PageTable;
 458       PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
 459       for (i = 0; i < 1024; i++)
 460       {
 461          if (PageTable[i] != 0)
 462          {
 463             DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
 464                      ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
 465             KEBUGCHECK(0);
 466          }
 467       }
 468       Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
 469       *(ADDR_TO_PDE(Address)) = 0;
 470    }
 471    MiFlushTlb(NULL, Address);
 472
 473    if (Address >= (PVOID)KERNEL_BASE)
 474    {
 475       //    MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
 476       KEBUGCHECK(0);
 477    }
 478    else
 479    {
 480       MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 481    }
 482    if (Process != NULL && Process != CurrentProcess)
 483    {
 484       KeDetachProcess();
 485    }
 486 }
 487
 488 static PULONGLONG
 489 MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
 490 {
 491    NTSTATUS Status;
 492    PFN_TYPE Pfn;
 493    ULONGLONG Entry;
 494    ULONGLONG ZeroEntry = 0LL;
 495    PULONGLONG Pt;
 496    PULONGLONG PageDir;
 497    PULONGLONG PageDirTable;
 498
 499    DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
 500           Process, Address, Create);
 501    if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)PAGETABLE_MAP + 0x800000)
 502    {
 503       KEBUGCHECK(0);
 504    }
 505    if (Address < (PVOID)KERNEL_BASE && Process && Process != PsGetCurrentProcess())
 506    {
 507       PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
 508       if (PageDirTable == NULL)
 509       {
 510          KEBUGCHECK(0);
 511       }
 512       PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
 513       MmDeleteHyperspaceMapping(PageDirTable);
 514       if (PageDir == NULL)
 515       {
 516          KEBUGCHECK(0);
 517       }
 518       PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
 519       Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
 520       if (Entry == 0LL)
 521       {
 522          if (Create == FALSE)
 523          {
 524             MmDeleteHyperspaceMapping(PageDir);
 525             return NULL;
 526          }
 527          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 528          if (!NT_SUCCESS(Status))
 529          {
 530             KEBUGCHECK(0);
 531          }
 532          Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
 533          Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
 534          if (Entry != 0LL)
 535          {
 536             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 537             Pfn = PAE_PTE_TO_PFN(Entry);
 538          }
 539       }
 540       else
 541       {
 542          Pfn = PAE_PTE_TO_PFN(Entry);
 543       }
 544       MmDeleteHyperspaceMapping(PageDir);
 545       Pt = MmCreateHyperspaceMapping(Pfn);
 546       if (Pt == NULL)
 547       {
 548          KEBUGCHECK(0);
 549       }
 550       return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
 551    }
 552    PageDir = PAE_ADDR_TO_PDE(Address);
 553    if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
 554    {
 555       if (Address >= (PVOID)KERNEL_BASE)
 556       {
 557          if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
 558          {
 559             if (Create == FALSE)
 560             {
 561                return NULL;
 562             }
 563             Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 564             if (!NT_SUCCESS(Status))
 565             {
 566                KEBUGCHECK(0);
 567             }
 568             Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
 569             if (Ke386GlobalPagesEnabled)
 570             {
 571                Entry |= PA_GLOBAL;
 572             }
 573             if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
 574             {
 575                MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 576             }
 577          }
 578          ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
 579       }
 580       else
 581       {
 582          if (Create == FALSE)
 583          {
 584             return NULL;
 585          }
 586          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 587          if (!NT_SUCCESS(Status))
 588          {
 589             KEBUGCHECK(0);
 590          }
 591          Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
 592          Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
 593          if (Entry != 0LL)
 594          {
 595             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 596          }
 597       }
 598    }
 599    return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
 600 }
 601
 602 static PULONG
 603 MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
 604 {
 605    ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
 606    NTSTATUS Status;
 607    PFN_TYPE Pfn;
 608    ULONG Entry;
 609    PULONG Pt, PageDir;
 610
 611    if (Address < (PVOID)KERNEL_BASE && Process && Process != PsGetCurrentProcess())
 612    {
 613       PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
 614       if (PageDir == NULL)
 615       {
 616          KEBUGCHECK(0);
 617       }
 618       if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
 619       {
 620          if (Create == FALSE)
 621          {
 622             MmDeleteHyperspaceMapping(PageDir);
 623             return NULL;
 624          }
 625          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 626          if (!NT_SUCCESS(Status) || Pfn == 0)
 627          {
 628             KEBUGCHECK(0);
 629          }
 630          Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
 631          if (Entry != 0)
 632          {
 633             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 634             Pfn = PTE_TO_PFN(Entry);
 635          }
 636       }
 637       else
 638       {
 639          Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
 640       }
 641       MmDeleteHyperspaceMapping(PageDir);
 642       Pt = MmCreateHyperspaceMapping(Pfn);
 643       if (Pt == NULL)
 644       {
 645          KEBUGCHECK(0);
 646       }
 647       return Pt + ADDR_TO_PTE_OFFSET(Address);
 648    }
 649    PageDir = ADDR_TO_PDE(Address);
 650    if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
 651    {
 652       if (Address >= (PVOID)KERNEL_BASE)
 653       {
 654          if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
 655          {
 656             if (Create == FALSE)
 657             {
 658                return NULL;
 659             }
 660             Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 661             if (!NT_SUCCESS(Status) || Pfn == 0)
 662             {
 663                KEBUGCHECK(0);
 664             }
 665             Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
 666             if (Ke386GlobalPagesEnabled)
 667             {
 668                Entry |= PA_GLOBAL;
 669             }
 670             if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
 671             {
 672                MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 673             }
 674          }
 675          InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
 676       }
 677       else
 678       {
 679          if (Create == FALSE)
 680          {
 681             return NULL;
 682          }
 683          Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
 684          if (!NT_SUCCESS(Status) || Pfn == 0)
 685          {
 686             KEBUGCHECK(0);
 687          }
 688          Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
 689          if (Entry != 0)
 690          {
 691             MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 692          }
 693       }
 694    }
 695    return (PULONG)ADDR_TO_PTE(Address);
 696 }
 697
 698 BOOLEAN MmUnmapPageTable(PULONG Pt)
 699 {
 700    if (Ke386Pae)
 701    {
 702       if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
 703       {
 704          return TRUE;
 705       }
 706    }
 707    else
 708    {
 709       if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
 710       {
 711          return TRUE;
 712       }
 713    }
 714    if (Pt)
 715    {
 716       MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
 717    }
 718    return FALSE;
 719 }
 720
 721 static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
 722 {
 723    ULONGLONG Pte;
 724    PULONGLONG Pt;
 725
 726    Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 727    if (Pt)
 728    {
 729       Pte = *Pt;
 730       MmUnmapPageTable((PULONG)Pt);
 731       return Pte;
 732    }
 733    return 0;
 734 }
 735
 736 static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
 737 {
 738    ULONG Pte;
 739    PULONG Pt;
 740
 741    Pt = MmGetPageTableForProcess(Process, Address, FALSE);
 742    if (Pt)
 743    {
 744       Pte = *Pt;
 745       MmUnmapPageTable(Pt);
 746       return Pte;
 747    }
 748    return 0;
 749 }
 750
 751 PFN_TYPE
 752 MmGetPfnForProcess(PEPROCESS Process,
 753                    PVOID Address)
 754 {
 755
 756    if (Ke386Pae)
 757    {
 758       ULONGLONG Entry;
 759       Entry = MmGetPageEntryForProcessForPAE(Process, Address);
 760       if (!(Entry & PA_PRESENT))
 761       {
 762          return 0;
 763       }
 764       return(PAE_PTE_TO_PFN(Entry));
 765    }
 766    else
 767    {
 768       ULONG Entry;
 769       Entry = MmGetPageEntryForProcess(Process, Address);
 770       if (!(Entry & PA_PRESENT))
 771       {
 772          return 0;
 773       }
 774       return(PTE_TO_PFN(Entry));
 775    }
 776 }
 777
 778 VOID
 779 MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOL* WasDirty, PPFN_TYPE Page)
 780 /*
 781  * FUNCTION: Delete a virtual mapping
 782  */
 783 {
 784    BOOLEAN WasValid;
 785    if (Ke386Pae)
 786    {
 787       ULONGLONG Pte;
 788       ULONGLONG tmpPte;
 789       PULONGLONG Pt;
 790
 791       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 792       if (Pt == NULL)
 793       {
 794          KEBUGCHECK(0);
 795       }
 796       /*
 797        * Atomically disable the present bit and get the old value.
 798        */
 799       do
 800       {
 801         Pte = *Pt;
 802         tmpPte = Pte & ~PA_PRESENT;
 803       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
 804
 805       MiFlushTlb((PULONG)Pt, Address);
 806       WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE;
 807       if (!WasValid)
 808       {
 809          KEBUGCHECK(0);
 810       }
 811
 812       /*
 813        * Return some information to the caller
 814        */
 815       if (WasDirty != NULL)
 816       {
 817           *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
 818       }
 819       if (Page != NULL)
 820       {
 821          *Page = PAE_PTE_TO_PFN(Pte);
 822       }
 823    }
 824    else
 825    {
 826       ULONG Pte;
 827       PULONG Pt;
 828
 829       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
 830       if (Pt == NULL)
 831       {
 832          KEBUGCHECK(0);
 833       }
 834       /*
 835        * Atomically disable the present bit and get the old value.
 836        */
 837       do
 838       {
 839         Pte = *Pt;
 840       } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte));
 841
 842       MiFlushTlb(Pt, Address);
 843       WasValid = (PAGE_MASK(Pte) != 0);
 844       if (!WasValid)
 845       {
 846          KEBUGCHECK(0);
 847       }
 848
 849       /*
 850        * Return some information to the caller
 851        */
 852       if (WasDirty != NULL)
 853       {
 854          *WasDirty = Pte & PA_DIRTY;
 855       }
 856       if (Page != NULL)
 857       {
 858          *Page = PTE_TO_PFN(Pte);
 859       }
 860    }
 861 }
 862
 863 VOID
 864 MmRawDeleteVirtualMapping(PVOID Address)
 865 {
 866    if (Ke386Pae)
 867    {
 868       PULONGLONG Pt;
 869       ULONGLONG ZeroPte = 0LL;
 870       Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE);
 871       if (Pt)
 872       {
 873          /*
 874           * Set the entry to zero
 875           */
 876          ExfpInterlockedExchange64UL(Pt, &ZeroPte);
 877          MiFlushTlb((PULONG)Pt, Address);
 878       }
 879    }
 880    else
 881    {
 882       PULONG Pt;
 883
 884       Pt = MmGetPageTableForProcess(NULL, Address, FALSE);
 885       if (Pt && *Pt)
 886       {
 887          /*
 888           * Set the entry to zero
 889           */
 890          InterlockedExchangeUL(Pt, 0);
 891          MiFlushTlb(Pt, Address);
 892       }
 893    }
 894 }
 895
 896 VOID
 897 MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOL FreePage,
 898                        BOOL* WasDirty, PPFN_TYPE Page)
 899 /*
 900  * FUNCTION: Delete a virtual mapping
 901  */
 902 {
 903    BOOLEAN WasValid = FALSE;
 904    PFN_TYPE Pfn;
 905
 906    DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
 907           Process, Address, FreePage, WasDirty, Page);
 908    if (Ke386Pae)
 909    {
 910       ULONGLONG Pte;
 911       PULONGLONG Pt;
 912
 913       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
 914       if (Pt == NULL)
 915       {
 916          if (WasDirty != NULL)
 917          {
 918             *WasDirty = FALSE;
 919          }
 920          if (Page != NULL)
 921          {
 922             *Page = 0;
 923          }
 924          return;
 925       }
 926
 927       /*
 928        * Atomically set the entry to zero and get the old value.
 929        */
 930       Pte = 0LL;
 931       Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
 932
 933       MiFlushTlb((PULONG)Pt, Address);
 934
 935       WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
 936       if (WasValid)
 937       {
 938          Pfn = PAE_PTE_TO_PFN(Pte);
 939          MmMarkPageUnmapped(Pfn);
 940       }
 941       else
 942       {
 943          Pfn = 0;
 944       }
 945
 946       if (FreePage && WasValid)
 947       {
 948          MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
 949       }
 950
 951       /*
 952        * Return some information to the caller
 953        */
 954       if (WasDirty != NULL)
 955       {
 956          *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
 957       }
 958       if (Page != NULL)
 959       {
 960          *Page = Pfn;
 961       }
 962    }
 963    else
 964    {
 965       ULONG Pte;
 966       PULONG Pt;
 967
 968       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
 969
 970       if (Pt == NULL)
 971       {
 972          if (WasDirty != NULL)
 973          {
 974             *WasDirty = FALSE;
 975          }
 976          if (Page != NULL)
 977          {
 978             *Page = 0;
 979          }
 980          return;
 981       }
 982
 983       /*
 984        * Atomically set the entry to zero and get the old value.
 985        */
 986       Pte = InterlockedExchangeUL(Pt, 0);
 987
 988       MiFlushTlb(Pt, Address);
 989
 990       WasValid = (PAGE_MASK(Pte) != 0);
 991       if (WasValid)
 992       {
 993          Pfn = PTE_TO_PFN(Pte);
 994          MmMarkPageUnmapped(Pfn);
 995       }
 996       else
 997       {
 998          Pfn = 0;
 999       }
1000
1001       if (FreePage && WasValid)
1002       {
1003          MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
1004       }
1005
1006       /*
1007        * Return some information to the caller
1008        */
1009       if (WasDirty != NULL)
1010       {
1011          *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
1012       }
1013       if (Page != NULL)
1014       {
1015          *Page = Pfn;
1016       }
1017    }
1018    /*
1019     * Decrement the reference count for this page table.
1020     */
1021    if (Process != NULL && WasValid &&
1022        Process->AddressSpace.PageTableRefCountTable != NULL &&
1023        Address < (PVOID)KERNEL_BASE)
1024    {
1025       PUSHORT Ptrc;
1026       ULONG Idx;
1027
1028       Ptrc = Process->AddressSpace.PageTableRefCountTable;
1029       Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1030
1031       Ptrc[Idx]--;
1032       if (Ptrc[Idx] == 0)
1033       {
1034          MmFreePageTable(Process, Address);
1035       }
1036    }
1037 }
1038
1039 VOID
1040 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
1041                         SWAPENTRY* SwapEntry)
1042 /*
1043  * FUNCTION: Delete a virtual mapping
1044  */
1045 {
1046    if (Ke386Pae)
1047    {
1048       ULONGLONG Pte;
1049       PULONGLONG Pt;
1050
1051       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1052       if (Pt == NULL)
1053       {
1054          *SwapEntry = 0;
1055          return;
1056       }
1057
1058       /*
1059        * Atomically set the entry to zero and get the old value.
1060        */
1061       Pte = 0LL;
1062       Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1063
1064       MiFlushTlb((PULONG)Pt, Address);
1065
1066       /*
1067        * Decrement the reference count for this page table.
1068        */
1069       if (Process != NULL && Pte &&
1070           Process->AddressSpace.PageTableRefCountTable != NULL &&
1071           Address < (PVOID)KERNEL_BASE)
1072       {
1073          PUSHORT Ptrc;
1074
1075          Ptrc = Process->AddressSpace.PageTableRefCountTable;
1076
1077          Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
1078          if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
1079          {
1080             MmFreePageTable(Process, Address);
1081          }
1082       }
1083
1084
1085       /*
1086        * Return some information to the caller
1087        */
1088       *SwapEntry = Pte >> 1;
1089    }
1090    else
1091    {
1092       ULONG Pte;
1093       PULONG Pt;
1094
1095       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1096
1097       if (Pt == NULL)
1098       {
1099          *SwapEntry = 0;
1100          return;
1101       }
1102
1103       /*
1104        * Atomically set the entry to zero and get the old value.
1105        */
1106       Pte = InterlockedExchangeUL(Pt, 0);
1107
1108       MiFlushTlb(Pt, Address);
1109
1110       /*
1111        * Decrement the reference count for this page table.
1112        */
1113       if (Process != NULL && Pte &&
1114           Process->AddressSpace.PageTableRefCountTable != NULL &&
1115           Address < (PVOID)KERNEL_BASE)
1116       {
1117          PUSHORT Ptrc;
1118
1119          Ptrc = Process->AddressSpace.PageTableRefCountTable;
1120
1121          Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
1122          if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
1123          {
1124             MmFreePageTable(Process, Address);
1125          }
1126       }
1127
1128
1129       /*
1130        * Return some information to the caller
1131        */
1132       *SwapEntry = Pte >> 1;
1133    }
1134 }
1135
1136 BOOLEAN
1137 Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
1138 {
1139    if (Ke386Pae)
1140    {
1141       PULONGLONG Pt;
1142       PULONGLONG Pde;
1143       Pde = PAE_ADDR_TO_PDE(PAddress);
1144       if (*Pde == 0LL)
1145       {
1146          Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
1147 #if 0
1148          /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1149          FLASH_TLB_ONE(PAddress);
1150 #endif
1151          if (Pt != NULL)
1152          {
1153             return TRUE;
1154          }
1155       }
1156    }
1157    else
1158    {
1159       PULONG Pt, Pde;
1160       Pde = ADDR_TO_PDE(PAddress);
1161       if (*Pde == 0)
1162       {
1163          Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
1164 #if 0
1165          /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1166          FLASH_TLB_ONE(PAddress);
1167 #endif
1168          if (Pt != NULL)
1169          {
1170             return TRUE;
1171          }
1172       }
1173    }
1174    return(FALSE);
1175 }
1176
1177 BOOLEAN MmIsDirtyPage(PEPROCESS Process, PVOID Address)
1178 {
1179    if (Ke386Pae)
1180    {
1181       return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1182    }
1183    else
1184    {
1185       return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1186    }
1187 }
1188
1189 BOOLEAN
1190 MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address)
1191 {
1192    if (Address < (PVOID)KERNEL_BASE && Process == NULL)
1193    {
1194       DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1195       KEBUGCHECK(0);
1196    }
1197    if (Ke386Pae)
1198    {
1199       PULONGLONG Pt;
1200       ULONGLONG Pte;
1201       ULONGLONG tmpPte;
1202
1203       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1204       if (Pt == NULL)
1205       {
1206          KEBUGCHECK(0);
1207       }
1208
1209       do
1210       {
1211          Pte = *Pt;
1212          tmpPte = Pte & ~PA_ACCESSED;
1213       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1214
1215       if (Pte & PA_ACCESSED)
1216       {
1217          MiFlushTlb((PULONG)Pt, Address);
1218          return TRUE;
1219       }
1220       else
1221       {
1222          MmUnmapPageTable((PULONG)Pt);
1223          return FALSE;
1224       }
1225    }
1226    else
1227    {
1228       PULONG Pt;
1229       ULONG Pte;
1230
1231       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1232       if (Pt == NULL)
1233       {
1234          KEBUGCHECK(0);
1235       }
1236
1237       do
1238       {
1239          Pte = *Pt;
1240       } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte));
1241
1242       if (Pte & PA_ACCESSED)
1243       {
1244          MiFlushTlb(Pt, Address);
1245          return TRUE;
1246       }
1247       else
1248       {
1249          MmUnmapPageTable(Pt);
1250          return FALSE;
1251       }
1252    }
1253 }
1254
1255 VOID MmSetCleanPage(PEPROCESS Process, PVOID Address)
1256 {
1257    if (Address < (PVOID)KERNEL_BASE && Process == NULL)
1258    {
1259       DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1260       KEBUGCHECK(0);
1261    }
1262    if (Ke386Pae)
1263    {
1264       PULONGLONG Pt;
1265       ULONGLONG Pte;
1266       ULONGLONG tmpPte;
1267
1268       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1269
1270       if (Pt == NULL)
1271       {
1272          KEBUGCHECK(0);
1273       }
1274
1275       do
1276       {
1277          Pte = *Pt;
1278          tmpPte = Pte & ~PA_DIRTY;
1279       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1280
1281       if (Pte & PA_DIRTY)
1282       {
1283          MiFlushTlb((PULONG)Pt, Address);
1284       }
1285       else
1286       {
1287          MmUnmapPageTable((PULONG)Pt);
1288       }
1289    }
1290    else
1291    {
1292       PULONG Pt;
1293       ULONG Pte;
1294
1295       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1296
1297       if (Pt == NULL)
1298       {
1299          KEBUGCHECK(0);
1300       }
1301
1302       do
1303       {
1304          Pte = *Pt;
1305       } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));
1306
1307       if (Pte & PA_DIRTY)
1308       {
1309          MiFlushTlb(Pt, Address);
1310       }
1311       else
1312       {
1313          MmUnmapPageTable(Pt);
1314       }
1315    }
1316 }
1317
1318 VOID MmSetDirtyPage(PEPROCESS Process, PVOID Address)
1319 {
1320    if (Address < (PVOID)KERNEL_BASE && Process == NULL)
1321    {
1322       DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1323       KEBUGCHECK(0);
1324    }
1325    if (Ke386Pae)
1326    {
1327       PULONGLONG Pt;
1328       ULONGLONG Pte;
1329       ULONGLONG tmpPte;
1330
1331       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1332       if (Pt == NULL)
1333       {
1334          KEBUGCHECK(0);
1335       }
1336
1337       do
1338       {
1339          Pte = *Pt;
1340          tmpPte = Pte | PA_DIRTY;
1341       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1342       if (!(Pte & PA_DIRTY))
1343       {
1344          MiFlushTlb((PULONG)Pt, Address);
1345       }
1346       else
1347       {
1348          MmUnmapPageTable((PULONG)Pt);
1349       }
1350    }
1351    else
1352    {
1353       PULONG Pt;
1354       ULONG Pte;
1355
1356       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1357       if (Pt == NULL)
1358       {
1359          KEBUGCHECK(0);
1360       }
1361
1362       do
1363       {
1364          Pte = *Pt;
1365       } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
1366       if (!(Pte & PA_DIRTY))
1367       {
1368          MiFlushTlb(Pt, Address);
1369       }
1370       else
1371       {
1372          MmUnmapPageTable(Pt);
1373       }
1374    }
1375 }
1376
1377 VOID MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
1378 {
1379    if (Ke386Pae)
1380    {
1381       PULONGLONG Pt;
1382       ULONGLONG Pte;
1383       ULONGLONG tmpPte;
1384
1385       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1386       if (Pt == NULL)
1387       {
1388          KEBUGCHECK(0);
1389       }
1390
1391       do
1392       {
1393          Pte = *Pt;
1394          tmpPte = Pte | PA_PRESENT;
1395       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1396       if (!(Pte & PA_PRESENT))
1397       {
1398          MiFlushTlb((PULONG)Pt, Address);
1399       }
1400       else
1401       {
1402          MmUnmapPageTable((PULONG)Pt);
1403       }
1404    }
1405    else
1406    {
1407       PULONG Pt;
1408       ULONG Pte;
1409
1410       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1411       if (Pt == NULL)
1412       {
1413          KEBUGCHECK(0);
1414       }
1415
1416       do
1417       {
1418          Pte = *Pt;
1419       } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte));
1420       if (!(Pte & PA_PRESENT))
1421       {
1422          MiFlushTlb(Pt, Address);
1423       }
1424       else
1425       {
1426          MmUnmapPageTable(Pt);
1427       }
1428    }
1429 }
1430
1431 BOOLEAN MmIsPagePresent(PEPROCESS Process, PVOID Address)
1432 {
1433    if (Ke386Pae)
1434    {
1435       return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1436    }
1437    else
1438    {
1439       return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1440    }
1441 }
1442
1443 BOOLEAN MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
1444 {
1445    if (Ke386Pae)
1446    {
1447       ULONGLONG Entry;
1448       Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1449       return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1450    }
1451    else
1452    {
1453       ULONG Entry;
1454       Entry = MmGetPageEntryForProcess(Process, Address);
1455       return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1456    }
1457 }
1458
1459 NTSTATUS
1460 MmCreateVirtualMappingForKernel(PVOID Address,
1461                                 ULONG flProtect,
1462                                 PPFN_TYPE Pages,
1463                                 ULONG PageCount)
1464 {
1465    ULONG Attributes;
1466    ULONG i;
1467    PVOID Addr;
1468    ULONG PdeOffset, oldPdeOffset;
1469    BOOLEAN NoExecute = FALSE;
1470
1471    DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1472            Address, flProtect, Pages, PageCount);
1473
1474    if (Address < (PVOID)KERNEL_BASE)
1475    {
1476       DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1477       KEBUGCHECK(0);
1478    }
1479
1480    Attributes = ProtectToPTE(flProtect);
1481    if (Attributes & 0x80000000)
1482    {
1483       NoExecute = TRUE;
1484    }
1485    Attributes &= 0xfff;
1486    if (Ke386GlobalPagesEnabled)
1487    {
1488       Attributes |= PA_GLOBAL;
1489    }
1490
1491    Addr = Address;
1492
1493    if (Ke386Pae)
1494    {
1495       PULONGLONG Pt = NULL;
1496       ULONGLONG Pte;
1497
1498       oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1499       for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1500       {
1501          if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1502          {
1503             DPRINT1("Setting physical address but not allowing access at address "
1504                     "0x%.8X with attributes %x/%x.\n",
1505                     Addr, Attributes, flProtect);
1506             KEBUGCHECK(0);
1507          }
1508
1509          PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1510          if (oldPdeOffset != PdeOffset)
1511          {
1512             Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE);
1513             if (Pt == NULL)
1514             {
1515                KEBUGCHECK(0);
1516             }
1517          }
1518          else
1519          {
1520             Pt++;
1521          }
1522          oldPdeOffset = PdeOffset;
1523
1524          Pte = PFN_TO_PTE(Pages[i]) | Attributes;
1525          if (NoExecute)
1526          {
1527             Pte |= 0x8000000000000000LL;
1528          }
1529          Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1530          if (Pte != 0LL)
1531          {
1532             KEBUGCHECK(0);
1533          }
1534       }
1535    }
1536    else
1537    {
1538       PULONG Pt;
1539       ULONG Pte;
1540
1541       oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1542       Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1543       if (Pt == NULL)
1544       {
1545          KEBUGCHECK(0);
1546       }
1547       Pt--;
1548
1549       for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1550       {
1551          if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1552          {
1553             DPRINT1("Setting physical address but not allowing access at address "
1554                     "0x%.8X with attributes %x/%x.\n",
1555                     Addr, Attributes, flProtect);
1556             KEBUGCHECK(0);
1557          }
1558
1559          PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1560          if (oldPdeOffset != PdeOffset)
1561          {
1562             Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1563             if (Pt == NULL)
1564             {
1565                KEBUGCHECK(0);
1566             }
1567          }
1568          else
1569          {
1570             Pt++;
1571          }
1572          oldPdeOffset = PdeOffset;
1573
1574          Pte = *Pt;
1575          if (Pte != 0)
1576          {
1577             KEBUGCHECK(0);
1578          }
1579          InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1580       }
1581    }
1582
1583    return(STATUS_SUCCESS);
1584 }
1585
1586 NTSTATUS
1587 MmCreatePageFileMapping(PEPROCESS Process,
1588                         PVOID Address,
1589                         SWAPENTRY SwapEntry)
1590 {
1591    if (Process == NULL && Address < (PVOID)KERNEL_BASE)
1592    {
1593       DPRINT1("No process\n");
1594       KEBUGCHECK(0);
1595    }
1596    if (Process != NULL && Address >= (PVOID)KERNEL_BASE)
1597    {
1598       DPRINT1("Setting kernel address with process context\n");
1599       KEBUGCHECK(0);
1600    }
1601    if (SwapEntry & (1 << 31))
1602    {
1603       KEBUGCHECK(0);
1604    }
1605
1606    if (Ke386Pae)
1607    {
1608       PULONGLONG Pt;
1609       ULONGLONG Pte;
1610       ULONGLONG tmpPte;
1611
1612       Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
1613       if (Pt == NULL)
1614       {
1615          KEBUGCHECK(0);
1616       }
1617       tmpPte = SwapEntry << 1;
1618       Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1619       if (PAE_PAGE_MASK((Pte)) != 0)
1620       {
1621          MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1622       }
1623
1624       if (Pte != 0)
1625       {
1626          MiFlushTlb((PULONG)Pt, Address);
1627       }
1628       else
1629       {
1630          MmUnmapPageTable((PULONG)Pt);
1631       }
1632    }
1633    else
1634    {
1635       PULONG Pt;
1636       ULONG Pte;
1637
1638       Pt = MmGetPageTableForProcess(Process, Address, TRUE);
1639       if (Pt == NULL)
1640       {
1641          KEBUGCHECK(0);
1642       }
1643       Pte = *Pt;
1644       if (PAGE_MASK((Pte)) != 0)
1645       {
1646          MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1647       }
1648       InterlockedExchangeUL(Pt, SwapEntry << 1);
1649       if (Pte != 0)
1650       {
1651          MiFlushTlb(Pt, Address);
1652       }
1653       else
1654       {
1655          MmUnmapPageTable(Pt);
1656       }
1657    }
1658    if (Process != NULL &&
1659        Process->AddressSpace.PageTableRefCountTable != NULL &&
1660        Address < (PVOID)KERNEL_BASE)
1661    {
1662      PUSHORT Ptrc;
1663      ULONG Idx;
1664
1665      Ptrc = Process->AddressSpace.PageTableRefCountTable;
1666      Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1667      Ptrc[Idx]++;
1668    }
1669    return(STATUS_SUCCESS);
1670 }
1671
1672
1673 NTSTATUS
1674 MmCreateVirtualMappingUnsafe(PEPROCESS Process,
1675                              PVOID Address,
1676                              ULONG flProtect,
1677                              PPFN_TYPE Pages,
1678                              ULONG PageCount)
1679 {
1680    ULONG Attributes;
1681    PVOID Addr;
1682    ULONG i;
1683    ULONG oldPdeOffset, PdeOffset;
1684    BOOLEAN NoExecute = FALSE;
1685
1686    DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1687           Process, Address, flProtect, Pages, *Pages, PageCount);
1688
1689    if (Process == NULL)
1690    {
1691       if (Address < (PVOID)KERNEL_BASE)
1692       {
1693          DPRINT1("No process\n");
1694          KEBUGCHECK(0);
1695       }
1696       if (PageCount > 0x10000 ||
1697           (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
1698       {
1699          DPRINT1("Page count to large\n");
1700          KEBUGCHECK(0);
1701       }
1702    }
1703    else
1704    {
1705       if (Address >= (PVOID)KERNEL_BASE)
1706       {
1707          DPRINT1("Setting kernel address with process context\n");
1708          KEBUGCHECK(0);
1709       }
1710       if (PageCount > KERNEL_BASE / PAGE_SIZE ||
1711           (ULONG_PTR) Address / PAGE_SIZE + PageCount > KERNEL_BASE / PAGE_SIZE)
1712       {
1713          DPRINT1("Page Count to large\n");
1714          KEBUGCHECK(0);
1715       }
1716    }
1717
1718    Attributes = ProtectToPTE(flProtect);
1719    if (Attributes & 0x80000000)
1720    {
1721       NoExecute = TRUE;
1722    }
1723    Attributes &= 0xfff;
1724    if (Address >= (PVOID)KERNEL_BASE)
1725    {
1726       Attributes &= ~PA_USER;
1727       if (Ke386GlobalPagesEnabled)
1728       {
1729          Attributes |= PA_GLOBAL;
1730       }
1731    }
1732    else
1733    {
1734       Attributes |= PA_USER;
1735    }
1736
1737    Addr = Address;
1738
1739    if (Ke386Pae)
1740    {
1741       ULONGLONG Pte, tmpPte;
1742       PULONGLONG Pt = NULL;
1743
1744       oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1745       for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1746       {
1747          if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1748          {
1749             DPRINT1("Setting physical address but not allowing access at address "
1750                     "0x%.8X with attributes %x/%x.\n",
1751                     Addr, Attributes, flProtect);
1752             KEBUGCHECK(0);
1753          }
1754          PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1755          if (oldPdeOffset != PdeOffset)
1756          {
1757             MmUnmapPageTable((PULONG)Pt);
1758             Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
1759             if (Pt == NULL)
1760             {
1761                KEBUGCHECK(0);
1762             }
1763          }
1764          else
1765          {
1766             Pt++;
1767          }
1768          oldPdeOffset = PdeOffset;
1769
1770          MmMarkPageMapped(Pages[i]);
1771          tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
1772          if (NoExecute)
1773          {
1774             tmpPte |= 0x8000000000000000LL;
1775          }
1776          Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1777          if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
1778          {
1779             KEBUGCHECK(0);
1780          }
1781          if (PAE_PAGE_MASK((Pte)) != 0LL)
1782          {
1783             MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1784          }
1785          if (Address < (PVOID)KERNEL_BASE &&
1786              Process->AddressSpace.PageTableRefCountTable != NULL &&
1787              Attributes & PA_PRESENT)
1788          {
1789             PUSHORT Ptrc;
1790
1791             Ptrc = Process->AddressSpace.PageTableRefCountTable;
1792
1793             Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
1794          }
1795          if (Pte != 0LL)
1796          {
1797             if (Address > (PVOID)KERNEL_BASE ||
1798                 (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
1799             {
1800               MiFlushTlb((PULONG)Pt, Address);
1801             }
1802          }
1803       }
1804       if (Addr > Address)
1805       {
1806          MmUnmapPageTable((PULONG)Pt);
1807       }
1808    }
1809    else
1810    {
1811       PULONG Pt = NULL;
1812       ULONG Pte;
1813       oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
1814       for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1815       {
1816          if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1817          {
1818             DPRINT1("Setting physical address but not allowing access at address "
1819                     "0x%.8X with attributes %x/%x.\n",
1820                     Addr, Attributes, flProtect);
1821             KEBUGCHECK(0);
1822          }
1823          PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1824          if (oldPdeOffset != PdeOffset)
1825          {
1826             MmUnmapPageTable(Pt);
1827             Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
1828             if (Pt == NULL)
1829             {
1830                KEBUGCHECK(0);
1831             }
1832          }
1833          else
1834          {
1835             Pt++;
1836          }
1837          oldPdeOffset = PdeOffset;
1838
1839          Pte = *Pt;
1840          MmMarkPageMapped(Pages[i]);
1841          if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
1842          {
1843             KEBUGCHECK(0);
1844          }
1845          if (PAGE_MASK((Pte)) != 0)
1846          {
1847             MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1848          }
1849          InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1850          if (Address < (PVOID)KERNEL_BASE &&
1851              Process->AddressSpace.PageTableRefCountTable != NULL &&
1852              Attributes & PA_PRESENT)
1853          {
1854             PUSHORT Ptrc;
1855
1856             Ptrc = Process->AddressSpace.PageTableRefCountTable;
1857
1858             Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
1859          }
1860          if (Pte != 0)
1861          {
1862             if (Address > (PVOID)KERNEL_BASE ||
1863                 (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
1864             {
1865                MiFlushTlb(Pt, Address);
1866             }
1867          }
1868       }
1869       if (Addr > Address)
1870       {
1871          MmUnmapPageTable(Pt);
1872       }
1873    }
1874    return(STATUS_SUCCESS);
1875 }
1876
1877 NTSTATUS
1878 MmCreateVirtualMapping(PEPROCESS Process,
1879                        PVOID Address,
1880                        ULONG flProtect,
1881                        PPFN_TYPE Pages,
1882                        ULONG PageCount)
1883 {
1884    ULONG i;
1885
1886    for (i = 0; i < PageCount; i++)
1887    {
1888       if (!MmIsUsablePage(Pages[i]))
1889       {
1890          DPRINT1("Page at address %x not usable\n", PFN_TO_PTE(Pages[i]));
1891          KEBUGCHECK(0);
1892       }
1893    }
1894
1895    return(MmCreateVirtualMappingUnsafe(Process,
1896                                        Address,
1897                                        flProtect,
1898                                        Pages,
1899                                        PageCount));
1900 }
1901
1902 ULONG
1903 MmGetPageProtect(PEPROCESS Process, PVOID Address)
1904 {
1905    ULONG Entry;
1906    ULONG Protect;
1907    if (Ke386Pae)
1908    {
1909       Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1910    }
1911    else
1912    {
1913       Entry = MmGetPageEntryForProcess(Process, Address);
1914    }
1915
1916    if (!(Entry & PA_PRESENT))
1917    {
1918       Protect = PAGE_NOACCESS;
1919    }
1920    else
1921    {
1922       if (Entry & PA_READWRITE)
1923       {
1924          Protect = PAGE_READWRITE;
1925       }
1926       else
1927       {
1928          Protect = PAGE_EXECUTE_READ;
1929       }
1930       if (Entry & PA_CD)
1931       {
1932          Protect |= PAGE_NOCACHE;
1933       }
1934       if (Entry & PA_WT)
1935       {
1936          Protect |= PAGE_WRITETHROUGH;
1937       }
1938       if (!(Entry & PA_USER))
1939       {
1940          Protect |= PAGE_SYSTEM;
1941       }
1942
1943    }
1944    return(Protect);
1945 }
1946
1947 VOID
1948 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
1949 {
1950    ULONG Attributes = 0;
1951    BOOLEAN NoExecute = FALSE;
1952
1953    DPRINT("MmSetPageProtect(Process %x  Address %x  flProtect %x)\n",
1954           Process, Address, flProtect);
1955
1956    Attributes = ProtectToPTE(flProtect);
1957    if (Attributes & 0x80000000)
1958    {
1959       NoExecute = TRUE;
1960    }
1961    Attributes &= 0xfff;
1962    if (Address >= (PVOID)KERNEL_BASE)
1963    {
1964       Attributes &= ~PA_USER;
1965       if (Ke386GlobalPagesEnabled)
1966       {
1967          Attributes |= PA_GLOBAL;
1968       }
1969    }
1970    else
1971    {
1972       Attributes |= PA_USER;
1973    }
1974    if (Ke386Pae)
1975    {
1976       PULONGLONG Pt;
1977       ULONGLONG tmpPte, Pte;
1978
1979       Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1980       if (Pt == NULL)
1981       {
1982          DPRINT1("Address %x\n", Address);
1983          KEBUGCHECK(0);
1984       }
1985       do
1986       {
1987         Pte = *Pt;
1988         tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
1989         if (NoExecute)
1990         {
1991            tmpPte |= 0x8000000000000000LL;
1992         }
1993         else
1994         {
1995            tmpPte &= ~0x8000000000000000LL;
1996         }
1997       } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1998
1999       MiFlushTlb((PULONG)Pt, Address);
2000    }
2001    else
2002    {
2003       PULONG Pt;
2004
2005       Pt = MmGetPageTableForProcess(Process, Address, FALSE);
2006       if (Pt == NULL)
2007       {
2008          KEBUGCHECK(0);
2009       }
2010       InterlockedExchange(Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
2011       MiFlushTlb(Pt, Address);
2012    }
2013 }
2014
2015 /*
2016  * @implemented
2017  */
2018 PHYSICAL_ADDRESS STDCALL
2019 MmGetPhysicalAddress(PVOID vaddr)
2020 /*
2021  * FUNCTION: Returns the physical address corresponding to a virtual address
2022  */
2023 {
2024    PHYSICAL_ADDRESS p;
2025
2026    DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr);
2027    if (Ke386Pae)
2028    {
2029       ULONGLONG Pte;
2030       Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr);
2031       if (Pte != 0 && Pte & PA_PRESENT)
2032       {
2033          p.QuadPart = PAE_PAGE_MASK(Pte);
2034          p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2035       }
2036       else
2037       {
2038          p.QuadPart = 0;
2039       }
2040    }
2041    else
2042    {
2043       ULONG Pte;
2044       Pte = MmGetPageEntryForProcess(NULL, vaddr);
2045       if (Pte != 0 && Pte & PA_PRESENT)
2046       {
2047          p.QuadPart = PAGE_MASK(Pte);
2048          p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2049       }
2050       else
2051       {
2052          p.QuadPart = 0;
2053       }
2054    }
2055    return p;
2056 }
2057
2058 PVOID
2059 MmCreateHyperspaceMapping(PFN_TYPE Page)
2060 {
2061    PVOID Address;
2062    LONG i;
2063
2064    if (Ke386Pae)
2065    {
2066       ULONGLONG Entry;
2067       ULONGLONG ZeroEntry = 0LL;
2068       PULONGLONG Pte;
2069
2070       Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2071       Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2072
2073       if (Page & 1024)
2074       {
2075          for (i = Page %1024; i < 1024; i++, Pte++)
2076          {
2077             if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2078             {
2079                break;
2080             }
2081          }
2082          if (i >= 1024)
2083          {
2084             Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
2085             for (i = 0; i < Page % 1024; i++, Pte++)
2086             {
2087                if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2088                {
2089                   break;
2090                }
2091             }
2092             if (i >= Page % 1024)
2093             {
2094                KEBUGCHECK(0);
2095             }
2096          }
2097       }
2098       else
2099       {
2100          for (i = Page %1024; i >= 0; i--, Pte--)
2101          {
2102             if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2103             {
2104                break;
2105             }
2106          }
2107          if (i < 0)
2108          {
2109             Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
2110             for (i = 1023; i > Page % 1024; i--, Pte--)
2111             {
2112                if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2113                {
2114                   break;
2115                }
2116             }
2117             if (i <= Page % 1024)
2118             {
2119                KEBUGCHECK(0);
2120             }
2121          }
2122       }
2123    }
2124    else
2125    {
2126       ULONG Entry;
2127       PULONG Pte;
2128       Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2129       Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2130       if (Page & 1024)
2131       {
2132          for (i = Page % 1024; i < 1024; i++, Pte++)
2133          {
2134             if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2135             {
2136                break;
2137             }
2138          }
2139          if (i >= 1024)
2140          {
2141             Pte = ADDR_TO_PTE(HYPERSPACE);
2142             for (i = 0; i < Page % 1024; i++, Pte++)
2143             {
2144                if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2145                {
2146                   break;
2147                }
2148             }
2149             if (i >= Page % 1024)
2150             {
2151                KEBUGCHECK(0);
2152             }
2153          }
2154       }
2155       else
2156       {
2157          for (i = Page % 1024; i >= 0; i--, Pte--)
2158          {
2159             if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2160             {
2161                break;
2162             }
2163          }
2164          if (i < 0)
2165          {
2166             Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
2167             for (i = 1023; i > Page % 1024; i--, Pte--)
2168             {
2169                if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2170                {
2171                   break;
2172                }
2173             }
2174             if (i <= Page % 1024)
2175             {
2176                KEBUGCHECK(0);
2177             }
2178          }
2179       }
2180    }
2181    Address = (PVOID)HYPERSPACE + i * PAGE_SIZE;
2182    FLUSH_TLB_ONE(Address);
2183    return Address;
2184 }
2185
2186 PFN_TYPE
2187 MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage)
2188 {
2189    PFN_TYPE Pfn;
2190    ASSERT (IS_HYPERSPACE(Address));
2191    if (Ke386Pae)
2192    {
2193       ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE;
2194       Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2195       Pfn = PAE_PTE_TO_PFN(Entry);
2196    }
2197    else
2198    {
2199       ULONG Entry;
2200       Entry = InterlockedExchange(ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE);
2201       Pfn = PTE_TO_PFN(Entry);
2202    }
2203    FLUSH_TLB_ONE(Address);
2204    return Pfn;
2205 }
2206
2207 PFN_TYPE
2208 MmDeleteHyperspaceMapping(PVOID Address)
2209 {
2210    PFN_TYPE Pfn;
2211    ASSERT (IS_HYPERSPACE(Address));
2212    if (Ke386Pae)
2213    {
2214       ULONGLONG Entry = 0LL;
2215       Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2216       Pfn = PAE_PTE_TO_PFN(Entry);
2217    }
2218    else
2219    {
2220       ULONG Entry;
2221       Entry = InterlockedExchange(ADDR_TO_PTE(Address), 0);
2222       Pfn = PTE_TO_PFN(Entry);
2223    }
2224    FLUSH_TLB_ONE(Address);
2225    return Pfn;
2226 }
2227
2228 VOID MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
2229 {
2230    ULONG StartOffset, EndOffset, Offset;
2231
2232    if (Address < (PVOID)KERNEL_BASE)
2233    {
2234       KEBUGCHECK(0);
2235    }
2236    if (Ke386Pae)
2237    {
2238       PULONGLONG PageDirTable;
2239       PULONGLONG Pde;
2240       ULONGLONG ZeroPde = 0LL;
2241       ULONG i;
2242
2243       for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET(Address + Size); i++)
2244       {
2245          if (i == PAE_ADDR_TO_PDTE_OFFSET(Address))
2246          {
2247             StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
2248          }
2249          else
2250          {
2251             StartOffset = 0;
2252          }
2253          if (i == PAE_ADDR_TO_PDTE_OFFSET(Address + Size))
2254          {
2255             EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address + Size);
2256          }
2257          else
2258          {
2259             EndOffset = 511;
2260          }
2261
2262          if (Process != NULL && Process != PsGetCurrentProcess())
2263          {
2264             PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
2265             Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i]));
2266             MmDeleteHyperspaceMapping(PageDirTable);
2267          }
2268          else
2269          {
2270             Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512;
2271          }
2272
2273          for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2274          {
2275             if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4)
2276             {
2277                ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde);
2278             }
2279          }
2280          MmUnmapPageTable((PULONG)Pde);
2281       }
2282    }
2283    else
2284    {
2285       PULONG Pde;
2286       StartOffset = ADDR_TO_PDE_OFFSET(Address);
2287       EndOffset = ADDR_TO_PDE_OFFSET(Address + Size);
2288
2289       if (Process != NULL && Process != PsGetCurrentProcess())
2290       {
2291          Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
2292       }
2293       else
2294       {
2295          Pde = (PULONG)PAGEDIRECTORY_MAP;
2296       }
2297       for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2298       {
2299          if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP))
2300          {
2301             InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0);
2302          }
2303       }
2304       if (Pde != (PULONG)PAGEDIRECTORY_MAP)
2305       {
2306          MmDeleteHyperspaceMapping(Pde);
2307       }
2308    }
2309 }
2310
2311 VOID INIT_FUNCTION
2312 MmInitGlobalKernelPageDirectory(VOID)
2313 {
2314    ULONG i;
2315
2316    DPRINT("MmInitGlobalKernelPageDirectory()\n");
2317
2318    if (Ke386Pae)
2319    {
2320       PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
2321       for (i = PAE_ADDR_TO_PDE_OFFSET(KERNEL_BASE); i < 4 * 512; i++)
2322       {
2323          if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
2324              !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
2325              0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
2326          {
2327             ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
2328             if (Ke386GlobalPagesEnabled)
2329             {
2330                MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
2331                CurrentPageDirectory[i] |= PA_GLOBAL;
2332             }
2333          }
2334       }
2335    }
2336    else
2337    {
2338       PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
2339       for (i = ADDR_TO_PDE_OFFSET(KERNEL_BASE); i < 1024; i++)
2340       {
2341          if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
2342              i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
2343              0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
2344          {
2345             MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
2346             if (Ke386GlobalPagesEnabled)
2347             {
2348                MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
2349                CurrentPageDirectory[i] |= PA_GLOBAL;
2350             }
2351          }
2352       }
2353    }
2354 }
2355
2356 ULONG
2357 MiGetUserPageDirectoryCount(VOID)
2358 {
2359    return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(KERNEL_BASE) : ADDR_TO_PDE_OFFSET(KERNEL_BASE);
2360 }
2361
2362 VOID INIT_FUNCTION
2363 MiInitPageDirectoryMap(VOID)
2364 {
2365    MEMORY_AREA* kernel_map_desc = NULL;
2366    MEMORY_AREA* hyperspace_desc = NULL;
2367    PHYSICAL_ADDRESS BoundaryAddressMultiple;
2368    PVOID BaseAddress;
2369
2370    DPRINT("MiInitPageDirectoryMap()\n");
2371
2372    BoundaryAddressMultiple.QuadPart = 0;
2373    BaseAddress = (PVOID)PAGETABLE_MAP;
2374    MmCreateMemoryArea(NULL,
2375                       MmGetKernelAddressSpace(),
2376                       MEMORY_AREA_SYSTEM,
2377                       &BaseAddress,
2378                       Ke386Pae ? 0x800000 : 0x400000,
2379                       0,
2380                       &kernel_map_desc,
2381                       TRUE,
2382                       FALSE,
2383                       BoundaryAddressMultiple);
2384    BaseAddress = (PVOID)HYPERSPACE;
2385    MmCreateMemoryArea(NULL,
2386                       MmGetKernelAddressSpace(),
2387                       MEMORY_AREA_SYSTEM,
2388                       &BaseAddress,
2389                       0x400000,
2390                       0,
2391                       &hyperspace_desc,
2392                       TRUE,
2393                       FALSE,
2394                       BoundaryAddressMultiple);
2395 }
2396
2397 /* EOF */