reactos/ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #line 15 "ARM³::PAGFAULT"
  16 #define MODULE_INVOLVED_IN_ARM3
  17 #include "../ARM3/miarm.h"
  18
  19 /* GLOBALS ********************************************************************/
  20
  21 /* PRIVATE FUNCTIONS **********************************************************/
  22
  23 PMMPTE
  24 NTAPI
  25 MiCheckVirtualAddress(IN PVOID VirtualAddress,
  26                       OUT PULONG ProtectCode,
  27                       OUT PMMVAD *ProtoVad)
  28 {
  29     PMMVAD Vad;
  30
  31     /* No prototype/section support for now */
  32     *ProtoVad = NULL;
  33
  34     /* Only valid for user VADs for now */
  35     ASSERT(VirtualAddress <= MM_HIGHEST_USER_ADDRESS);
  36
  37     /* Special case for shared data */
  38     if (PAGE_ALIGN(VirtualAddress) == (PVOID)USER_SHARED_DATA)
  39     {
  40         /* It's a read-only page */
  41         *ProtectCode = MM_READONLY;
  42         return MmSharedUserDataPte;
  43     }
  44
  45     /* Find the VAD, it must exist, since we only handle PEB/TEB */
  46     Vad = MiLocateAddress(VirtualAddress);
  47     ASSERT(Vad);
  48
  49     /* This must be a TEB/PEB VAD */
  50     ASSERT(Vad->u.VadFlags.PrivateMemory == TRUE);
  51     ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
  52     ASSERT(Vad->u.VadFlags.VadType == VadNone);
  53
  54     /* Return the protection on it */
  55     *ProtectCode = Vad->u.VadFlags.Protection;
  56     return NULL;
  57 }
  58
  59 NTSTATUS
  60 FASTCALL
  61 MiCheckPdeForPagedPool(IN PVOID Address)
  62 {
  63     PMMPDE PointerPde;
  64     NTSTATUS Status = STATUS_SUCCESS;
  65
  66     /* No session support in ReactOS yet */
  67     ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
  68     ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
  69
  70     //
  71     // Check if this is a fault while trying to access the page table itself
  72     //
  73     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
  74     {
  75         //
  76         // Send a hint to the page fault handler that this is only a valid fault
  77         // if we already detected this was access within the page table range
  78         //
  79         PointerPde = (PMMPDE)MiAddressToPte(Address);
  80         Status = STATUS_WAIT_1;
  81     }
  82     else if (Address < MmSystemRangeStart)
  83     {
  84         //
  85         // This is totally illegal
  86         //
  87         return STATUS_ACCESS_VIOLATION;
  88     }
  89     else
  90     {
  91         //
  92         // Get the PDE for the address
  93         //
  94         PointerPde = MiAddressToPde(Address);
  95     }
  96
  97     //
  98     // Check if it's not valid
  99     //
 100     if (PointerPde->u.Hard.Valid == 0)
 101     {
 102 #ifndef _M_AMD64
 103         /* This seems to be making the assumption that one PDE is one page long */
 104         C_ASSERT(PAGE_SIZE == (PD_COUNT * (sizeof(MMPTE) * PDE_COUNT)));
 105 #endif
 106
 107         //
 108         // Copy it from our double-mapped system page directory
 109         //
 110         InterlockedExchangePte(PointerPde,
 111                                MmSystemPagePtes[((ULONG_PTR)PointerPde &
 112                                                  (PAGE_SIZE - 1)) /
 113                                                 sizeof(MMPTE)].u.Long);
 114     }
 115
 116     //
 117     // Return status
 118     //
 119     return Status;
 120 }
 121
 122 VOID
 123 NTAPI
 124 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 125 {
 126     PMMPTE ZeroPte;
 127     MMPTE TempPte;
 128     PMMPFN Pfn1;
 129     PVOID ZeroAddress;
 130
 131     /* Get the PFN for this page */
 132     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 133     ASSERT(Pfn1);
 134
 135     /* Grab a system PTE we can use to zero the page */
 136     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 137     ASSERT(ZeroPte);
 138
 139     /* Initialize the PTE for it */
 140     TempPte = ValidKernelPte;
 141     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 142
 143     /* Setup caching */
 144     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 145     {
 146         /* Write combining, no caching */
 147         MI_PAGE_DISABLE_CACHE(&TempPte);
 148         MI_PAGE_WRITE_COMBINED(&TempPte);
 149     }
 150     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 151     {
 152         /* Write through, no caching */
 153         MI_PAGE_DISABLE_CACHE(&TempPte);
 154         MI_PAGE_WRITE_THROUGH(&TempPte);
 155     }
 156
 157     /* Make the system PTE valid with our PFN */
 158     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 159
 160     /* Get the address it maps to, and zero it out */
 161     ZeroAddress = MiPteToAddress(ZeroPte);
 162     KeZeroPages(ZeroAddress, PAGE_SIZE);
 163
 164     /* Now get rid of it */
 165     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 166 }
 167
 168 NTSTATUS
 169 NTAPI
 170 MiResolveDemandZeroFault(IN PVOID Address,
 171                          IN PMMPTE PointerPte,
 172                          IN PEPROCESS Process,
 173                          IN KIRQL OldIrql)
 174 {
 175     PFN_NUMBER PageFrameNumber;
 176     MMPTE TempPte;
 177     BOOLEAN NeedZero = FALSE;
 178     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 179             Address,
 180             Process);
 181
 182     /* Must currently only be called by paging path */
 183     ASSERT(OldIrql == MM_NOIRQL);
 184     if (Process)
 185     {
 186         /* Sanity check */
 187         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 188
 189         /* No forking yet */
 190         ASSERT(Process->ForkInProgress == NULL);
 191
 192         /* We'll need a zero page */
 193         NeedZero = TRUE;
 194     }
 195
 196     //
 197     // Lock the PFN database
 198     //
 199     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 200     ASSERT(PointerPte->u.Hard.Valid == 0);
 201
 202     /* Get a page */
 203     PageFrameNumber = MiRemoveAnyPage(0);
 204     DPRINT("New pool page: %lx\n", PageFrameNumber);
 205
 206     /* Initialize it */
 207     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 208
 209     //
 210     // Release PFN lock
 211     //
 212     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 213
 214     //
 215     // Increment demand zero faults
 216     //
 217     InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
 218
 219     /* Zero the page if need be */
 220     if (NeedZero) MiZeroPfn(PageFrameNumber);
 221
 222     /* Build the PTE */
 223     if (PointerPte <= MiHighestUserPte)
 224     {
 225         /* For user mode */
 226         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 227                                   PointerPte,
 228                                   PointerPte->u.Soft.Protection,
 229                                   PageFrameNumber);
 230     }
 231     else
 232     {
 233         /* For kernel mode */
 234         MI_MAKE_HARDWARE_PTE(&TempPte,
 235                              PointerPte,
 236                              PointerPte->u.Soft.Protection,
 237                              PageFrameNumber);
 238     }
 239
 240     /* Set it dirty if it's a writable page */
 241     if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
 242
 243     /* Write it */
 244     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 245
 246     //
 247     // It's all good now
 248     //
 249     DPRINT("Paged pool page has now been paged in\n");
 250     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 251 }
 252
 253 NTSTATUS
 254 NTAPI
 255 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 256                         IN PVOID Address,
 257                         IN PMMPTE PointerPte,
 258                         IN PMMPTE PointerProtoPte,
 259                         IN KIRQL OldIrql,
 260                         IN PMMPFN Pfn1)
 261 {
 262     MMPTE TempPte;
 263     PFN_NUMBER PageFrameIndex;
 264
 265     /* Must be called with an valid prototype PTE, with the PFN lock held */
 266     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 267     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 268
 269     /* Quick-n-dirty */
 270     ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
 271
 272     /* Get the page */
 273     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 274
 275     /* Release the PFN lock */
 276     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 277
 278     /* Build the user PTE */
 279     ASSERT(Address < MmSystemRangeStart);
 280     MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, MM_READONLY, PageFrameIndex);
 281
 282     /* Write the PTE */
 283     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 284
 285     /* Return success */
 286     return STATUS_SUCCESS;
 287 }
 288
 289 NTSTATUS
 290 NTAPI
 291 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
 292                        IN PVOID Address,
 293                        IN PMMPTE PointerPte,
 294                        IN PMMPTE PointerProtoPte,
 295                        IN OUT PMMPFN *OutPfn,
 296                        OUT PVOID *PageFileData,
 297                        OUT PMMPTE PteValue,
 298                        IN PEPROCESS Process,
 299                        IN KIRQL OldIrql,
 300                        IN PVOID TrapInformation)
 301 {
 302     MMPTE TempPte;
 303     PMMPFN Pfn1;
 304     PFN_NUMBER PageFrameIndex;
 305
 306     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
 307     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 308     ASSERT(PointerPte->u.Hard.Valid == 0);
 309     ASSERT(PointerPte->u.Soft.Prototype == 1);
 310
 311     /* Read the prototype PTE -- it must be valid since we only handle shared data */
 312     TempPte = *PointerProtoPte;
 313     ASSERT(TempPte.u.Hard.Valid == 1);
 314
 315     /* One more user of this mapped page */
 316     PageFrameIndex = PFN_FROM_PTE(&TempPte);
 317     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 318     Pfn1->u2.ShareCount++;
 319
 320     /* Call it a transition */
 321     InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
 322
 323     /* Complete the prototype PTE fault -- this will release the PFN lock */
 324     return MiCompleteProtoPteFault(StoreInstruction,
 325                                    Address,
 326                                    PointerPte,
 327                                    PointerProtoPte,
 328                                    OldIrql,
 329                                    NULL);
 330 }
 331
 332 NTSTATUS
 333 NTAPI
 334 MiDispatchFault(IN BOOLEAN StoreInstruction,
 335                 IN PVOID Address,
 336                 IN PMMPTE PointerPte,
 337                 IN PMMPTE PointerProtoPte,
 338                 IN BOOLEAN Recursive,
 339                 IN PEPROCESS Process,
 340                 IN PVOID TrapInformation,
 341                 IN PVOID Vad)
 342 {
 343     MMPTE TempPte;
 344     KIRQL OldIrql, LockIrql;
 345     NTSTATUS Status;
 346     PMMPTE SuperProtoPte;
 347     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
 348              Address,
 349              Process);
 350
 351     //
 352     // Make sure APCs are off and we're not at dispatch
 353     //
 354     OldIrql = KeGetCurrentIrql();
 355     ASSERT(OldIrql <= APC_LEVEL);
 356     ASSERT(KeAreAllApcsDisabled() == TRUE);
 357
 358     //
 359     // Grab a copy of the PTE
 360     //
 361     TempPte = *PointerPte;
 362
 363     /* Do we have a prototype PTE? */
 364     if (PointerProtoPte)
 365     {
 366         /* This should never happen */
 367         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
 368
 369         /* We currently only handle the shared user data PTE path */
 370         ASSERT(Address < MmSystemRangeStart);
 371         ASSERT(PointerPte->u.Soft.Prototype == 1);
 372         ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
 373         ASSERT(Vad == NULL);
 374
 375         /* Lock the PFN database */
 376         LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 377
 378         /* For the shared data page, this should be true */
 379         SuperProtoPte = MiAddressToPte(PointerProtoPte);
 380         ASSERT(SuperProtoPte->u.Hard.Valid == 1);
 381         ASSERT(TempPte.u.Hard.Valid == 0);
 382
 383         /* Resolve the fault -- this will release the PFN lock */
 384         Status = MiResolveProtoPteFault(StoreInstruction,
 385                                         Address,
 386                                         PointerPte,
 387                                         PointerProtoPte,
 388                                         NULL,
 389                                         NULL,
 390                                         NULL,
 391                                         Process,
 392                                         LockIrql,
 393                                         TrapInformation);
 394         ASSERT(Status == STATUS_SUCCESS);
 395
 396         /* Complete this as a transition fault */
 397         ASSERT(OldIrql == KeGetCurrentIrql());
 398         ASSERT(OldIrql <= APC_LEVEL);
 399         ASSERT(KeAreAllApcsDisabled() == TRUE);
 400         return STATUS_PAGE_FAULT_TRANSITION;
 401     }
 402
 403     //
 404     // The PTE must be invalid, but not totally blank
 405     //
 406     ASSERT(TempPte.u.Hard.Valid == 0);
 407     ASSERT(TempPte.u.Long != 0);
 408
 409     //
 410     // No prototype, transition or page file software PTEs in ARM3 yet
 411     //
 412     ASSERT(TempPte.u.Soft.Prototype == 0);
 413     ASSERT(TempPte.u.Soft.Transition == 0);
 414     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 415
 416     //
 417     // If we got this far, the PTE can only be a demand zero PTE, which is what
 418     // we want. Go handle it!
 419     //
 420     Status = MiResolveDemandZeroFault(Address,
 421                                       PointerPte,
 422                                       Process,
 423                                       MM_NOIRQL);
 424     ASSERT(KeAreAllApcsDisabled () == TRUE);
 425     if (NT_SUCCESS(Status))
 426     {
 427         //
 428         // Make sure we're returning in a sane state and pass the status down
 429         //
 430         ASSERT(OldIrql == KeGetCurrentIrql ());
 431         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 432         return Status;
 433     }
 434
 435     //
 436     // Generate an access fault
 437     //
 438     return STATUS_ACCESS_VIOLATION;
 439 }
 440
 441 NTSTATUS
 442 NTAPI
 443 MmArmAccessFault(IN BOOLEAN StoreInstruction,
 444                  IN PVOID Address,
 445                  IN KPROCESSOR_MODE Mode,
 446                  IN PVOID TrapInformation)
 447 {
 448     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
 449     PMMPTE PointerPte, ProtoPte;
 450     PMMPDE PointerPde;
 451     MMPTE TempPte;
 452     PETHREAD CurrentThread;
 453     PEPROCESS CurrentProcess;
 454     NTSTATUS Status;
 455     PMMSUPPORT WorkingSet;
 456     ULONG ProtectionCode;
 457     PMMVAD Vad;
 458     PFN_NUMBER PageFrameIndex;
 459     DPRINT("ARM3 FAULT AT: %p\n", Address);
 460
 461     //
 462     // Get the PTE and PDE
 463     //
 464     PointerPte = MiAddressToPte(Address);
 465     PointerPde = MiAddressToPde(Address);
 466 #if (_MI_PAGING_LEVELS >= 3)
 467     /* We need the PPE and PXE addresses */
 468     ASSERT(FALSE);
 469 #endif
 470
 471     //
 472     // Check for dispatch-level snafu
 473     //
 474     if (OldIrql > APC_LEVEL)
 475     {
 476         //
 477         // There are some special cases where this is okay, but not in ARM3 yet
 478         //
 479         DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
 480                  Address,
 481                  OldIrql);
 482         ASSERT(OldIrql <= APC_LEVEL);
 483     }
 484
 485     //
 486     // Check for kernel fault
 487     //
 488     if (Address >= MmSystemRangeStart)
 489     {
 490         //
 491         // What are you even DOING here?
 492         //
 493         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
 494
 495 #if (_MI_PAGING_LEVELS >= 3)
 496         /* Need to check PXE and PDE validity */
 497         ASSERT(FALSE);
 498 #endif
 499
 500         //
 501         // Is the PDE valid?
 502         //
 503         if (!PointerPde->u.Hard.Valid == 0)
 504         {
 505             //
 506             // Debug spew (eww!)
 507             //
 508             DPRINT("Invalid PDE\n");
 509 #if (_MI_PAGING_LEVELS == 2)
 510             //
 511             // Handle mapping in "Special" PDE directoreis
 512             //
 513             MiCheckPdeForPagedPool(Address);
 514 #endif
 515             //
 516             // Now we SHOULD be good
 517             //
 518             if (PointerPde->u.Hard.Valid == 0)
 519             {
 520                 //
 521                 // FIXFIX: Do the S-LIST hack
 522                 //
 523
 524                 //
 525                 // Kill the system
 526                 //
 527                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 528                              (ULONG_PTR)Address,
 529                              StoreInstruction,
 530                              (ULONG_PTR)TrapInformation,
 531                              2);
 532             }
 533         }
 534
 535         //
 536         // The PDE is valid, so read the PTE
 537         //
 538         TempPte = *PointerPte;
 539         if (TempPte.u.Hard.Valid == 1)
 540         {
 541             //
 542             // Only two things can go wrong here:
 543             // Executing NX page (we couldn't care less)
 544             // Writing to a read-only page (the stuff ARM3 works with is write,
 545             // so again, moot point).
 546             //
 547             if (StoreInstruction)
 548             {
 549                 DPRINT1("Should NEVER happen on ARM3!!!\n");
 550                 return STATUS_ACCESS_VIOLATION;
 551             }
 552
 553             //
 554             // Otherwise, the PDE was probably invalid, and all is good now
 555             //
 556             return STATUS_SUCCESS;
 557         }
 558
 559         //
 560         // Check for a fault on the page table or hyperspace itself
 561         //
 562         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
 563         {
 564             //
 565             // This might happen...not sure yet
 566             //
 567             DPRINT1("FAULT ON PAGE TABLES: %p %lx %lx!\n", Address, *PointerPte, *PointerPde);
 568 #if (_MI_PAGING_LEVELS == 2)
 569             //
 570             // Map in the page table
 571             //
 572             if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
 573             {
 574                 DPRINT1("PAGE TABLES FAULTED IN!\n");
 575                 return STATUS_SUCCESS;
 576             }
 577 #endif
 578             //
 579             // Otherwise the page table doesn't actually exist
 580             //
 581             DPRINT1("FAILING\n");
 582             return STATUS_ACCESS_VIOLATION;
 583         }
 584
 585         /* In this path, we are using the system working set */
 586         CurrentThread = PsGetCurrentThread();
 587         WorkingSet = &MmSystemCacheWs;
 588
 589         /* Acquire it */
 590         KeRaiseIrql(APC_LEVEL, &LockIrql);
 591         MiLockWorkingSet(CurrentThread, WorkingSet);
 592
 593         //
 594         // Re-read PTE now that the IRQL has been raised
 595         //
 596         TempPte = *PointerPte;
 597         if (TempPte.u.Hard.Valid == 1)
 598         {
 599             //
 600             // Only two things can go wrong here:
 601             // Executing NX page (we couldn't care less)
 602             // Writing to a read-only page (the stuff ARM3 works with is write,
 603             // so again, moot point.
 604             //
 605             if (StoreInstruction)
 606             {
 607                 DPRINT1("Should NEVER happen on ARM3!!!\n");
 608                 return STATUS_ACCESS_VIOLATION;
 609             }
 610
 611             /* Release the working set */
 612             MiUnlockWorkingSet(CurrentThread, WorkingSet);
 613             KeLowerIrql(LockIrql);
 614
 615             //
 616             // Otherwise, the PDE was probably invalid, and all is good now
 617             //
 618             return STATUS_SUCCESS;
 619         }
 620
 621         //
 622         // We don't implement prototype PTEs
 623         //
 624         ASSERT(TempPte.u.Soft.Prototype == 0);
 625
 626         //
 627         // We don't implement transition PTEs
 628         //
 629         ASSERT(TempPte.u.Soft.Transition == 0);
 630
 631         //
 632         // Now do the real fault handling
 633         //
 634         Status = MiDispatchFault(StoreInstruction,
 635                                  Address,
 636                                  PointerPte,
 637                                  NULL,
 638                                  FALSE,
 639                                  NULL,
 640                                  TrapInformation,
 641                                  NULL);
 642
 643         /* Release the working set */
 644         ASSERT(KeAreAllApcsDisabled() == TRUE);
 645         MiUnlockWorkingSet(CurrentThread, WorkingSet);
 646         KeLowerIrql(LockIrql);
 647
 648         //
 649         // We are done!
 650         //
 651         DPRINT("Fault resolved with status: %lx\n", Status);
 652         return Status;
 653     }
 654
 655     /* This is a user fault */
 656     CurrentThread = PsGetCurrentThread();
 657     CurrentProcess = PsGetCurrentProcess();
 658
 659     /* Lock the working set */
 660     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
 661
 662 #if (_MI_PAGING_LEVELS >= 3)
 663     /* Need to check/handle PPE and PXE validity too */
 664     ASSERT(FALSE);
 665 #endif
 666
 667     /* First things first, is the PDE valid? */
 668     ASSERT(PointerPde != MiAddressToPde(PTE_BASE));
 669     ASSERT(PointerPde->u.Hard.LargePage == 0);
 670     if (PointerPde->u.Hard.Valid == 0)
 671     {
 672         /* Right now, we only handle scenarios where the PDE is totally empty */
 673         ASSERT(PointerPde->u.Long == 0);
 674
 675         /* Check if this address range belongs to a valid allocation (VAD) */
 676         MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 677
 678         /* Right now, we expect a valid protection mask on the VAD */
 679         ASSERT(ProtectionCode != MM_NOACCESS);
 680
 681         /* Make the PDE demand-zero */
 682         MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde);
 683
 684         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
 685         Status = MiDispatchFault(TRUE,
 686                                  PointerPte,
 687                                  PointerPde,
 688                                  NULL,
 689                                  FALSE,
 690                                  PsGetCurrentProcess(),
 691                                  TrapInformation,
 692                                  NULL);
 693
 694         /* We should come back with APCs enabled, and with a valid PDE */
 695         ASSERT(KeAreAllApcsDisabled() == TRUE);
 696 #if (_MI_PAGING_LEVELS >= 3)
 697         /* Need to check/handle PPE and PXE validity too */
 698         ASSERT(FALSE);
 699 #endif
 700         ASSERT(PointerPde->u.Hard.Valid == 1);
 701     }
 702
 703     /* Now capture the PTE. We only handle cases where it's totally empty */
 704     TempPte = *PointerPte;
 705     ASSERT(TempPte.u.Long == 0);
 706
 707     /* Check if this address range belongs to a valid allocation (VAD) */
 708     ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 709     ASSERT(ProtectionCode != MM_NOACCESS);
 710
 711     /* Did we get a prototype PTE back? */
 712     if (!ProtoPte)
 713     {
 714         /* No, create a new PTE. First, write the protection */
 715         PointerPte->u.Soft.Protection = ProtectionCode;
 716
 717         /* Lock the PFN database since we're going to grab a page */
 718         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 719
 720         /* Grab a page out of there. Later we should grab a colored zero page */
 721         PageFrameIndex = MiRemoveAnyPage(0);
 722         ASSERT(PageFrameIndex);
 723
 724         /* Release the lock since we need to do some zeroing */
 725         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 726
 727         /* Zero out the page, since it's for user-mode */
 728         MiZeroPfn(PageFrameIndex);
 729
 730         /* Grab the lock again so we can initialize the PFN entry */
 731         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 732
 733         /* Initialize the PFN entry now */
 734         MiInitializePfn(PageFrameIndex, PointerPte, 1);
 735
 736         /* And we're done with the lock */
 737         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 738
 739         /* One more demand-zero fault */
 740         InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
 741
 742         /* Was the fault on an actual user page, or a kernel page for the user? */
 743         if (PointerPte <= MiHighestUserPte)
 744         {
 745             /* User fault, build a user PTE */
 746             MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 747                                       PointerPte,
 748                                       PointerPte->u.Soft.Protection,
 749                                       PageFrameIndex);
 750         }
 751         else
 752         {
 753             /* Session, kernel, or user PTE, figure it out and build it */
 754             MI_MAKE_HARDWARE_PTE(&TempPte,
 755                                  PointerPte,
 756                                  PointerPte->u.Soft.Protection,
 757                                  PageFrameIndex);
 758         }
 759
 760         /* Write the dirty bit for writeable pages */
 761         if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
 762
 763         /* And now write down the PTE, making the address valid */
 764         MI_WRITE_VALID_PTE(PointerPte, TempPte);
 765
 766         /* Demand zero */
 767         Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
 768     }
 769     else
 770     {
 771         /* The only "prototype PTE" we support is the shared user data path */
 772         ASSERT(ProtectionCode == MM_READONLY);
 773
 774         /* Write the prototype PTE */
 775         TempPte = PrototypePte;
 776         TempPte.u.Soft.Protection = ProtectionCode;
 777         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 778
 779         /* Handle the fault */
 780         Status = MiDispatchFault(StoreInstruction,
 781                                  Address,
 782                                  PointerPte,
 783                                  ProtoPte,
 784                                  FALSE,
 785                                  CurrentProcess,
 786                                  TrapInformation,
 787                                  Vad);
 788         ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
 789         ASSERT(PointerPte->u.Hard.Valid == 1);
 790         ASSERT(PointerPte->u.Hard.PageFrameNumber == MmSharedUserDataPte->u.Hard.PageFrameNumber);
 791     }
 792
 793     /* Release the working set */
 794     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 795     return Status;
 796 }
 797
 798 /* EOF */