reactos/ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #line 15 "ARM³::PAGFAULT"
  16 #define MODULE_INVOLVED_IN_ARM3
  17 #include "../ARM3/miarm.h"
  18
  19 /* GLOBALS ********************************************************************/
  20
  21 /* PRIVATE FUNCTIONS **********************************************************/
  22
  23 PMMPTE
  24 NTAPI
  25 MiCheckVirtualAddress(IN PVOID VirtualAddress,
  26                       OUT PULONG ProtectCode,
  27                       OUT PMMVAD *ProtoVad)
  28 {
  29     PMMVAD Vad;
  30
  31     /* No prototype/section support for now */
  32     *ProtoVad = NULL;
  33
  34     /* Only valid for user VADs for now */
  35     ASSERT(VirtualAddress <= MM_HIGHEST_USER_ADDRESS);
  36
  37     /* Special case for shared data */
  38     if (PAGE_ALIGN(VirtualAddress) == (PVOID)USER_SHARED_DATA)
  39     {
  40         /* It's a read-only page */
  41         *ProtectCode = MM_READONLY;
  42         return MmSharedUserDataPte;
  43     }
  44
  45     /* Find the VAD, it must exist, since we only handle PEB/TEB */
  46     Vad = MiLocateAddress(VirtualAddress);
  47     ASSERT(Vad);
  48
  49     /* This must be a TEB/PEB VAD */
  50     ASSERT(Vad->u.VadFlags.PrivateMemory == TRUE);
  51     ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
  52     ASSERT(Vad->u.VadFlags.VadType == VadNone);
  53
  54     /* Return the protection on it */
  55     *ProtectCode = Vad->u.VadFlags.Protection;
  56     return NULL;
  57 }
  58
  59 NTSTATUS
  60 FASTCALL
  61 MiCheckPdeForPagedPool(IN PVOID Address)
  62 {
  63     PMMPDE PointerPde;
  64     NTSTATUS Status = STATUS_SUCCESS;
  65
  66     /* No session support in ReactOS yet */
  67     ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
  68     ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
  69
  70     //
  71     // Check if this is a fault while trying to access the page table itself
  72     //
  73     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
  74     {
  75         //
  76         // Send a hint to the page fault handler that this is only a valid fault
  77         // if we already detected this was access within the page table range
  78         //
  79         PointerPde = (PMMPDE)MiAddressToPte(Address);
  80         Status = STATUS_WAIT_1;
  81     }
  82     else if (Address < MmSystemRangeStart)
  83     {
  84         //
  85         // This is totally illegal
  86         //
  87         return STATUS_ACCESS_VIOLATION;
  88     }
  89     else
  90     {
  91         //
  92         // Get the PDE for the address
  93         //
  94         PointerPde = MiAddressToPde(Address);
  95     }
  96
  97     //
  98     // Check if it's not valid
  99     //
 100     if (PointerPde->u.Hard.Valid == 0)
 101     {
 102 #ifdef _M_AMD64
 103         ASSERT(FALSE);
 104 #else
 105         /* This seems to be making the assumption that one PDE is one page long */
 106         C_ASSERT(PAGE_SIZE == (PD_COUNT * (sizeof(MMPTE) * PDE_COUNT)));
 107
 108         //
 109         // Copy it from our double-mapped system page directory
 110         //
 111         InterlockedExchangePte(PointerPde,
 112                                MmSystemPagePtes[((ULONG_PTR)PointerPde &
 113                                                  (PAGE_SIZE - 1)) /
 114                                                 sizeof(MMPTE)].u.Long);
 115 #endif
 116     }
 117
 118     //
 119     // Return status
 120     //
 121     return Status;
 122 }
 123
 124 VOID
 125 NTAPI
 126 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 127 {
 128     PMMPTE ZeroPte;
 129     MMPTE TempPte;
 130     PMMPFN Pfn1;
 131     PVOID ZeroAddress;
 132
 133     /* Get the PFN for this page */
 134     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 135     ASSERT(Pfn1);
 136
 137     /* Grab a system PTE we can use to zero the page */
 138     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 139     ASSERT(ZeroPte);
 140
 141     /* Initialize the PTE for it */
 142     TempPte = ValidKernelPte;
 143     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 144
 145     /* Setup caching */
 146     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 147     {
 148         /* Write combining, no caching */
 149         MI_PAGE_DISABLE_CACHE(&TempPte);
 150         MI_PAGE_WRITE_COMBINED(&TempPte);
 151     }
 152     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 153     {
 154         /* Write through, no caching */
 155         MI_PAGE_DISABLE_CACHE(&TempPte);
 156         MI_PAGE_WRITE_THROUGH(&TempPte);
 157     }
 158
 159     /* Make the system PTE valid with our PFN */
 160     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 161
 162     /* Get the address it maps to, and zero it out */
 163     ZeroAddress = MiPteToAddress(ZeroPte);
 164     KeZeroPages(ZeroAddress, PAGE_SIZE);
 165
 166     /* Now get rid of it */
 167     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 168 }
 169
 170 NTSTATUS
 171 NTAPI
 172 MiResolveDemandZeroFault(IN PVOID Address,
 173                          IN PMMPTE PointerPte,
 174                          IN PEPROCESS Process,
 175                          IN KIRQL OldIrql)
 176 {
 177     PFN_NUMBER PageFrameNumber;
 178     MMPTE TempPte;
 179     BOOLEAN NeedZero = FALSE;
 180     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 181             Address,
 182             Process);
 183
 184     /* Must currently only be called by paging path */
 185     ASSERT(OldIrql == MM_NOIRQL);
 186     if (Process)
 187     {
 188         /* Sanity check */
 189         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 190
 191         /* No forking yet */
 192         ASSERT(Process->ForkInProgress == NULL);
 193
 194         /* We'll need a zero page */
 195         NeedZero = TRUE;
 196     }
 197
 198     //
 199     // Lock the PFN database
 200     //
 201     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 202     ASSERT(PointerPte->u.Hard.Valid == 0);
 203
 204     /* Get a page */
 205     PageFrameNumber = MiRemoveAnyPage(0);
 206     DPRINT("New pool page: %lx\n", PageFrameNumber);
 207
 208     /* Initialize it */
 209     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 210
 211     //
 212     // Release PFN lock
 213     //
 214     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 215
 216     //
 217     // Increment demand zero faults
 218     //
 219     InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
 220
 221     /* Zero the page if need be */
 222     if (NeedZero) MiZeroPfn(PageFrameNumber);
 223
 224     /* Build the PTE */
 225     if (PointerPte <= MiHighestUserPte)
 226     {
 227         /* For user mode */
 228         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 229                                   PointerPte,
 230                                   PointerPte->u.Soft.Protection,
 231                                   PageFrameNumber);
 232     }
 233     else
 234     {
 235         /* For kernel mode */
 236         MI_MAKE_HARDWARE_PTE(&TempPte,
 237                              PointerPte,
 238                              PointerPte->u.Soft.Protection,
 239                              PageFrameNumber);
 240     }
 241
 242     /* Set it dirty if it's a writable page */
 243     if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
 244
 245     /* Write it */
 246     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 247
 248     //
 249     // It's all good now
 250     //
 251     DPRINT("Paged pool page has now been paged in\n");
 252     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 253 }
 254
 255 NTSTATUS
 256 NTAPI
 257 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 258                         IN PVOID Address,
 259                         IN PMMPTE PointerPte,
 260                         IN PMMPTE PointerProtoPte,
 261                         IN KIRQL OldIrql,
 262                         IN PMMPFN Pfn1)
 263 {
 264     MMPTE TempPte;
 265     PFN_NUMBER PageFrameIndex;
 266
 267     /* Must be called with an valid prototype PTE, with the PFN lock held */
 268     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 269     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 270
 271     /* Quick-n-dirty */
 272     ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
 273
 274     /* Get the page */
 275     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 276
 277     /* Release the PFN lock */
 278     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 279
 280     /* Build the user PTE */
 281     ASSERT(Address < MmSystemRangeStart);
 282     MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, MM_READONLY, PageFrameIndex);
 283
 284     /* Write the PTE */
 285     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 286
 287     /* Return success */
 288     return STATUS_SUCCESS;
 289 }
 290
 291 NTSTATUS
 292 NTAPI
 293 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
 294                        IN PVOID Address,
 295                        IN PMMPTE PointerPte,
 296                        IN PMMPTE PointerProtoPte,
 297                        IN OUT PMMPFN *OutPfn,
 298                        OUT PVOID *PageFileData,
 299                        OUT PMMPTE PteValue,
 300                        IN PEPROCESS Process,
 301                        IN KIRQL OldIrql,
 302                        IN PVOID TrapInformation)
 303 {
 304     MMPTE TempPte;
 305     PMMPFN Pfn1;
 306     PFN_NUMBER PageFrameIndex;
 307
 308     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
 309     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 310     ASSERT(PointerPte->u.Hard.Valid == 0);
 311     ASSERT(PointerPte->u.Soft.Prototype == 1);
 312
 313     /* Read the prototype PTE -- it must be valid since we only handle shared data */
 314     TempPte = *PointerProtoPte;
 315     ASSERT(TempPte.u.Hard.Valid == 1);
 316
 317     /* One more user of this mapped page */
 318     PageFrameIndex = PFN_FROM_PTE(&TempPte);
 319     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 320     Pfn1->u2.ShareCount++;
 321
 322     /* Call it a transition */
 323     InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
 324
 325     /* Complete the prototype PTE fault -- this will release the PFN lock */
 326     return MiCompleteProtoPteFault(StoreInstruction,
 327                                    Address,
 328                                    PointerPte,
 329                                    PointerProtoPte,
 330                                    OldIrql,
 331                                    NULL);
 332 }
 333
 334 NTSTATUS
 335 NTAPI
 336 MiDispatchFault(IN BOOLEAN StoreInstruction,
 337                 IN PVOID Address,
 338                 IN PMMPTE PointerPte,
 339                 IN PMMPTE PointerProtoPte,
 340                 IN BOOLEAN Recursive,
 341                 IN PEPROCESS Process,
 342                 IN PVOID TrapInformation,
 343                 IN PVOID Vad)
 344 {
 345     MMPTE TempPte;
 346     KIRQL OldIrql, LockIrql;
 347     NTSTATUS Status;
 348     PMMPTE SuperProtoPte;
 349     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
 350              Address,
 351              Process);
 352
 353     //
 354     // Make sure APCs are off and we're not at dispatch
 355     //
 356     OldIrql = KeGetCurrentIrql();
 357     ASSERT(OldIrql <= APC_LEVEL);
 358     ASSERT(KeAreAllApcsDisabled() == TRUE);
 359
 360     //
 361     // Grab a copy of the PTE
 362     //
 363     TempPte = *PointerPte;
 364
 365     /* Do we have a prototype PTE? */
 366     if (PointerProtoPte)
 367     {
 368         /* This should never happen */
 369         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
 370
 371         /* We currently only handle the shared user data PTE path */
 372         ASSERT(Address < MmSystemRangeStart);
 373         ASSERT(PointerPte->u.Soft.Prototype == 1);
 374         ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
 375         ASSERT(Vad == NULL);
 376
 377         /* Lock the PFN database */
 378         LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 379
 380         /* For the shared data page, this should be true */
 381         SuperProtoPte = MiAddressToPte(PointerProtoPte);
 382         ASSERT(SuperProtoPte->u.Hard.Valid == 1);
 383         ASSERT(TempPte.u.Hard.Valid == 0);
 384
 385         /* Resolve the fault -- this will release the PFN lock */
 386         Status = MiResolveProtoPteFault(StoreInstruction,
 387                                         Address,
 388                                         PointerPte,
 389                                         PointerProtoPte,
 390                                         NULL,
 391                                         NULL,
 392                                         NULL,
 393                                         Process,
 394                                         LockIrql,
 395                                         TrapInformation);
 396         ASSERT(Status == STATUS_SUCCESS);
 397
 398         /* Complete this as a transition fault */
 399         ASSERT(OldIrql == KeGetCurrentIrql());
 400         ASSERT(OldIrql <= APC_LEVEL);
 401         ASSERT(KeAreAllApcsDisabled() == TRUE);
 402         return STATUS_PAGE_FAULT_TRANSITION;
 403     }
 404
 405     //
 406     // The PTE must be invalid, but not totally blank
 407     //
 408     ASSERT(TempPte.u.Hard.Valid == 0);
 409     ASSERT(TempPte.u.Long != 0);
 410
 411     //
 412     // No prototype, transition or page file software PTEs in ARM3 yet
 413     //
 414     ASSERT(TempPte.u.Soft.Prototype == 0);
 415     ASSERT(TempPte.u.Soft.Transition == 0);
 416     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 417
 418     //
 419     // If we got this far, the PTE can only be a demand zero PTE, which is what
 420     // we want. Go handle it!
 421     //
 422     Status = MiResolveDemandZeroFault(Address,
 423                                       PointerPte,
 424                                       Process,
 425                                       MM_NOIRQL);
 426     ASSERT(KeAreAllApcsDisabled () == TRUE);
 427     if (NT_SUCCESS(Status))
 428     {
 429         //
 430         // Make sure we're returning in a sane state and pass the status down
 431         //
 432         ASSERT(OldIrql == KeGetCurrentIrql ());
 433         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 434         return Status;
 435     }
 436
 437     //
 438     // Generate an access fault
 439     //
 440     return STATUS_ACCESS_VIOLATION;
 441 }
 442
 443 NTSTATUS
 444 NTAPI
 445 MmArmAccessFault(IN BOOLEAN StoreInstruction,
 446                  IN PVOID Address,
 447                  IN KPROCESSOR_MODE Mode,
 448                  IN PVOID TrapInformation)
 449 {
 450     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
 451     PMMPTE PointerPte, ProtoPte;
 452     PMMPDE PointerPde;
 453     MMPTE TempPte;
 454     PETHREAD CurrentThread;
 455     PEPROCESS CurrentProcess;
 456     NTSTATUS Status;
 457     PMMSUPPORT WorkingSet;
 458     ULONG ProtectionCode;
 459     PMMVAD Vad;
 460     PFN_NUMBER PageFrameIndex;
 461     DPRINT("ARM3 FAULT AT: %p\n", Address);
 462
 463     //
 464     // Get the PTE and PDE
 465     //
 466     PointerPte = MiAddressToPte(Address);
 467     PointerPde = MiAddressToPde(Address);
 468 #if (_MI_PAGING_LEVELS >= 3)
 469     /* We need the PPE and PXE addresses */
 470     ASSERT(FALSE);
 471 #endif
 472
 473     //
 474     // Check for dispatch-level snafu
 475     //
 476     if (OldIrql > APC_LEVEL)
 477     {
 478         //
 479         // There are some special cases where this is okay, but not in ARM3 yet
 480         //
 481         DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
 482                  Address,
 483                  OldIrql);
 484         ASSERT(OldIrql <= APC_LEVEL);
 485     }
 486
 487     //
 488     // Check for kernel fault
 489     //
 490     if (Address >= MmSystemRangeStart)
 491     {
 492         //
 493         // What are you even DOING here?
 494         //
 495         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
 496
 497 #if (_MI_PAGING_LEVELS >= 3)
 498         /* Need to check PXE and PDE validity */
 499         ASSERT(FALSE);
 500 #endif
 501
 502         //
 503         // Is the PDE valid?
 504         //
 505         if (!PointerPde->u.Hard.Valid == 0)
 506         {
 507             //
 508             // Debug spew (eww!)
 509             //
 510             DPRINT("Invalid PDE\n");
 511 #if (_MI_PAGING_LEVELS == 2)
 512             //
 513             // Handle mapping in "Special" PDE directoreis
 514             //
 515             MiCheckPdeForPagedPool(Address);
 516 #endif
 517             //
 518             // Now we SHOULD be good
 519             //
 520             if (PointerPde->u.Hard.Valid == 0)
 521             {
 522                 //
 523                 // FIXFIX: Do the S-LIST hack
 524                 //
 525
 526                 //
 527                 // Kill the system
 528                 //
 529                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 530                              (ULONG_PTR)Address,
 531                              StoreInstruction,
 532                              (ULONG_PTR)TrapInformation,
 533                              2);
 534             }
 535         }
 536
 537         //
 538         // The PDE is valid, so read the PTE
 539         //
 540         TempPte = *PointerPte;
 541         if (TempPte.u.Hard.Valid == 1)
 542         {
 543             //
 544             // Only two things can go wrong here:
 545             // Executing NX page (we couldn't care less)
 546             // Writing to a read-only page (the stuff ARM3 works with is write,
 547             // so again, moot point).
 548             //
 549             if (StoreInstruction)
 550             {
 551                 DPRINT1("Should NEVER happen on ARM3!!!\n");
 552                 return STATUS_ACCESS_VIOLATION;
 553             }
 554
 555             //
 556             // Otherwise, the PDE was probably invalid, and all is good now
 557             //
 558             return STATUS_SUCCESS;
 559         }
 560
 561         //
 562         // Check for a fault on the page table or hyperspace itself
 563         //
 564         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
 565         {
 566             //
 567             // This might happen...not sure yet
 568             //
 569             DPRINT1("FAULT ON PAGE TABLES: %p %lx %lx!\n", Address, *PointerPte, *PointerPde);
 570 #if (_MI_PAGING_LEVELS == 2)
 571             //
 572             // Map in the page table
 573             //
 574             if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
 575             {
 576                 DPRINT1("PAGE TABLES FAULTED IN!\n");
 577                 return STATUS_SUCCESS;
 578             }
 579 #endif
 580             //
 581             // Otherwise the page table doesn't actually exist
 582             //
 583             DPRINT1("FAILING\n");
 584             return STATUS_ACCESS_VIOLATION;
 585         }
 586
 587         /* In this path, we are using the system working set */
 588         CurrentThread = PsGetCurrentThread();
 589         WorkingSet = &MmSystemCacheWs;
 590
 591         /* Acquire it */
 592         KeRaiseIrql(APC_LEVEL, &LockIrql);
 593         MiLockWorkingSet(CurrentThread, WorkingSet);
 594
 595         //
 596         // Re-read PTE now that the IRQL has been raised
 597         //
 598         TempPte = *PointerPte;
 599         if (TempPte.u.Hard.Valid == 1)
 600         {
 601             //
 602             // Only two things can go wrong here:
 603             // Executing NX page (we couldn't care less)
 604             // Writing to a read-only page (the stuff ARM3 works with is write,
 605             // so again, moot point.
 606             //
 607             if (StoreInstruction)
 608             {
 609                 DPRINT1("Should NEVER happen on ARM3!!!\n");
 610                 return STATUS_ACCESS_VIOLATION;
 611             }
 612
 613             /* Release the working set */
 614             MiUnlockWorkingSet(CurrentThread, WorkingSet);
 615             KeLowerIrql(LockIrql);
 616
 617             //
 618             // Otherwise, the PDE was probably invalid, and all is good now
 619             //
 620             return STATUS_SUCCESS;
 621         }
 622
 623         //
 624         // We don't implement prototype PTEs
 625         //
 626         ASSERT(TempPte.u.Soft.Prototype == 0);
 627
 628         //
 629         // We don't implement transition PTEs
 630         //
 631         ASSERT(TempPte.u.Soft.Transition == 0);
 632
 633         //
 634         // Now do the real fault handling
 635         //
 636         Status = MiDispatchFault(StoreInstruction,
 637                                  Address,
 638                                  PointerPte,
 639                                  NULL,
 640                                  FALSE,
 641                                  NULL,
 642                                  TrapInformation,
 643                                  NULL);
 644
 645         /* Release the working set */
 646         ASSERT(KeAreAllApcsDisabled() == TRUE);
 647         MiUnlockWorkingSet(CurrentThread, WorkingSet);
 648         KeLowerIrql(LockIrql);
 649
 650         //
 651         // We are done!
 652         //
 653         DPRINT("Fault resolved with status: %lx\n", Status);
 654         return Status;
 655     }
 656
 657     /* This is a user fault */
 658     CurrentThread = PsGetCurrentThread();
 659     CurrentProcess = PsGetCurrentProcess();
 660
 661     /* Lock the working set */
 662     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
 663
 664 #if (_MI_PAGING_LEVELS >= 3)
 665     /* Need to check/handle PPE and PXE validity too */
 666     ASSERT(FALSE);
 667 #endif
 668
 669     /* First things first, is the PDE valid? */
 670     ASSERT(PointerPde != MiAddressToPde(PTE_BASE));
 671     ASSERT(PointerPde->u.Hard.LargePage == 0);
 672     if (PointerPde->u.Hard.Valid == 0)
 673     {
 674         /* Right now, we only handle scenarios where the PDE is totally empty */
 675         ASSERT(PointerPde->u.Long == 0);
 676
 677         /* Check if this address range belongs to a valid allocation (VAD) */
 678         MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 679
 680         /* Right now, we expect a valid protection mask on the VAD */
 681         ASSERT(ProtectionCode != MM_NOACCESS);
 682
 683         /* Make the PDE demand-zero */
 684         MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde);
 685
 686         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
 687         Status = MiDispatchFault(TRUE,
 688                                  PointerPte,
 689                                  PointerPde,
 690                                  NULL,
 691                                  FALSE,
 692                                  PsGetCurrentProcess(),
 693                                  TrapInformation,
 694                                  NULL);
 695
 696         /* We should come back with APCs enabled, and with a valid PDE */
 697         ASSERT(KeAreAllApcsDisabled() == TRUE);
 698 #if (_MI_PAGING_LEVELS >= 3)
 699         /* Need to check/handle PPE and PXE validity too */
 700         ASSERT(FALSE);
 701 #endif
 702         ASSERT(PointerPde->u.Hard.Valid == 1);
 703     }
 704
 705     /* Now capture the PTE. We only handle cases where it's totally empty */
 706     TempPte = *PointerPte;
 707     ASSERT(TempPte.u.Long == 0);
 708
 709     /* Check if this address range belongs to a valid allocation (VAD) */
 710     ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 711     ASSERT(ProtectionCode != MM_NOACCESS);
 712
 713     /* Did we get a prototype PTE back? */
 714     if (!ProtoPte)
 715     {
 716         /* No, create a new PTE. First, write the protection */
 717         PointerPte->u.Soft.Protection = ProtectionCode;
 718
 719         /* Lock the PFN database since we're going to grab a page */
 720         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 721
 722         /* Grab a page out of there. Later we should grab a colored zero page */
 723         PageFrameIndex = MiRemoveAnyPage(0);
 724         ASSERT(PageFrameIndex);
 725
 726         /* Release the lock since we need to do some zeroing */
 727         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 728
 729         /* Zero out the page, since it's for user-mode */
 730         MiZeroPfn(PageFrameIndex);
 731
 732         /* Grab the lock again so we can initialize the PFN entry */
 733         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 734
 735         /* Initialize the PFN entry now */
 736         MiInitializePfn(PageFrameIndex, PointerPte, 1);
 737
 738         /* And we're done with the lock */
 739         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 740
 741         /* One more demand-zero fault */
 742         InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
 743
 744         /* Was the fault on an actual user page, or a kernel page for the user? */
 745         if (PointerPte <= MiHighestUserPte)
 746         {
 747             /* User fault, build a user PTE */
 748             MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 749                                       PointerPte,
 750                                       PointerPte->u.Soft.Protection,
 751                                       PageFrameIndex);
 752         }
 753         else
 754         {
 755             /* Session, kernel, or user PTE, figure it out and build it */
 756             MI_MAKE_HARDWARE_PTE(&TempPte,
 757                                  PointerPte,
 758                                  PointerPte->u.Soft.Protection,
 759                                  PageFrameIndex);
 760         }
 761
 762         /* Write the dirty bit for writeable pages */
 763         if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
 764
 765         /* And now write down the PTE, making the address valid */
 766         MI_WRITE_VALID_PTE(PointerPte, TempPte);
 767
 768         /* Demand zero */
 769         Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
 770     }
 771     else
 772     {
 773         /* The only "prototype PTE" we support is the shared user data path */
 774         ASSERT(ProtectionCode == MM_READONLY);
 775
 776         /* Write the prototype PTE */
 777         TempPte = PrototypePte;
 778         TempPte.u.Soft.Protection = ProtectionCode;
 779         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 780
 781         /* Handle the fault */
 782         Status = MiDispatchFault(StoreInstruction,
 783                                  Address,
 784                                  PointerPte,
 785                                  ProtoPte,
 786                                  FALSE,
 787                                  CurrentProcess,
 788                                  TrapInformation,
 789                                  Vad);
 790         ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
 791         ASSERT(PointerPte->u.Hard.Valid == 1);
 792         ASSERT(PointerPte->u.Hard.PageFrameNumber == MmSharedUserDataPte->u.Hard.PageFrameNumber);
 793     }
 794
 795     /* Release the working set */
 796     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 797     return Status;
 798 }
 799
 800 /* EOF */