[NTOS]: Add support for handling a very specific type of user-fault on ARM3 memory...
[reactos.git] / reactos / ntoskrnl / mm / ARM3 / pagfault.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pagfault.c
5 * PURPOSE: ARM Memory Manager Page Fault Handling
6 * PROGRAMMERS: ReactOS Portable Systems Group
7 */
8
9 /* INCLUDES *******************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 #line 15 "ARMĀ³::PAGFAULT"
16 #define MODULE_INVOLVED_IN_ARM3
17 #include "../ARM3/miarm.h"
18
19 /* GLOBALS ********************************************************************/
20
21 /* PRIVATE FUNCTIONS **********************************************************/
22
23 PMMPTE
24 NTAPI
25 MiCheckVirtualAddress(IN PVOID VirtualAddress,
26 OUT PULONG ProtectCode,
27 OUT PMMVAD *ProtoVad)
28 {
29 PMMVAD Vad;
30
31 /* No prototype/section support for now */
32 *ProtoVad = NULL;
33
34 /* Only valid for user VADs for now */
35 ASSERT(VirtualAddress <= MM_HIGHEST_USER_ADDRESS);
36
37 /* Find the VAD, it must exist, since we only handle PEB/TEB */
38 Vad = MiLocateAddress(VirtualAddress);
39 ASSERT(Vad);
40
41 /* This must be a TEB/PEB VAD */
42 ASSERT(Vad->u.VadFlags.PrivateMemory == TRUE);
43 ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
44 ASSERT(Vad->u.VadFlags.VadType == VadNone);
45
46 /* Return the protection on it */
47 *ProtectCode = Vad->u.VadFlags.Protection;
48 return NULL;
49 }
50
51 NTSTATUS
52 FASTCALL
53 MiCheckPdeForPagedPool(IN PVOID Address)
54 {
55 PMMPDE PointerPde;
56 NTSTATUS Status = STATUS_SUCCESS;
57
58 /* No session support in ReactOS yet */
59 ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
60 ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
61
62 //
63 // Check if this is a fault while trying to access the page table itself
64 //
65 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
66 {
67 //
68 // Send a hint to the page fault handler that this is only a valid fault
69 // if we already detected this was access within the page table range
70 //
71 PointerPde = (PMMPDE)MiAddressToPte(Address);
72 Status = STATUS_WAIT_1;
73 }
74 else if (Address < MmSystemRangeStart)
75 {
76 //
77 // This is totally illegal
78 //
79 return STATUS_ACCESS_VIOLATION;
80 }
81 else
82 {
83 //
84 // Get the PDE for the address
85 //
86 PointerPde = MiAddressToPde(Address);
87 }
88
89 //
90 // Check if it's not valid
91 //
92 if (PointerPde->u.Hard.Valid == 0)
93 {
94 #ifndef _M_AMD64
95 /* This seems to be making the assumption that one PDE is one page long */
96 C_ASSERT(PAGE_SIZE == (PD_COUNT * (sizeof(MMPTE) * PDE_COUNT)));
97 #endif
98
99 //
100 // Copy it from our double-mapped system page directory
101 //
102 InterlockedExchangePte(PointerPde,
103 MmSystemPagePtes[((ULONG_PTR)PointerPde &
104 (PAGE_SIZE - 1)) /
105 sizeof(MMPTE)].u.Long);
106 }
107
108 //
109 // Return status
110 //
111 return Status;
112 }
113
114 VOID
115 NTAPI
116 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
117 {
118 PMMPTE ZeroPte;
119 MMPTE TempPte;
120 PMMPFN Pfn1;
121 PVOID ZeroAddress;
122
123 /* Get the PFN for this page */
124 Pfn1 = MiGetPfnEntry(PageFrameNumber);
125 ASSERT(Pfn1);
126
127 /* Grab a system PTE we can use to zero the page */
128 ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
129 ASSERT(ZeroPte);
130
131 /* Initialize the PTE for it */
132 TempPte = ValidKernelPte;
133 TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
134
135 /* Setup caching */
136 if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
137 {
138 /* Write combining, no caching */
139 MI_PAGE_DISABLE_CACHE(&TempPte);
140 MI_PAGE_WRITE_COMBINED(&TempPte);
141 }
142 else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
143 {
144 /* Write through, no caching */
145 MI_PAGE_DISABLE_CACHE(&TempPte);
146 MI_PAGE_WRITE_THROUGH(&TempPte);
147 }
148
149 /* Make the system PTE valid with our PFN */
150 MI_WRITE_VALID_PTE(ZeroPte, TempPte);
151
152 /* Get the address it maps to, and zero it out */
153 ZeroAddress = MiPteToAddress(ZeroPte);
154 KeZeroPages(ZeroAddress, PAGE_SIZE);
155
156 /* Now get rid of it */
157 MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
158 }
159
160 NTSTATUS
161 NTAPI
162 MiResolveDemandZeroFault(IN PVOID Address,
163 IN PMMPTE PointerPte,
164 IN PEPROCESS Process,
165 IN KIRQL OldIrql)
166 {
167 PFN_NUMBER PageFrameNumber;
168 MMPTE TempPte;
169 BOOLEAN NeedZero = FALSE;
170 DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
171 Address,
172 Process);
173
174 /* Must currently only be called by paging path */
175 ASSERT(OldIrql == MM_NOIRQL);
176 if (Process)
177 {
178 /* Sanity check */
179 ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
180
181 /* No forking yet */
182 ASSERT(Process->ForkInProgress == NULL);
183
184 /* We'll need a zero page */
185 NeedZero = TRUE;
186 }
187
188 //
189 // Lock the PFN database
190 //
191 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
192 ASSERT(PointerPte->u.Hard.Valid == 0);
193
194 /* Get a page */
195 PageFrameNumber = MiRemoveAnyPage(0);
196 DPRINT("New pool page: %lx\n", PageFrameNumber);
197
198 /* Initialize it */
199 MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
200
201 //
202 // Release PFN lock
203 //
204 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
205
206 //
207 // Increment demand zero faults
208 //
209 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
210
211 /* Zero the page if need be */
212 if (NeedZero) MiZeroPfn(PageFrameNumber);
213
214 /* Build the PTE */
215 if (PointerPte <= MiHighestUserPte)
216 {
217 /* For user mode */
218 MI_MAKE_HARDWARE_PTE_USER(&TempPte,
219 PointerPte,
220 PointerPte->u.Soft.Protection,
221 PageFrameNumber);
222 }
223 else
224 {
225 /* For kernel mode */
226 MI_MAKE_HARDWARE_PTE(&TempPte,
227 PointerPte,
228 PointerPte->u.Soft.Protection,
229 PageFrameNumber);
230 }
231
232 /* Set it dirty if it's a writable page */
233 if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
234
235 /* Write it */
236 MI_WRITE_VALID_PTE(PointerPte, TempPte);
237
238 //
239 // It's all good now
240 //
241 DPRINT("Paged pool page has now been paged in\n");
242 return STATUS_PAGE_FAULT_DEMAND_ZERO;
243 }
244
245 NTSTATUS
246 NTAPI
247 MiDispatchFault(IN BOOLEAN StoreInstruction,
248 IN PVOID Address,
249 IN PMMPTE PointerPte,
250 IN PMMPTE PrototypePte,
251 IN BOOLEAN Recursive,
252 IN PEPROCESS Process,
253 IN PVOID TrapInformation,
254 IN PVOID Vad)
255 {
256 MMPTE TempPte;
257 KIRQL OldIrql;
258 NTSTATUS Status;
259 DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
260 Address,
261 Process);
262
263 //
264 // Make sure APCs are off and we're not at dispatch
265 //
266 OldIrql = KeGetCurrentIrql ();
267 ASSERT(OldIrql <= APC_LEVEL);
268 ASSERT(KeAreAllApcsDisabled () == TRUE);
269
270 //
271 // Grab a copy of the PTE
272 //
273 TempPte = *PointerPte;
274
275 /* No prototype */
276 ASSERT(PrototypePte == NULL);
277
278 //
279 // The PTE must be invalid, but not totally blank
280 //
281 ASSERT(TempPte.u.Hard.Valid == 0);
282 ASSERT(TempPte.u.Long != 0);
283
284 //
285 // No prototype, transition or page file software PTEs in ARM3 yet
286 //
287 ASSERT(TempPte.u.Soft.Prototype == 0);
288 ASSERT(TempPte.u.Soft.Transition == 0);
289 ASSERT(TempPte.u.Soft.PageFileHigh == 0);
290
291 //
292 // If we got this far, the PTE can only be a demand zero PTE, which is what
293 // we want. Go handle it!
294 //
295 Status = MiResolveDemandZeroFault(Address,
296 PointerPte,
297 Process,
298 MM_NOIRQL);
299 ASSERT(KeAreAllApcsDisabled () == TRUE);
300 if (NT_SUCCESS(Status))
301 {
302 //
303 // Make sure we're returning in a sane state and pass the status down
304 //
305 ASSERT(OldIrql == KeGetCurrentIrql ());
306 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
307 return Status;
308 }
309
310 //
311 // Generate an access fault
312 //
313 return STATUS_ACCESS_VIOLATION;
314 }
315
316 NTSTATUS
317 NTAPI
318 MmArmAccessFault(IN BOOLEAN StoreInstruction,
319 IN PVOID Address,
320 IN KPROCESSOR_MODE Mode,
321 IN PVOID TrapInformation)
322 {
323 KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
324 PMMPTE PointerPte;
325 PMMPDE PointerPde;
326 MMPTE TempPte;
327 PETHREAD CurrentThread;
328 PEPROCESS CurrentProcess;
329 NTSTATUS Status;
330 PMMSUPPORT WorkingSet;
331 ULONG ProtectionCode;
332 PMMVAD Vad;
333 PFN_NUMBER PageFrameIndex;
334 DPRINT("ARM3 FAULT AT: %p\n", Address);
335
336 //
337 // Get the PTE and PDE
338 //
339 PointerPte = MiAddressToPte(Address);
340 PointerPde = MiAddressToPde(Address);
341
342 //
343 // Check for dispatch-level snafu
344 //
345 if (OldIrql > APC_LEVEL)
346 {
347 //
348 // There are some special cases where this is okay, but not in ARM3 yet
349 //
350 DbgPrint("MM:***PAGE FAULT AT IRQL > 1 Va %p, IRQL %lx\n",
351 Address,
352 OldIrql);
353 ASSERT(OldIrql <= APC_LEVEL);
354 }
355
356 //
357 // Check for kernel fault
358 //
359 if (Address >= MmSystemRangeStart)
360 {
361 //
362 // What are you even DOING here?
363 //
364 if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
365
366 //
367 // Is the PDE valid?
368 //
369 if (!PointerPde->u.Hard.Valid == 0)
370 {
371 //
372 // Debug spew (eww!)
373 //
374 DPRINT("Invalid PDE\n");
375
376 //
377 // Handle mapping in "Special" PDE directoreis
378 //
379 MiCheckPdeForPagedPool(Address);
380
381 //
382 // Now we SHOULD be good
383 //
384 if (PointerPde->u.Hard.Valid == 0)
385 {
386 //
387 // FIXFIX: Do the S-LIST hack
388 //
389
390 //
391 // Kill the system
392 //
393 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
394 (ULONG_PTR)Address,
395 StoreInstruction,
396 (ULONG_PTR)TrapInformation,
397 2);
398 }
399 }
400
401 //
402 // The PDE is valid, so read the PTE
403 //
404 TempPte = *PointerPte;
405 if (TempPte.u.Hard.Valid == 1)
406 {
407 //
408 // Only two things can go wrong here:
409 // Executing NX page (we couldn't care less)
410 // Writing to a read-only page (the stuff ARM3 works with is write,
411 // so again, moot point).
412 //
413 if (StoreInstruction)
414 {
415 DPRINT1("Should NEVER happen on ARM3!!!\n");
416 return STATUS_ACCESS_VIOLATION;
417 }
418
419 //
420 // Otherwise, the PDE was probably invalid, and all is good now
421 //
422 return STATUS_SUCCESS;
423 }
424
425 //
426 // Check for a fault on the page table or hyperspace itself
427 //
428 if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
429 {
430 //
431 // This might happen...not sure yet
432 //
433 DPRINT1("FAULT ON PAGE TABLES: %p %lx %lx!\n", Address, *PointerPte, *PointerPde);
434
435 //
436 // Map in the page table
437 //
438 if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
439 {
440 DPRINT1("PAGE TABLES FAULTED IN!\n");
441 return STATUS_SUCCESS;
442 }
443
444 //
445 // Otherwise the page table doesn't actually exist
446 //
447 DPRINT1("FAILING\n");
448 return STATUS_ACCESS_VIOLATION;
449 }
450
451 /* In this path, we are using the system working set */
452 CurrentThread = PsGetCurrentThread();
453 WorkingSet = &MmSystemCacheWs;
454
455 /* Acquire it */
456 KeRaiseIrql(APC_LEVEL, &LockIrql);
457 MiLockWorkingSet(CurrentThread, WorkingSet);
458
459 //
460 // Re-read PTE now that the IRQL has been raised
461 //
462 TempPte = *PointerPte;
463 if (TempPte.u.Hard.Valid == 1)
464 {
465 //
466 // Only two things can go wrong here:
467 // Executing NX page (we couldn't care less)
468 // Writing to a read-only page (the stuff ARM3 works with is write,
469 // so again, moot point.
470 //
471 if (StoreInstruction)
472 {
473 DPRINT1("Should NEVER happen on ARM3!!!\n");
474 return STATUS_ACCESS_VIOLATION;
475 }
476
477 /* Release the working set */
478 MiUnlockWorkingSet(CurrentThread, WorkingSet);
479 KeLowerIrql(LockIrql);
480
481 //
482 // Otherwise, the PDE was probably invalid, and all is good now
483 //
484 return STATUS_SUCCESS;
485 }
486
487 //
488 // We don't implement prototype PTEs
489 //
490 ASSERT(TempPte.u.Soft.Prototype == 0);
491
492 //
493 // We don't implement transition PTEs
494 //
495 ASSERT(TempPte.u.Soft.Transition == 0);
496
497 //
498 // Now do the real fault handling
499 //
500 Status = MiDispatchFault(StoreInstruction,
501 Address,
502 PointerPte,
503 NULL,
504 FALSE,
505 NULL,
506 TrapInformation,
507 NULL);
508
509 /* Release the working set */
510 ASSERT(KeAreAllApcsDisabled() == TRUE);
511 MiUnlockWorkingSet(CurrentThread, WorkingSet);
512 KeLowerIrql(LockIrql);
513
514 //
515 // We are done!
516 //
517 DPRINT("Fault resolved with status: %lx\n", Status);
518 return Status;
519 }
520
521 /* This is a user fault */
522 CurrentThread = PsGetCurrentThread();
523 CurrentProcess = PsGetCurrentProcess();
524
525 /* Lock the working set */
526 MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
527
528 /* First things first, is the PDE valid? */
529 ASSERT(PointerPde != MiAddressToPde(PTE_BASE));
530 ASSERT(PointerPde->u.Hard.LargePage == 0);
531 if (PointerPde->u.Hard.Valid == 0)
532 {
533 /* Right now, we only handle scenarios where the PDE is totally empty */
534 ASSERT(PointerPde->u.Long == 0);
535
536 /* Check if this address range belongs to a valid allocation (VAD) */
537 MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
538
539 /* Right now, we expect a valid protection mask on the VAD */
540 ASSERT(ProtectionCode != MM_NOACCESS);
541
542 /* Make the PDE demand-zero */
543 MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde);
544
545 /* And go dispatch the fault on the PDE. This should handle the demand-zero */
546 Status = MiDispatchFault(TRUE,
547 PointerPte,
548 PointerPde,
549 NULL,
550 FALSE,
551 PsGetCurrentProcess(),
552 TrapInformation,
553 NULL);
554
555 /* We should come back with APCs enabled, and with a valid PDE */
556 ASSERT(KeAreAllApcsDisabled() == TRUE);
557 ASSERT(PointerPde->u.Hard.Valid == 1);
558 }
559
560 /* Now capture the PTE. We only handle cases where it's totally empty */
561 TempPte = *PointerPte;
562 ASSERT(TempPte.u.Long == 0);
563
564 /* Check if this address range belongs to a valid allocation (VAD) */
565 MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
566
567 /* Right now, we expect a valid protection mask on the VAD */
568 ASSERT(ProtectionCode != MM_NOACCESS);
569 PointerPte->u.Soft.Protection = ProtectionCode;
570
571 /* Lock the PFN database since we're going to grab a page */
572 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
573
574 /* Grab a page out of there. Later we should grab a colored zero page */
575 PageFrameIndex = MiRemoveAnyPage(0);
576 ASSERT(PageFrameIndex);
577
578 /* Release the lock since we need to do some zeroing */
579 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
580
581 /* Zero out the page, since it's for user-mode */
582 MiZeroPfn(PageFrameIndex);
583
584 /* Grab the lock again so we can initialize the PFN entry */
585 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
586
587 /* Initialize the PFN entry now */
588 MiInitializePfn(PageFrameIndex, PointerPte, 1);
589
590 /* And we're done with the lock */
591 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
592
593 /* One more demand-zero fault */
594 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
595
596 /* Was the fault on an actual user page, or a kernel page for the user? */
597 if (PointerPte <= MiHighestUserPte)
598 {
599 /* User fault, build a user PTE */
600 MI_MAKE_HARDWARE_PTE_USER(&TempPte,
601 PointerPte,
602 PointerPte->u.Soft.Protection,
603 PageFrameIndex);
604 }
605 else
606 {
607 /* Session, kernel, or user PTE, figure it out and build it */
608 MI_MAKE_HARDWARE_PTE(&TempPte,
609 PointerPte,
610 PointerPte->u.Soft.Protection,
611 PageFrameIndex);
612 }
613
614 /* Write the dirty bit for writeable pages */
615 if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
616
617 /* And now write down the PTE, making the address valid */
618 MI_WRITE_VALID_PTE(PointerPte, TempPte);
619
620 /* Release the working set */
621 MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
622 return STATUS_PAGE_FAULT_DEMAND_ZERO;
623 }
624
625 /* EOF */