projects / reactos.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Hopefully fail to break anything in the process of syncing with trunk (r47786)
[reactos.git] / ntoskrnl / mm / i386 / pagepae.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/i386/page.c
5 * PURPOSE: Low level memory managment manipulation
6 *
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 */
9
10 /* INCLUDES ***************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 #if defined (ALLOC_PRAGMA)
17 #pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
18 #pragma alloc_text(INIT, MiInitPageDirectoryMap)
19 #endif
20
21
22 /* GLOBALS *****************************************************************/
23
24 #define PA_BIT_PRESENT (0)
25 #define PA_BIT_READWRITE (1)
26 #define PA_BIT_USER (2)
27 #define PA_BIT_WT (3)
28 #define PA_BIT_CD (4)
29 #define PA_BIT_ACCESSED (5)
30 #define PA_BIT_DIRTY (6)
31 #define PA_BIT_GLOBAL (8)
32
33 #define PA_PRESENT (1 << PA_BIT_PRESENT)
34 #define PA_READWRITE (1 << PA_BIT_READWRITE)
35 #define PA_USER (1 << PA_BIT_USER)
36 #define PA_DIRTY (1 << PA_BIT_DIRTY)
37 #define PA_WT (1 << PA_BIT_WT)
38 #define PA_CD (1 << PA_BIT_CD)
39 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
40 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
41
42 #define PAGETABLE_MAP (0xc0000000)
43 #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024)))
44
45 #define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512)))
46
47 #define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000)
48 #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
49
50 ULONG MmGlobalKernelPageDirectory[1024];
51 ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];
52
53 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
54 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
55
56 #define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
57 #define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
58
59 #if defined(__GNUC__)
60 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
61 #else
62 __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage)
63 {
64 LARGE_INTEGER dummy;
65 dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage));
66 return dummy;
67 }
68 #endif
69
70 extern BOOLEAN Ke386Pae;
71 extern BOOLEAN Ke386NoExecute;
72
73 /* FUNCTIONS ***************************************************************/
74
75 BOOLEAN MmUnmapPageTable(PULONG Pt);
76
77 ULONG_PTR
78 NTAPI
79 MiFlushTlbIpiRoutine(ULONG_PTR Address)
80 {
81 if (Address == (ULONGLONG)-1)
82 {
83 KeFlushCurrentTb();
84 }
85 else if (Address == (ULONGLONG)-2)
86 {
87 KeFlushCurrentTb();
88 }
89 else
90 {
91 __invlpg((PVOID)Address);
92 }
93 return 0;
94 }
95
96 VOID
97 MiFlushTlb(PULONG Pt, PVOID Address)
98 {
99 #ifdef CONFIG_SMP
100 if (Pt)
101 {
102 MmUnmapPageTable(Pt);
103 }
104 if (KeNumberProcessors > 1)
105 {
106 KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address);
107 }
108 else
109 {
110 MiFlushTlbIpiRoutine((ULONG_PTR)Address);
111 }
112 #else
113 if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart)
114 {
115 __invlpg(Address);
116 }
117 #endif
118 }
119
120
121
122 PULONG
123 MmGetPageDirectory(VOID)
124 {
125 return (PULONG)__readcr3();
126 }
127
128 static ULONG
129 ProtectToPTE(ULONG flProtect)
130 {
131 ULONG Attributes = 0;
132
133 if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
134 {
135 Attributes = 0;
136 }
137 else if (flProtect & PAGE_IS_WRITABLE)
138 {
139 Attributes = PA_PRESENT | PA_READWRITE;
140 }
141 else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
142 {
143 Attributes = PA_PRESENT;
144 }
145 else
146 {
147 DPRINT1("Unknown main protection type.\n");
148 ASSERT(FALSE);
149 }
150 if (Ke386NoExecute &&
151 !(flProtect & PAGE_IS_EXECUTABLE))
152 {
153 Attributes = Attributes | 0x80000000;
154 }
155
156 if (flProtect & PAGE_SYSTEM)
157 {
158 }
159 else
160 {
161 Attributes = Attributes | PA_USER;
162 }
163 if (flProtect & PAGE_NOCACHE)
164 {
165 Attributes = Attributes | PA_CD;
166 }
167 if (flProtect & PAGE_WRITETHROUGH)
168 {
169 Attributes = Attributes | PA_WT;
170 }
171 return(Attributes);
172 }
173
174 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
175
176 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
177 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
178 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
179
180 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
181
182 #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
183
184
185 #define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE))
186
187 #define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
188 ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
189 #define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
190
191
192 #define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
193
194 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
195
196 #define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
197
198 #define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
199
200
201 NTSTATUS
202 NTAPI
203 Mmi386ReleaseMmInfo(PEPROCESS Process)
204 {
205 PUSHORT LdtDescriptor;
206 ULONG LdtBase;
207 ULONG i, j;
208
209 DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process);
210
211 LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor;
212 LdtBase = LdtDescriptor[1] |
213 ((LdtDescriptor[2] & 0xff) << 16) |
214 ((LdtDescriptor[3] & ~0xff) << 16);
215
216 DPRINT("LdtBase: %x\n", LdtBase);
217
218 if (LdtBase)
219 {
220 ExFreePool((PVOID) LdtBase);
221 }
222
223 if (Ke386Pae)
224 {
225 PULONGLONG PageDirTable;
226 PULONGLONG PageDir;
227 PULONGLONG Pde;
228 ULONG k;
229
230 PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
231 for (i = 0; i < 4; i++)
232 {
233 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i]));
234 if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart))
235 {
236 for (j = 0; j < 512; j++)
237 {
238 if (PageDir[j] != 0LL)
239 {
240 DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
241 Process->UniqueProcessId,
242 (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1,
243 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i*512 + j]);
244 Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j]));
245 for (k = 0; k < 512; k++)
246 {
247 if(Pde[k] != 0)
248 {
249 if (Pde[k] & PA_PRESENT)
250 {
251 DPRINT1("Page at %08x is not freed\n",
252 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
253 }
254 else
255 {
256 DPRINT1("Swapentry %x at %x is not freed\n",
257 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
258 }
259 }
260 }
261 MmDeleteHyperspaceMapping(Pde);
262 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j]));
263 }
264 }
265 }
266 if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE))
267 {
268 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)]));
269 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1]));
270 }
271 MmDeleteHyperspaceMapping(PageDir);
272 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i]));
273 }
274 MmDeleteHyperspaceMapping((PVOID)PageDirTable);
275 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
276 }
277 else
278 {
279 PULONG Pde;
280 PULONG PageDir;
281 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
282 for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++)
283 {
284 if (PageDir[i] != 0)
285 {
286 DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
287 i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1,
288 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]);
289 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i]));
290 for (j = 0; j < 1024; j++)
291 {
292 if(Pde[j] != 0)
293 {
294 if (Pde[j] & PA_PRESENT)
295 {
296 DPRINT1("Page at %08x is not freed\n",
297 i * 4 * 1024 * 1024 + j * PAGE_SIZE);
298 }
299 else
300 {
301 DPRINT1("Swapentry %x at %x is not freed\n",
302 Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE);
303 }
304 }
305 }
306 MmDeleteHyperspaceMapping(Pde);
307 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i]));
308 }
309 }
310 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)]));
311 MmDeleteHyperspaceMapping(PageDir);
312 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
313 }
314
315 #if defined(__GNUC__)
316
317 Process->Pcb.DirectoryTableBase.QuadPart = 0LL;
318 #else
319
320 Process->Pcb.DirectoryTableBase.QuadPart = 0;
321 #endif
322
323 DPRINT("Finished Mmi386ReleaseMmInfo()\n");
324 return(STATUS_SUCCESS);
325 }
326
327 NTSTATUS
328 NTAPI
329 MmInitializeHandBuiltProcess(IN PEPROCESS Process,
330 IN PLARGE_INTEGER DirectoryTableBase)
331 {
332 /* Share the directory base with the idle process */
333 *DirectoryTableBase = PsGetCurrentProcess()->Pcb.DirectoryTableBase;
334
335 /* Initialize the Addresss Space */
336 MmInitializeAddressSpace(Process, (PMADDRESS_SPACE)&Process->VadRoot);
337
338 /* The process now has an address space */
339 Process->HasAddressSpace = TRUE;
340 return STATUS_SUCCESS;
341 }
342
343 BOOLEAN
344 NTAPI
345 MmCreateProcessAddressSpace(IN ULONG MinWs,
346 IN PEPROCESS Process,
347 IN PLARGE_INTEGER DirectoryTableBase)
348 {
349 NTSTATUS Status;
350 ULONG i, j;
351 PFN_TYPE Pfn[7];
352 ULONG Count;
353
354 DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process);
355
356 Count = Ke386Pae ? 7 : 2;
357
358 for (i = 0; i < Count; i++)
359 {
360 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
361 if (!NT_SUCCESS(Status))
362 {
363 for (j = 0; j < i; j++)
364 {
365 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
366 }
367
368 return FALSE;
369 }
370 }
371
372 if (Ke386Pae)
373 {
374 PULONGLONG PageDirTable;
375 PULONGLONG PageDir;
376
377 PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
378 for (i = 0; i < 4; i++)
379 {
380 PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
381 }
382 MmDeleteHyperspaceMapping(PageDirTable);
383 for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++)
384 {
385 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
386 memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
387 if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
388 {
389 for (j = 0; j < 4; j++)
390 {
391 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
392 }
393 }
394 if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
395 {
396 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
397 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
398 }
399 MmDeleteHyperspaceMapping(PageDir);
400 }
401 }
402 else
403 {
404 PULONG PageDirectory;
405 PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);
406
407 memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
408 MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
409 (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG));
410
411 DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
412 PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
413 PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;
414
415 MmDeleteHyperspaceMapping(PageDirectory);
416 }
417
418 DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]);
419 DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart);
420 return TRUE;
421 }
422
423 VOID
424 NTAPI
425 MmDeletePageTable(PEPROCESS Process, PVOID Address)
426 {
427 PEPROCESS CurrentProcess = PsGetCurrentProcess();
428
429 if (Process != NULL && Process != CurrentProcess)
430 {
431 KeAttachProcess(&Process->Pcb);
432 }
433
434 if (Ke386Pae)
435 {
436 ULONGLONG ZeroPde = 0LL;
437 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde);
438 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
439 }
440 else
441 {
442 *(ADDR_TO_PDE(Address)) = 0;
443 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
444 }
445 if (Address >= MmSystemRangeStart)
446 {
447 ASSERT(FALSE);
448 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
449 }
450 if (Process != NULL && Process != CurrentProcess)
451 {
452 KeDetachProcess();
453 }
454 }
455
456 VOID
457 NTAPI
458 MmFreePageTable(PEPROCESS Process, PVOID Address)
459 {
460 PEPROCESS CurrentProcess = PsGetCurrentProcess();
461 ULONG i;
462 PFN_TYPE Pfn;
463
464 DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
465 if (Process != NULL && Process != CurrentProcess)
466 {
467 KeAttachProcess(&Process->Pcb);
468 }
469 if (Ke386Pae)
470 {
471 PULONGLONG PageTable;
472 ULONGLONG ZeroPte = 0LL;
473 PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
474 for (i = 0; i < 512; i++)
475 {
476 if (PageTable[i] != 0LL)
477 {
478 DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
479 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
480 ASSERT(FALSE);
481 }
482 }
483 Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
484 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
485 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
486 }
487 else
488 {
489 PULONG PageTable;
490 PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
491 for (i = 0; i < 1024; i++)
492 {
493 if (PageTable[i] != 0)
494 {
495 DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
496 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
497 ASSERT(FALSE);
498 }
499 }
500 Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
501 *(ADDR_TO_PDE(Address)) = 0;
502 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
503 }
504
505 if (Address >= MmSystemRangeStart)
506 {
507 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
508 ASSERT(FALSE);
509 }
510 else
511 {
512 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
513 }
514 if (Process != NULL && Process != CurrentProcess)
515 {
516 KeDetachProcess();
517 }
518 }
519
520 static PULONGLONG
521 MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
522 {
523 NTSTATUS Status;
524 PFN_TYPE Pfn;
525 ULONGLONG Entry;
526 ULONGLONG ZeroEntry = 0LL;
527 PULONGLONG Pt;
528 PULONGLONG PageDir;
529 PULONGLONG PageDirTable;
530
531 DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
532 Process, Address, Create);
533 if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000))
534 {
535 ASSERT(FALSE);
536 }
537 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
538 {
539 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
540 if (PageDirTable == NULL)
541 {
542 ASSERT(FALSE);
543 }
544 PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
545 MmDeleteHyperspaceMapping(PageDirTable);
546 if (PageDir == NULL)
547 {
548 ASSERT(FALSE);
549 }
550 PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
551 Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
552 if (Entry == 0LL)
553 {
554 if (Create == FALSE)
555 {
556 MmDeleteHyperspaceMapping(PageDir);
557 return NULL;
558 }
559 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
560 if (!NT_SUCCESS(Status))
561 {
562 ASSERT(FALSE);
563 }
564 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
565 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
566 if (Entry != 0LL)
567 {
568 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
569 Pfn = PAE_PTE_TO_PFN(Entry);
570 }
571 }
572 else
573 {
574 Pfn = PAE_PTE_TO_PFN(Entry);
575 }
576 MmDeleteHyperspaceMapping(PageDir);
577 Pt = MmCreateHyperspaceMapping(Pfn);
578 if (Pt == NULL)
579 {
580 ASSERT(FALSE);
581 }
582 return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
583 }
584 PageDir = PAE_ADDR_TO_PDE(Address);
585 if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
586 {
587 if (Address >= MmSystemRangeStart)
588 {
589 if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
590 {
591 if (Create == FALSE)
592 {
593 return NULL;
594 }
595 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
596 if (!NT_SUCCESS(Status))
597 {
598 ASSERT(FALSE);
599 }
600 Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
601 if (Ke386GlobalPagesEnabled)
602 {
603 Entry |= PA_GLOBAL;
604 }
605 if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
606 {
607 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
608 }
609 }
610 (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
611 }
612 else
613 {
614 if (Create == FALSE)
615 {
616 return NULL;
617 }
618 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
619 if (!NT_SUCCESS(Status))
620 {
621 ASSERT(FALSE);
622 }
623 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
624 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
625 if (Entry != 0LL)
626 {
627 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
628 }
629 }
630 }
631 return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
632 }
633
634 static PULONG
635 MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
636 {
637 ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
638 NTSTATUS Status;
639 PFN_TYPE Pfn;
640 ULONG Entry;
641 PULONG Pt, PageDir;
642
643 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
644 {
645 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.LowPart));
646 if (PageDir == NULL)
647 {
648 ASSERT(FALSE);
649 }
650 if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
651 {
652 if (Create == FALSE)
653 {
654 MmDeleteHyperspaceMapping(PageDir);
655 return NULL;
656 }
657 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
658 if (!NT_SUCCESS(Status) || Pfn == 0)
659 {
660 ASSERT(FALSE);
661 }
662 Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
663 if (Entry != 0)
664 {
665 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
666 Pfn = PTE_TO_PFN(Entry);
667 }
668 }
669 else
670 {
671 Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
672 }
673 MmDeleteHyperspaceMapping(PageDir);
674 Pt = MmCreateHyperspaceMapping(Pfn);
675 if (Pt == NULL)
676 {
677 ASSERT(FALSE);
678 }
679 return Pt + ADDR_TO_PTE_OFFSET(Address);
680 }
681 PageDir = ADDR_TO_PDE(Address);
682 if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
683 {
684 if (Address >= MmSystemRangeStart)
685 {
686 if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
687 {
688 if (Create == FALSE)
689 {
690 return NULL;
691 }
692 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
693 if (!NT_SUCCESS(Status) || Pfn == 0)
694 {
695 ASSERT(FALSE);
696 }
697 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
698 if (Ke386GlobalPagesEnabled)
699 {
700 Entry |= PA_GLOBAL;
701 }
702 if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
703 {
704 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
705 }
706 }
707 (void)InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
708 }
709 else
710 {
711 if (Create == FALSE)
712 {
713 return NULL;
714 }
715 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
716 if (!NT_SUCCESS(Status) || Pfn == 0)
717 {
718 ASSERT(FALSE);
719 }
720 Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
721 if (Entry != 0)
722 {
723 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
724 }
725 }
726 }
727 return (PULONG)ADDR_TO_PTE(Address);
728 }
729
730 BOOLEAN MmUnmapPageTable(PULONG Pt)
731 {
732 if (Ke386Pae)
733 {
734 if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
735 {
736 return TRUE;
737 }
738 }
739 else
740 {
741 if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
742 {
743 return TRUE;
744 }
745 }
746 if (Pt)
747 {
748 MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
749 }
750 return FALSE;
751 }
752
753 static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
754 {
755 ULONGLONG Pte;
756 PULONGLONG Pt;
757
758 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
759 if (Pt)
760 {
761 Pte = *Pt;
762 MmUnmapPageTable((PULONG)Pt);
763 return Pte;
764 }
765 return 0;
766 }
767
768 static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
769 {
770 ULONG Pte;
771 PULONG Pt;
772
773 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
774 if (Pt)
775 {
776 Pte = *Pt;
777 MmUnmapPageTable(Pt);
778 return Pte;
779 }
780 return 0;
781 }
782
783 PFN_TYPE
784 NTAPI
785 MmGetPfnForProcess(PEPROCESS Process,
786 PVOID Address)
787 {
788
789 if (Ke386Pae)
790 {
791 ULONGLONG Entry;
792 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
793 if (!(Entry & PA_PRESENT))
794 {
795 return 0;
796 }
797 return(PAE_PTE_TO_PFN(Entry));
798 }
799 else
800 {
801 ULONG Entry;
802 Entry = MmGetPageEntryForProcess(Process, Address);
803 if (!(Entry & PA_PRESENT))
804 {
805 return 0;
806 }
807 return(PTE_TO_PFN(Entry));
808 }
809 }
810
811 VOID
812 NTAPI
813 MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPFN_TYPE Page)
814 /*
815 * FUNCTION: Delete a virtual mapping
816 */
817 {
818 BOOLEAN WasValid;
819 if (Ke386Pae)
820 {
821 ULONGLONG Pte;
822 ULONGLONG tmpPte;
823 PULONGLONG Pt;
824
825 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
826 if (Pt == NULL)
827 {
828 ASSERT(FALSE);
829 }
830 /*
831 * Atomically disable the present bit and get the old value.
832 */
833 do
834 {
835 Pte = *Pt;
836 tmpPte = Pte & ~PA_PRESENT;
837 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
838
839 MiFlushTlb((PULONG)Pt, Address);
840 WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE;
841 if (!WasValid)
842 {
843 ASSERT(FALSE);
844 }
845
846 /*
847 * Return some information to the caller
848 */
849 if (WasDirty != NULL)
850 {
851 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
852 }
853 if (Page != NULL)
854 {
855 *Page = PAE_PTE_TO_PFN(Pte);
856 }
857 }
858 else
859 {
860 ULONG Pte;
861 PULONG Pt;
862
863 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
864 if (Pt == NULL)
865 {
866 ASSERT(FALSE);
867 }
868 /*
869 * Atomically disable the present bit and get the old value.
870 */
871 do
872 {
873 Pte = *Pt;
874 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte));
875
876 MiFlushTlb(Pt, Address);
877 WasValid = (PAGE_MASK(Pte) != 0);
878 if (!WasValid)
879 {
880 ASSERT(FALSE);
881 }
882
883 /*
884 * Return some information to the caller
885 */
886 if (WasDirty != NULL)
887 {
888 *WasDirty = Pte & PA_DIRTY;
889 }
890 if (Page != NULL)
891 {
892 *Page = PTE_TO_PFN(Pte);
893 }
894 }
895 }
896
897 VOID
898 NTAPI
899 MmRawDeleteVirtualMapping(PVOID Address)
900 {
901 if (Ke386Pae)
902 {
903 PULONGLONG Pt;
904 ULONGLONG ZeroPte = 0LL;
905 Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE);
906 if (Pt)
907 {
908 /*
909 * Set the entry to zero
910 */
911 (void)ExfpInterlockedExchange64UL(Pt, &ZeroPte);
912 MiFlushTlb((PULONG)Pt, Address);
913 }
914 }
915 else
916 {
917 PULONG Pt;
918
919 Pt = MmGetPageTableForProcess(NULL, Address, FALSE);
920 if (Pt && *Pt)
921 {
922 /*
923 * Set the entry to zero
924 */
925 (void)InterlockedExchangeUL(Pt, 0);
926 MiFlushTlb(Pt, Address);
927 }
928 }
929 }
930
931 VOID
932 NTAPI
933 MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage,
934 BOOLEAN* WasDirty, PPFN_TYPE Page)
935 /*
936 * FUNCTION: Delete a virtual mapping
937 */
938 {
939 BOOLEAN WasValid = FALSE;
940 PFN_TYPE Pfn;
941
942 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
943 Process, Address, FreePage, WasDirty, Page);
944 if (Ke386Pae)
945 {
946 ULONGLONG Pte;
947 PULONGLONG Pt;
948
949 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
950 if (Pt == NULL)
951 {
952 if (WasDirty != NULL)
953 {
954 *WasDirty = FALSE;
955 }
956 if (Page != NULL)
957 {
958 *Page = 0;
959 }
960 return;
961 }
962
963 /*
964 * Atomically set the entry to zero and get the old value.
965 */
966 Pte = 0LL;
967 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
968
969 MiFlushTlb((PULONG)Pt, Address);
970
971 WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
972 if (WasValid)
973 {
974 Pfn = PAE_PTE_TO_PFN(Pte);
975 MmMarkPageUnmapped(Pfn);
976 }
977 else
978 {
979 Pfn = 0;
980 }
981
982 if (FreePage && WasValid)
983 {
984 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
985 }
986
987 /*
988 * Return some information to the caller
989 */
990 if (WasDirty != NULL)
991 {
992 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
993 }
994 if (Page != NULL)
995 {
996 *Page = Pfn;
997 }
998 }
999 else
1000 {
1001 ULONG Pte;
1002 PULONG Pt;
1003
1004 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1005
1006 if (Pt == NULL)
1007 {
1008 if (WasDirty != NULL)
1009 {
1010 *WasDirty = FALSE;
1011 }
1012 if (Page != NULL)
1013 {
1014 *Page = 0;
1015 }
1016 return;
1017 }
1018
1019 /*
1020 * Atomically set the entry to zero and get the old value.
1021 */
1022 Pte = InterlockedExchangeUL(Pt, 0);
1023
1024 MiFlushTlb(Pt, Address);
1025
1026 WasValid = (PAGE_MASK(Pte) != 0);
1027 if (WasValid)
1028 {
1029 Pfn = PTE_TO_PFN(Pte);
1030 MmMarkPageUnmapped(Pfn);
1031 }
1032 else
1033 {
1034 Pfn = 0;
1035 }
1036
1037 if (FreePage && WasValid)
1038 {
1039 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
1040 }
1041
1042 /*
1043 * Return some information to the caller
1044 */
1045 if (WasDirty != NULL)
1046 {
1047 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
1048 }
1049 if (Page != NULL)
1050 {
1051 *Page = Pfn;
1052 }
1053 }
1054 /*
1055 * Decrement the reference count for this page table.
1056 */
1057 if (Process != NULL && WasValid &&
1058 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1059 Address < MmSystemRangeStart)
1060 {
1061 PUSHORT Ptrc;
1062 ULONG Idx;
1063
1064 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1065 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1066
1067 Ptrc[Idx]--;
1068 if (Ptrc[Idx] == 0)
1069 {
1070 MmFreePageTable(Process, Address);
1071 }
1072 }
1073 }
1074
1075 VOID
1076 NTAPI
1077 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
1078 SWAPENTRY* SwapEntry)
1079 /*
1080 * FUNCTION: Delete a virtual mapping
1081 */
1082 {
1083 if (Ke386Pae)
1084 {
1085 ULONGLONG Pte;
1086 PULONGLONG Pt;
1087
1088 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1089 if (Pt == NULL)
1090 {
1091 *SwapEntry = 0;
1092 return;
1093 }
1094
1095 /*
1096 * Atomically set the entry to zero and get the old value.
1097 */
1098 Pte = 0LL;
1099 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1100
1101 MiFlushTlb((PULONG)Pt, Address);
1102
1103 /*
1104 * Decrement the reference count for this page table.
1105 */
1106 if (Process != NULL && Pte &&
1107 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1108 Address < MmSystemRangeStart)
1109 {
1110 PUSHORT Ptrc;
1111
1112 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1113
1114 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
1115 if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
1116 {
1117 MmFreePageTable(Process, Address);
1118 }
1119 }
1120
1121
1122 /*
1123 * Return some information to the caller
1124 */
1125 *SwapEntry = Pte >> 1;
1126 }
1127 else
1128 {
1129 ULONG Pte;
1130 PULONG Pt;
1131
1132 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1133
1134 if (Pt == NULL)
1135 {
1136 *SwapEntry = 0;
1137 return;
1138 }
1139
1140 /*
1141 * Atomically set the entry to zero and get the old value.
1142 */
1143 Pte = InterlockedExchangeUL(Pt, 0);
1144
1145 MiFlushTlb(Pt, Address);
1146
1147 /*
1148 * Decrement the reference count for this page table.
1149 */
1150 if (Process != NULL && Pte &&
1151 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1152 Address < MmSystemRangeStart)
1153 {
1154 PUSHORT Ptrc;
1155
1156 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1157
1158 Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
1159 if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
1160 {
1161 MmFreePageTable(Process, Address);
1162 }
1163 }
1164
1165
1166 /*
1167 * Return some information to the caller
1168 */
1169 *SwapEntry = Pte >> 1;
1170 }
1171 }
1172
1173 BOOLEAN
1174 Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
1175 {
1176 if (Ke386Pae)
1177 {
1178 PULONGLONG Pt;
1179 PULONGLONG Pde;
1180 Pde = PAE_ADDR_TO_PDE(PAddress);
1181 if (*Pde == 0LL)
1182 {
1183 Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
1184 #if 0
1185 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1186 FLASH_TLB_ONE(PAddress);
1187 #endif
1188 if (Pt != NULL)
1189 {
1190 return TRUE;
1191 }
1192 }
1193 }
1194 else
1195 {
1196 PULONG Pt, Pde;
1197 Pde = ADDR_TO_PDE(PAddress);
1198 if (*Pde == 0)
1199 {
1200 Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
1201 #if 0
1202 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1203 FLASH_TLB_ONE(PAddress);
1204 #endif
1205 if (Pt != NULL)
1206 {
1207 return TRUE;
1208 }
1209 }
1210 }
1211 return(FALSE);
1212 }
1213
1214 BOOLEAN
1215 NTAPI
1216 MmIsDirtyPage(PEPROCESS Process, PVOID Address)
1217 {
1218 if (Ke386Pae)
1219 {
1220 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1221 }
1222 else
1223 {
1224 return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1225 }
1226 }
1227
1228 BOOLEAN
1229 NTAPI
1230 MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address)
1231 {
1232 if (Address < MmSystemRangeStart && Process == NULL)
1233 {
1234 DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1235 ASSERT(FALSE);
1236 }
1237 if (Ke386Pae)
1238 {
1239 PULONGLONG Pt;
1240 ULONGLONG Pte;
1241 ULONGLONG tmpPte;
1242
1243 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1244 if (Pt == NULL)
1245 {
1246 ASSERT(FALSE);
1247 }
1248
1249 do
1250 {
1251 Pte = *Pt;
1252 tmpPte = Pte & ~PA_ACCESSED;
1253 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1254
1255 if (Pte & PA_ACCESSED)
1256 {
1257 MiFlushTlb((PULONG)Pt, Address);
1258 return TRUE;
1259 }
1260 else
1261 {
1262 MmUnmapPageTable((PULONG)Pt);
1263 return FALSE;
1264 }
1265 }
1266 else
1267 {
1268 PULONG Pt;
1269 ULONG Pte;
1270
1271 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1272 if (Pt == NULL)
1273 {
1274 ASSERT(FALSE);
1275 }
1276
1277 do
1278 {
1279 Pte = *Pt;
1280 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte));
1281
1282 if (Pte & PA_ACCESSED)
1283 {
1284 MiFlushTlb(Pt, Address);
1285 return TRUE;
1286 }
1287 else
1288 {
1289 MmUnmapPageTable(Pt);
1290 return FALSE;
1291 }
1292 }
1293 }
1294
1295 VOID
1296 NTAPI
1297 MmSetCleanPage(PEPROCESS Process, PVOID Address)
1298 {
1299 if (Address < MmSystemRangeStart && Process == NULL)
1300 {
1301 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1302 ASSERT(FALSE);
1303 }
1304 if (Ke386Pae)
1305 {
1306 PULONGLONG Pt;
1307 ULONGLONG Pte;
1308 ULONGLONG tmpPte;
1309
1310 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1311
1312 if (Pt == NULL)
1313 {
1314 ASSERT(FALSE);
1315 }
1316
1317 do
1318 {
1319 Pte = *Pt;
1320 tmpPte = Pte & ~PA_DIRTY;
1321 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1322
1323 if (Pte & PA_DIRTY)
1324 {
1325 MiFlushTlb((PULONG)Pt, Address);
1326 }
1327 else
1328 {
1329 MmUnmapPageTable((PULONG)Pt);
1330 }
1331 }
1332 else
1333 {
1334 PULONG Pt;
1335 ULONG Pte;
1336
1337 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1338
1339 if (Pt == NULL)
1340 {
1341 ASSERT(FALSE);
1342 }
1343
1344 do
1345 {
1346 Pte = *Pt;
1347 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));
1348
1349 if (Pte & PA_DIRTY)
1350 {
1351 MiFlushTlb(Pt, Address);
1352 }
1353 else
1354 {
1355 MmUnmapPageTable(Pt);
1356 }
1357 }
1358 }
1359
1360 VOID
1361 NTAPI
1362 MmSetDirtyPage(PEPROCESS Process, PVOID Address)
1363 {
1364 if (Address < MmSystemRangeStart && Process == NULL)
1365 {
1366 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1367 ASSERT(FALSE);
1368 }
1369 if (Ke386Pae)
1370 {
1371 PULONGLONG Pt;
1372 ULONGLONG Pte;
1373 ULONGLONG tmpPte;
1374
1375 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1376 if (Pt == NULL)
1377 {
1378 ASSERT(FALSE);
1379 }
1380
1381 do
1382 {
1383 Pte = *Pt;
1384 tmpPte = Pte | PA_DIRTY;
1385 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1386 if (!(Pte & PA_DIRTY))
1387 {
1388 MiFlushTlb((PULONG)Pt, Address);
1389 }
1390 else
1391 {
1392 MmUnmapPageTable((PULONG)Pt);
1393 }
1394 }
1395 else
1396 {
1397 PULONG Pt;
1398 ULONG Pte;
1399
1400 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1401 if (Pt == NULL)
1402 {
1403 ASSERT(FALSE);
1404 }
1405
1406 do
1407 {
1408 Pte = *Pt;
1409 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
1410 if (!(Pte & PA_DIRTY))
1411 {
1412 MiFlushTlb(Pt, Address);
1413 }
1414 else
1415 {
1416 MmUnmapPageTable(Pt);
1417 }
1418 }
1419 }
1420
1421 VOID
1422 NTAPI
1423 MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
1424 {
1425 if (Ke386Pae)
1426 {
1427 PULONGLONG Pt;
1428 ULONGLONG Pte;
1429 ULONGLONG tmpPte;
1430
1431 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1432 if (Pt == NULL)
1433 {
1434 ASSERT(FALSE);
1435 }
1436
1437 do
1438 {
1439 Pte = *Pt;
1440 tmpPte = Pte | PA_PRESENT;
1441 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1442 if (!(Pte & PA_PRESENT))
1443 {
1444 MiFlushTlb((PULONG)Pt, Address);
1445 }
1446 else
1447 {
1448 MmUnmapPageTable((PULONG)Pt);
1449 }
1450 }
1451 else
1452 {
1453 PULONG Pt;
1454 ULONG Pte;
1455
1456 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1457 if (Pt == NULL)
1458 {
1459 ASSERT(FALSE);
1460 }
1461
1462 do
1463 {
1464 Pte = *Pt;
1465 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte));
1466 if (!(Pte & PA_PRESENT))
1467 {
1468 MiFlushTlb(Pt, Address);
1469 }
1470 else
1471 {
1472 MmUnmapPageTable(Pt);
1473 }
1474 }
1475 }
1476
1477 BOOLEAN
1478 NTAPI
1479 MmIsPagePresent(PEPROCESS Process, PVOID Address)
1480 {
1481 if (Ke386Pae)
1482 {
1483 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1484 }
1485 else
1486 {
1487 return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1488 }
1489 }
1490
1491 BOOLEAN
1492 NTAPI
1493 MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
1494 {
1495 if (Ke386Pae)
1496 {
1497 ULONGLONG Entry;
1498 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1499 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1500 }
1501 else
1502 {
1503 ULONG Entry;
1504 Entry = MmGetPageEntryForProcess(Process, Address);
1505 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1506 }
1507 }
1508
1509 NTSTATUS
1510 NTAPI
1511 MmCreateVirtualMappingForKernel(PVOID Address,
1512 ULONG flProtect,
1513 PPFN_TYPE Pages,
1514 ULONG PageCount)
1515 {
1516 ULONG Attributes;
1517 ULONG i;
1518 PVOID Addr;
1519 ULONG PdeOffset, oldPdeOffset;
1520 BOOLEAN NoExecute = FALSE;
1521
1522 DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1523 Address, flProtect, Pages, PageCount);
1524
1525 if (Address < MmSystemRangeStart)
1526 {
1527 DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1528 ASSERT(FALSE);
1529 }
1530
1531 Attributes = ProtectToPTE(flProtect);
1532 if (Attributes & 0x80000000)
1533 {
1534 NoExecute = TRUE;
1535 }
1536 Attributes &= 0xfff;
1537 if (Ke386GlobalPagesEnabled)
1538 {
1539 Attributes |= PA_GLOBAL;
1540 }
1541
1542 Addr = Address;
1543
1544 if (Ke386Pae)
1545 {
1546 PULONGLONG Pt = NULL;
1547 ULONGLONG Pte;
1548
1549 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1550 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1551 {
1552 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1553 {
1554 DPRINT1("Setting physical address but not allowing access at address "
1555 "0x%.8X with attributes %x/%x.\n",
1556 Addr, Attributes, flProtect);
1557 ASSERT(FALSE);
1558 }
1559
1560 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1561 if (oldPdeOffset != PdeOffset)
1562 {
1563 Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE);
1564 if (Pt == NULL)
1565 {
1566 ASSERT(FALSE);
1567 }
1568 }
1569 else
1570 {
1571 Pt++;
1572 }
1573 oldPdeOffset = PdeOffset;
1574
1575 Pte = PFN_TO_PTE(Pages[i]) | Attributes;
1576 if (NoExecute)
1577 {
1578 Pte |= 0x8000000000000000LL;
1579 }
1580 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1581 if (Pte != 0LL)
1582 {
1583 ASSERT(FALSE);
1584 }
1585 }
1586 }
1587 else
1588 {
1589 PULONG Pt;
1590 ULONG Pte;
1591
1592 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1593 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1594 if (Pt == NULL)
1595 {
1596 ASSERT(FALSE);
1597 }
1598 Pt--;
1599
1600 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1601 {
1602 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1603 {
1604 DPRINT1("Setting physical address but not allowing access at address "
1605 "0x%.8X with attributes %x/%x.\n",
1606 Addr, Attributes, flProtect);
1607 ASSERT(FALSE);
1608 }
1609
1610 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1611 if (oldPdeOffset != PdeOffset)
1612 {
1613 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1614 if (Pt == NULL)
1615 {
1616 ASSERT(FALSE);
1617 }
1618 }
1619 else
1620 {
1621 Pt++;
1622 }
1623 oldPdeOffset = PdeOffset;
1624
1625 Pte = *Pt;
1626 if (Pte != 0)
1627 {
1628 ASSERT(FALSE);
1629 }
1630 (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1631 }
1632 }
1633
1634 return(STATUS_SUCCESS);
1635 }
1636
1637 NTSTATUS
1638 NTAPI
1639 MmCreatePageFileMapping(PEPROCESS Process,
1640 PVOID Address,
1641 SWAPENTRY SwapEntry)
1642 {
1643 if (Process == NULL && Address < MmSystemRangeStart)
1644 {
1645 DPRINT1("No process\n");
1646 ASSERT(FALSE);
1647 }
1648 if (Process != NULL && Address >= MmSystemRangeStart)
1649 {
1650 DPRINT1("Setting kernel address with process context\n");
1651 ASSERT(FALSE);
1652 }
1653 if (SwapEntry & (1 << 31))
1654 {
1655 ASSERT(FALSE);
1656 }
1657
1658 if (Ke386Pae)
1659 {
1660 PULONGLONG Pt;
1661 ULONGLONG Pte;
1662 ULONGLONG tmpPte;
1663
1664 Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
1665 if (Pt == NULL)
1666 {
1667 ASSERT(FALSE);
1668 }
1669 tmpPte = SwapEntry << 1;
1670 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1671 if (PAE_PAGE_MASK((Pte)) != 0)
1672 {
1673 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1674 }
1675
1676 if (Pte != 0)
1677 {
1678 MiFlushTlb((PULONG)Pt, Address);
1679 }
1680 else
1681 {
1682 MmUnmapPageTable((PULONG)Pt);
1683 }
1684 }
1685 else
1686 {
1687 PULONG Pt;
1688 ULONG Pte;
1689
1690 Pt = MmGetPageTableForProcess(Process, Address, TRUE);
1691 if (Pt == NULL)
1692 {
1693 ASSERT(FALSE);
1694 }
1695 Pte = *Pt;
1696 if (PAGE_MASK((Pte)) != 0)
1697 {
1698 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1699 }
1700 (void)InterlockedExchangeUL(Pt, SwapEntry << 1);
1701 if (Pte != 0)
1702 {
1703 MiFlushTlb(Pt, Address);
1704 }
1705 else
1706 {
1707 MmUnmapPageTable(Pt);
1708 }
1709 }
1710 if (Process != NULL &&
1711 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1712 Address < MmSystemRangeStart)
1713 {
1714 PUSHORT Ptrc;
1715 ULONG Idx;
1716
1717 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1718 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1719 Ptrc[Idx]++;
1720 }
1721 return(STATUS_SUCCESS);
1722 }
1723
1724
1725 NTSTATUS
1726 NTAPI
1727 MmCreateVirtualMappingUnsafe(PEPROCESS Process,
1728 PVOID Address,
1729 ULONG flProtect,
1730 PPFN_TYPE Pages,
1731 ULONG PageCount)
1732 {
1733 ULONG Attributes;
1734 PVOID Addr;
1735 ULONG i;
1736 ULONG oldPdeOffset, PdeOffset;
1737 BOOLEAN NoExecute = FALSE;
1738
1739 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1740 Process, Address, flProtect, Pages, *Pages, PageCount);
1741
1742 if (Process == NULL)
1743 {
1744 if (Address < MmSystemRangeStart)
1745 {
1746 DPRINT1("No process\n");
1747 ASSERT(FALSE);
1748 }
1749 if (PageCount > 0x10000 ||
1750 (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
1751 {
1752 DPRINT1("Page count to large\n");
1753 ASSERT(FALSE);
1754 }
1755 }
1756 else
1757 {
1758 if (Address >= MmSystemRangeStart)
1759 {
1760 DPRINT1("Setting kernel address with process context\n");
1761 ASSERT(FALSE);
1762 }
1763 if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE ||
1764 (ULONG_PTR) Address / PAGE_SIZE + PageCount >
1765 (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE)
1766 {
1767 DPRINT1("Page Count to large\n");
1768 ASSERT(FALSE);
1769 }
1770 }
1771
1772 Attributes = ProtectToPTE(flProtect);
1773 if (Attributes & 0x80000000)
1774 {
1775 NoExecute = TRUE;
1776 }
1777 Attributes &= 0xfff;
1778 if (Address >= MmSystemRangeStart)
1779 {
1780 Attributes &= ~PA_USER;
1781 if (Ke386GlobalPagesEnabled)
1782 {
1783 Attributes |= PA_GLOBAL;
1784 }
1785 }
1786 else
1787 {
1788 Attributes |= PA_USER;
1789 }
1790
1791 Addr = Address;
1792
1793 if (Ke386Pae)
1794 {
1795 ULONGLONG Pte, tmpPte;
1796 PULONGLONG Pt = NULL;
1797
1798 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1799 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1800 {
1801 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1802 {
1803 DPRINT1("Setting physical address but not allowing access at address "
1804 "0x%.8X with attributes %x/%x.\n",
1805 Addr, Attributes, flProtect);
1806 ASSERT(FALSE);
1807 }
1808 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1809 if (oldPdeOffset != PdeOffset)
1810 {
1811 MmUnmapPageTable((PULONG)Pt);
1812 Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
1813 if (Pt == NULL)
1814 {
1815 ASSERT(FALSE);
1816 }
1817 }
1818 else
1819 {
1820 Pt++;
1821 }
1822 oldPdeOffset = PdeOffset;
1823
1824 MmMarkPageMapped(Pages[i]);
1825 tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
1826 if (NoExecute)
1827 {
1828 tmpPte |= 0x8000000000000000LL;
1829 }
1830 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1831 if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
1832 {
1833 ASSERT(FALSE);
1834 }
1835 if (PAE_PAGE_MASK((Pte)) != 0LL)
1836 {
1837 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1838 }
1839 if (Address < MmSystemRangeStart &&
1840 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1841 Attributes & PA_PRESENT)
1842 {
1843 PUSHORT Ptrc;
1844
1845 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1846
1847 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
1848 }
1849 if (Pte != 0LL)
1850 {
1851 if (Address > MmSystemRangeStart ||
1852 (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
1853 {
1854 MiFlushTlb((PULONG)Pt, Address);
1855 }
1856 }
1857 }
1858 if (Addr > Address)
1859 {
1860 MmUnmapPageTable((PULONG)Pt);
1861 }
1862 }
1863 else
1864 {
1865 PULONG Pt = NULL;
1866 ULONG Pte;
1867 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
1868 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1869 {
1870 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1871 {
1872 DPRINT1("Setting physical address but not allowing access at address "
1873 "0x%.8X with attributes %x/%x.\n",
1874 Addr, Attributes, flProtect);
1875 ASSERT(FALSE);
1876 }
1877 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1878 if (oldPdeOffset != PdeOffset)
1879 {
1880 MmUnmapPageTable(Pt);
1881 Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
1882 if (Pt == NULL)
1883 {
1884 ASSERT(FALSE);
1885 }
1886 }
1887 else
1888 {
1889 Pt++;
1890 }
1891 oldPdeOffset = PdeOffset;
1892
1893 Pte = *Pt;
1894 MmMarkPageMapped(Pages[i]);
1895 if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
1896 {
1897 ASSERT(FALSE);
1898 }
1899 if (PAGE_MASK((Pte)) != 0)
1900 {
1901 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1902 }
1903 (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1904 if (Address < MmSystemRangeStart &&
1905 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1906 Attributes & PA_PRESENT)
1907 {
1908 PUSHORT Ptrc;
1909
1910 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1911
1912 Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
1913 }
1914 if (Pte != 0)
1915 {
1916 if (Address > MmSystemRangeStart ||
1917 (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
1918 {
1919 MiFlushTlb(Pt, Address);
1920 }
1921 }
1922 }
1923 if (Addr > Address)
1924 {
1925 MmUnmapPageTable(Pt);
1926 }
1927 }
1928 return(STATUS_SUCCESS);
1929 }
1930
1931 NTSTATUS
1932 NTAPI
1933 MmCreateVirtualMapping(PEPROCESS Process,
1934 PVOID Address,
1935 ULONG flProtect,
1936 PPFN_TYPE Pages,
1937 ULONG PageCount)
1938 {
1939 ULONG i;
1940
1941 for (i = 0; i < PageCount; i++)
1942 {
1943 if (!MmIsPageInUse(Pages[i]))
1944 {
1945 DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages[i]));
1946 ASSERT(FALSE);
1947 }
1948 }
1949
1950 return(MmCreateVirtualMappingUnsafe(Process,
1951 Address,
1952 flProtect,
1953 Pages,
1954 PageCount));
1955 }
1956
1957 ULONG
1958 NTAPI
1959 MmGetPageProtect(PEPROCESS Process, PVOID Address)
1960 {
1961 ULONG Entry;
1962 ULONG Protect;
1963 if (Ke386Pae)
1964 {
1965 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1966 }
1967 else
1968 {
1969 Entry = MmGetPageEntryForProcess(Process, Address);
1970 }
1971
1972 if (!(Entry & PA_PRESENT))
1973 {
1974 Protect = PAGE_NOACCESS;
1975 }
1976 else
1977 {
1978 if (Entry & PA_READWRITE)
1979 {
1980 Protect = PAGE_READWRITE;
1981 }
1982 else
1983 {
1984 Protect = PAGE_EXECUTE_READ;
1985 }
1986 if (Entry & PA_CD)
1987 {
1988 Protect |= PAGE_NOCACHE;
1989 }
1990 if (Entry & PA_WT)
1991 {
1992 Protect |= PAGE_WRITETHROUGH;
1993 }
1994 if (!(Entry & PA_USER))
1995 {
1996 Protect |= PAGE_SYSTEM;
1997 }
1998
1999 }
2000 return(Protect);
2001 }
2002
2003 VOID
2004 NTAPI
2005 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
2006 {
2007 ULONG Attributes = 0;
2008 BOOLEAN NoExecute = FALSE;
2009
2010 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
2011 Process, Address, flProtect);
2012
2013 Attributes = ProtectToPTE(flProtect);
2014 if (Attributes & 0x80000000)
2015 {
2016 NoExecute = TRUE;
2017 }
2018 Attributes &= 0xfff;
2019 if (Address >= MmSystemRangeStart)
2020 {
2021 Attributes &= ~PA_USER;
2022 if (Ke386GlobalPagesEnabled)
2023 {
2024 Attributes |= PA_GLOBAL;
2025 }
2026 }
2027 else
2028 {
2029 Attributes |= PA_USER;
2030 }
2031 if (Ke386Pae)
2032 {
2033 PULONGLONG Pt;
2034 ULONGLONG tmpPte, Pte;
2035
2036 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
2037 if (Pt == NULL)
2038 {
2039 DPRINT1("Address %x\n", Address);
2040 ASSERT(FALSE);
2041 }
2042 do
2043 {
2044 Pte = *Pt;
2045 tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
2046 if (NoExecute)
2047 {
2048 tmpPte |= 0x8000000000000000LL;
2049 }
2050 else
2051 {
2052 tmpPte &= ~0x8000000000000000LL;
2053 }
2054 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
2055
2056 MiFlushTlb((PULONG)Pt, Address);
2057 }
2058 else
2059 {
2060 PULONG Pt;
2061
2062 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
2063 if (Pt == NULL)
2064 {
2065 ASSERT(FALSE);
2066 }
2067 InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
2068 MiFlushTlb(Pt, Address);
2069 }
2070 }
2071
2072 /*
2073 * @implemented
2074 */
2075 PHYSICAL_ADDRESS NTAPI
2076 MmGetPhysicalAddress(PVOID vaddr)
2077 /*
2078 * FUNCTION: Returns the physical address corresponding to a virtual address
2079 */
2080 {
2081 PHYSICAL_ADDRESS p;
2082
2083 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr);
2084 if (Ke386Pae)
2085 {
2086 ULONGLONG Pte;
2087 Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr);
2088 if (Pte != 0 && Pte & PA_PRESENT)
2089 {
2090 p.QuadPart = PAE_PAGE_MASK(Pte);
2091 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2092 }
2093 else
2094 {
2095 p.QuadPart = 0;
2096 }
2097 }
2098 else
2099 {
2100 ULONG Pte;
2101 Pte = MmGetPageEntryForProcess(NULL, vaddr);
2102 if (Pte != 0 && Pte & PA_PRESENT)
2103 {
2104 p.QuadPart = PAGE_MASK(Pte);
2105 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2106 }
2107 else
2108 {
2109 p.QuadPart = 0;
2110 }
2111 }
2112 return p;
2113 }
2114
2115 PVOID
2116 NTAPI
2117 MmCreateHyperspaceMapping(PFN_TYPE Page)
2118 {
2119 PVOID Address;
2120 ULONG i;
2121
2122 if (Ke386Pae)
2123 {
2124 ULONGLONG Entry;
2125 ULONGLONG ZeroEntry = 0LL;
2126 PULONGLONG Pte;
2127
2128 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2129 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2130
2131 if (Page & 1024)
2132 {
2133 for (i = Page %1024; i < 1024; i++, Pte++)
2134 {
2135 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2136 {
2137 break;
2138 }
2139 }
2140 if (i >= 1024)
2141 {
2142 Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
2143 for (i = 0; i < Page % 1024; i++, Pte++)
2144 {
2145 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2146 {
2147 break;
2148 }
2149 }
2150 if (i >= Page % 1024)
2151 {
2152 ASSERT(FALSE);
2153 }
2154 }
2155 }
2156 else
2157 {
2158 for (i = Page %1024; (LONG)i >= 0; i--, Pte--)
2159 {
2160 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2161 {
2162 break;
2163 }
2164 }
2165 if ((LONG)i < 0)
2166 {
2167 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
2168 for (i = 1023; i > Page % 1024; i--, Pte--)
2169 {
2170 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2171 {
2172 break;
2173 }
2174 }
2175 if (i <= Page % 1024)
2176 {
2177 ASSERT(FALSE);
2178 }
2179 }
2180 }
2181 }
2182 else
2183 {
2184 ULONG Entry;
2185 PULONG Pte;
2186 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2187 Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2188 if (Page & 1024)
2189 {
2190 for (i = Page % 1024; i < 1024; i++, Pte++)
2191 {
2192 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2193 {
2194 break;
2195 }
2196 }
2197 if (i >= 1024)
2198 {
2199 Pte = ADDR_TO_PTE(HYPERSPACE);
2200 for (i = 0; i < Page % 1024; i++, Pte++)
2201 {
2202 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2203 {
2204 break;
2205 }
2206 }
2207 if (i >= Page % 1024)
2208 {
2209 ASSERT(FALSE);
2210 }
2211 }
2212 }
2213 else
2214 {
2215 for (i = Page % 1024; (LONG)i >= 0; i--, Pte--)
2216 {
2217 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2218 {
2219 break;
2220 }
2221 }
2222 if ((LONG)i < 0)
2223 {
2224 Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
2225 for (i = 1023; i > Page % 1024; i--, Pte--)
2226 {
2227 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2228 {
2229 break;
2230 }
2231 }
2232 if (i <= Page % 1024)
2233 {
2234 ASSERT(FALSE);
2235 }
2236 }
2237 }
2238 }
2239 Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE);
2240 __invlpg(Address);
2241 return Address;
2242 }
2243
2244 PFN_TYPE
2245 NTAPI
2246 MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage)
2247 {
2248 PFN_TYPE Pfn;
2249 ASSERT (IS_HYPERSPACE(Address));
2250 if (Ke386Pae)
2251 {
2252 ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE;
2253 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2254 Pfn = PAE_PTE_TO_PFN(Entry);
2255 }
2256 else
2257 {
2258 ULONG Entry;
2259 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE);
2260 Pfn = PTE_TO_PFN(Entry);
2261 }
2262 __invlpg(Address);
2263 return Pfn;
2264 }
2265
2266 PFN_TYPE
2267 NTAPI
2268 MmDeleteHyperspaceMapping(PVOID Address)
2269 {
2270 PFN_TYPE Pfn;
2271 ASSERT (IS_HYPERSPACE(Address));
2272 if (Ke386Pae)
2273 {
2274 ULONGLONG Entry = 0LL;
2275 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2276 Pfn = PAE_PTE_TO_PFN(Entry);
2277 }
2278 else
2279 {
2280 ULONG Entry;
2281 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0);
2282 Pfn = PTE_TO_PFN(Entry);
2283 }
2284 __invlpg(Address);
2285 return Pfn;
2286 }
2287
2288 VOID
2289 NTAPI
2290 MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
2291 {
2292 ULONG StartOffset, EndOffset, Offset;
2293
2294 if (Address < MmSystemRangeStart)
2295 {
2296 ASSERT(FALSE);
2297 }
2298 if (Ke386Pae)
2299 {
2300 PULONGLONG PageDirTable;
2301 PULONGLONG Pde;
2302 ULONGLONG ZeroPde = 0LL;
2303 ULONG i;
2304
2305 for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++)
2306 {
2307 if (i == PAE_ADDR_TO_PDTE_OFFSET(Address))
2308 {
2309 StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
2310 }
2311 else
2312 {
2313 StartOffset = 0;
2314 }
2315 if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)))
2316 {
2317 EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2318 }
2319 else
2320 {
2321 EndOffset = 511;
2322 }
2323
2324 if (Process != NULL && Process != PsGetCurrentProcess())
2325 {
2326 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
2327 Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i]));
2328 MmDeleteHyperspaceMapping(PageDirTable);
2329 }
2330 else
2331 {
2332 Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512;
2333 }
2334
2335 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2336 {
2337 if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4)
2338 {
2339 (void)ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde);
2340 }
2341 }
2342 MmUnmapPageTable((PULONG)Pde);
2343 }
2344 }
2345 else
2346 {
2347 PULONG Pde;
2348 StartOffset = ADDR_TO_PDE_OFFSET(Address);
2349 EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2350
2351 if (Process != NULL && Process != PsGetCurrentProcess())
2352 {
2353 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
2354 }
2355 else
2356 {
2357 Pde = (PULONG)PAGEDIRECTORY_MAP;
2358 }
2359 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2360 {
2361 if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP))
2362 {
2363 (void)InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0);
2364 }
2365 }
2366 if (Pde != (PULONG)PAGEDIRECTORY_MAP)
2367 {
2368 MmDeleteHyperspaceMapping(Pde);
2369 }
2370 }
2371 }
2372
2373 VOID
2374 INIT_FUNCTION
2375 NTAPI
2376 MmInitGlobalKernelPageDirectory(VOID)
2377 {
2378 ULONG i;
2379
2380 DPRINT("MmInitGlobalKernelPageDirectory()\n");
2381
2382 if (Ke386Pae)
2383 {
2384 PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
2385 for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++)
2386 {
2387 if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
2388 !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
2389 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
2390 {
2391 (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
2392 if (Ke386GlobalPagesEnabled)
2393 {
2394 MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
2395 CurrentPageDirectory[i] |= PA_GLOBAL;
2396 }
2397 }
2398 }
2399 }
2400 else
2401 {
2402 PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
2403 for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++)
2404 {
2405 if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
2406 i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
2407 0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
2408 {
2409 MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
2410 if (Ke386GlobalPagesEnabled)
2411 {
2412 MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
2413 CurrentPageDirectory[i] |= PA_GLOBAL;
2414 }
2415 }
2416 }
2417 }
2418 }
2419
2420 ULONG
2421 NTAPI
2422 MiGetUserPageDirectoryCount(VOID)
2423 {
2424 return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart);
2425 }
2426
2427 VOID
2428 INIT_FUNCTION
2429 NTAPI
2430 MiInitPageDirectoryMap(VOID)
2431 {
2432 MEMORY_AREA* kernel_map_desc = NULL;
2433 MEMORY_AREA* hyperspace_desc = NULL;
2434 PHYSICAL_ADDRESS BoundaryAddressMultiple;
2435 PVOID BaseAddress;
2436 NTSTATUS Status;
2437
2438 DPRINT("MiInitPageDirectoryMap()\n");
2439
2440 BoundaryAddressMultiple.QuadPart = 0;
2441 BaseAddress = (PVOID)PAGETABLE_MAP;
2442 Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
2443 MEMORY_AREA_SYSTEM,
2444 &BaseAddress,
2445 Ke386Pae ? 0x800000 : 0x400000,
2446 PAGE_READWRITE,
2447 &kernel_map_desc,
2448 TRUE,
2449 0,
2450 BoundaryAddressMultiple);
2451 if (!NT_SUCCESS(Status))
2452 {
2453 ASSERT(FALSE);
2454 }
2455 BaseAddress = (PVOID)HYPERSPACE;
2456 Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
2457 MEMORY_AREA_SYSTEM,
2458 &BaseAddress,
2459 0x400000,
2460 PAGE_READWRITE,
2461 &hyperspace_desc,
2462 TRUE,
2463 0,
2464 BoundaryAddressMultiple);
2465 if (!NT_SUCCESS(Status))
2466 {
2467 ASSERT(FALSE);
2468 }
2469 }
2470
2471 /* EOF */
A free Windows-compatible Operating System