projects / reactos.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
[THEMES]
[reactos.git] / reactos / ntoskrnl / mm / i386 / pagepae.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/i386/page.c
5 * PURPOSE: Low level memory managment manipulation
6 *
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 */
9
10 /* INCLUDES ***************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 #if defined (ALLOC_PRAGMA)
17 #pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
18 #pragma alloc_text(INIT, MiInitPageDirectoryMap)
19 #endif
20
21
22 /* GLOBALS *****************************************************************/
23
24 #define PA_BIT_PRESENT (0)
25 #define PA_BIT_READWRITE (1)
26 #define PA_BIT_USER (2)
27 #define PA_BIT_WT (3)
28 #define PA_BIT_CD (4)
29 #define PA_BIT_ACCESSED (5)
30 #define PA_BIT_DIRTY (6)
31 #define PA_BIT_GLOBAL (8)
32
33 #define PA_PRESENT (1 << PA_BIT_PRESENT)
34 #define PA_READWRITE (1 << PA_BIT_READWRITE)
35 #define PA_USER (1 << PA_BIT_USER)
36 #define PA_DIRTY (1 << PA_BIT_DIRTY)
37 #define PA_WT (1 << PA_BIT_WT)
38 #define PA_CD (1 << PA_BIT_CD)
39 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
40 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
41
42 #define PAGETABLE_MAP (0xc0000000)
43 #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024)))
44
45 #define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512)))
46
47 #define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000)
48 #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
49
50 ULONG MmGlobalKernelPageDirectory[1024];
51 ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];
52
53 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
54 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
55
56 #define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
57 #define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
58
59 #if defined(__GNUC__)
60 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
61 #else
62 __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage)
63 {
64 LARGE_INTEGER dummy;
65 dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage));
66 return dummy;
67 }
68 #endif
69
70 extern BOOLEAN Ke386Pae;
71 extern BOOLEAN Ke386NoExecute;
72
73 /* FUNCTIONS ***************************************************************/
74
75 BOOLEAN MmUnmapPageTable(PULONG Pt);
76
77 ULONG_PTR
78 NTAPI
79 MiFlushTlbIpiRoutine(ULONG_PTR Address)
80 {
81 if (Address == (ULONGLONG)-1)
82 {
83 KeFlushCurrentTb();
84 }
85 else if (Address == (ULONGLONG)-2)
86 {
87 KeFlushCurrentTb();
88 }
89 else
90 {
91 __invlpg((PVOID)Address);
92 }
93 return 0;
94 }
95
96 VOID
97 MiFlushTlb(PULONG Pt, PVOID Address)
98 {
99 #ifdef CONFIG_SMP
100 if (Pt)
101 {
102 MmUnmapPageTable(Pt);
103 }
104 if (KeNumberProcessors > 1)
105 {
106 KeIpiGenericCall(MiFlushTlbIpiRoutine, (ULONG_PTR)Address);
107 }
108 else
109 {
110 MiFlushTlbIpiRoutine((ULONG_PTR)Address);
111 }
112 #else
113 if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart)
114 {
115 __invlpg(Address);
116 }
117 #endif
118 }
119
120
121
122 PULONG
123 MmGetPageDirectory(VOID)
124 {
125 return (PULONG)__readcr3();
126 }
127
128 static ULONG
129 ProtectToPTE(ULONG flProtect)
130 {
131 ULONG Attributes = 0;
132
133 if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
134 {
135 Attributes = 0;
136 }
137 else if (flProtect & PAGE_IS_WRITABLE)
138 {
139 Attributes = PA_PRESENT | PA_READWRITE;
140 }
141 else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
142 {
143 Attributes = PA_PRESENT;
144 }
145 else
146 {
147 DPRINT1("Unknown main protection type.\n");
148 ASSERT(FALSE);
149 }
150 if (Ke386NoExecute &&
151 !(flProtect & PAGE_IS_EXECUTABLE))
152 {
153 Attributes = Attributes | 0x80000000;
154 }
155
156 if (flProtect & PAGE_SYSTEM)
157 {
158 }
159 else
160 {
161 Attributes = Attributes | PA_USER;
162 }
163 if (flProtect & PAGE_NOCACHE)
164 {
165 Attributes = Attributes | PA_CD;
166 }
167 if (flProtect & PAGE_WRITETHROUGH)
168 {
169 Attributes = Attributes | PA_WT;
170 }
171 return(Attributes);
172 }
173
174 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
175
176 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
177 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
178 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
179
180 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
181
182 #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
183
184
185 #define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE))
186
187 #define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
188 ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
189 #define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
190
191
192 #define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
193
194 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
195
196 #define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
197
198 #define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
199
200
201 NTSTATUS
202 NTAPI
203 Mmi386ReleaseMmInfo(PEPROCESS Process)
204 {
205 PUSHORT LdtDescriptor;
206 ULONG LdtBase;
207 ULONG i, j;
208
209 DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process);
210
211 LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor;
212 LdtBase = LdtDescriptor[1] |
213 ((LdtDescriptor[2] & 0xff) << 16) |
214 ((LdtDescriptor[3] & ~0xff) << 16);
215
216 DPRINT("LdtBase: %x\n", LdtBase);
217
218 if (LdtBase)
219 {
220 ExFreePool((PVOID) LdtBase);
221 }
222
223 if (Ke386Pae)
224 {
225 PULONGLONG PageDirTable;
226 PULONGLONG PageDir;
227 PULONGLONG Pde;
228 ULONG k;
229
230 PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
231 for (i = 0; i < 4; i++)
232 {
233 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i]));
234 if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart))
235 {
236 for (j = 0; j < 512; j++)
237 {
238 if (PageDir[j] != 0LL)
239 {
240 DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
241 Process->UniqueProcessId,
242 (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1,
243 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i*512 + j]);
244 Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j]));
245 for (k = 0; k < 512; k++)
246 {
247 if(Pde[k] != 0)
248 {
249 if (Pde[k] & PA_PRESENT)
250 {
251 DPRINT1("Page at %08x is not freed\n",
252 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
253 }
254 else
255 {
256 DPRINT1("Swapentry %x at %x is not freed\n",
257 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
258 }
259 }
260 }
261 MmDeleteHyperspaceMapping(Pde);
262 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j]));
263 }
264 }
265 }
266 if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE))
267 {
268 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)]));
269 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1]));
270 }
271 MmDeleteHyperspaceMapping(PageDir);
272 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i]));
273 }
274 MmDeleteHyperspaceMapping((PVOID)PageDirTable);
275 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
276 }
277 else
278 {
279 PULONG Pde;
280 PULONG PageDir;
281 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
282 for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++)
283 {
284 if (PageDir[i] != 0)
285 {
286 DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
287 i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1,
288 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable[i]);
289 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i]));
290 for (j = 0; j < 1024; j++)
291 {
292 if(Pde[j] != 0)
293 {
294 if (Pde[j] & PA_PRESENT)
295 {
296 DPRINT1("Page at %08x is not freed\n",
297 i * 4 * 1024 * 1024 + j * PAGE_SIZE);
298 }
299 else
300 {
301 DPRINT1("Swapentry %x at %x is not freed\n",
302 Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE);
303 }
304 }
305 }
306 MmDeleteHyperspaceMapping(Pde);
307 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i]));
308 }
309 }
310 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)]));
311 MmDeleteHyperspaceMapping(PageDir);
312 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
313 }
314
315 #if defined(__GNUC__)
316
317 Process->Pcb.DirectoryTableBase.QuadPart = 0LL;
318 #else
319
320 Process->Pcb.DirectoryTableBase.QuadPart = 0;
321 #endif
322
323 DPRINT("Finished Mmi386ReleaseMmInfo()\n");
324 return(STATUS_SUCCESS);
325 }
326
327 NTSTATUS
328 NTAPI
329 MmInitializeHandBuiltProcess(IN PEPROCESS Process,
330 IN PLARGE_INTEGER DirectoryTableBase)
331 {
332 /* Share the directory base with the idle process */
333 *DirectoryTableBase = PsGetCurrentProcess()->Pcb.DirectoryTableBase;
334
335 /* Initialize the Addresss Space */
336 MmInitializeAddressSpace(Process, (PMADDRESS_SPACE)&Process->VadRoot);
337
338 /* The process now has an address space */
339 Process->HasAddressSpace = TRUE;
340 return STATUS_SUCCESS;
341 }
342
343 BOOLEAN
344 NTAPI
345 MmCreateProcessAddressSpace(IN ULONG MinWs,
346 IN PEPROCESS Process,
347 IN PLARGE_INTEGER DirectoryTableBase)
348 {
349 NTSTATUS Status;
350 ULONG i, j;
351 PFN_NUMBER Pfn[7];
352 ULONG Count;
353
354 DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs, Process);
355
356 Count = Ke386Pae ? 7 : 2;
357
358 for (i = 0; i < Count; i++)
359 {
360 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
361 if (!NT_SUCCESS(Status))
362 {
363 for (j = 0; j < i; j++)
364 {
365 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
366 }
367
368 return FALSE;
369 }
370 }
371
372 if (Ke386Pae)
373 {
374 PULONGLONG PageDirTable;
375 PULONGLONG PageDir;
376
377 PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
378 for (i = 0; i < 4; i++)
379 {
380 PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
381 }
382 MmDeleteHyperspaceMapping(PageDirTable);
383 for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++)
384 {
385 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
386 memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
387 if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
388 {
389 for (j = 0; j < 4; j++)
390 {
391 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
392 }
393 }
394 if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
395 {
396 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
397 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
398 }
399 MmDeleteHyperspaceMapping(PageDir);
400 }
401 }
402 else
403 {
404 PULONG PageDirectory;
405 PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);
406
407 memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
408 MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
409 (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG));
410
411 DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
412 PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
413 PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;
414
415 MmDeleteHyperspaceMapping(PageDirectory);
416 }
417
418 DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]);
419 DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart);
420 return TRUE;
421 }
422
423 VOID
424 NTAPI
425 MmDeletePageTable(PEPROCESS Process, PVOID Address)
426 {
427 PEPROCESS CurrentProcess = PsGetCurrentProcess();
428
429 if (Process != NULL && Process != CurrentProcess)
430 {
431 KeAttachProcess(&Process->Pcb);
432 }
433
434 if (Ke386Pae)
435 {
436 ULONGLONG ZeroPde = 0LL;
437 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde);
438 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
439 }
440 else
441 {
442 *(ADDR_TO_PDE(Address)) = 0;
443 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
444 }
445 if (Address >= MmSystemRangeStart)
446 {
447 ASSERT(FALSE);
448 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
449 }
450 if (Process != NULL && Process != CurrentProcess)
451 {
452 KeDetachProcess();
453 }
454 }
455
456 VOID
457 NTAPI
458 MmFreePageTable(PEPROCESS Process, PVOID Address)
459 {
460 PEPROCESS CurrentProcess = PsGetCurrentProcess();
461 ULONG i;
462 PFN_NUMBER Pfn;
463
464 DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
465 if (Process != NULL && Process != CurrentProcess)
466 {
467 KeAttachProcess(&Process->Pcb);
468 }
469 if (Ke386Pae)
470 {
471 PULONGLONG PageTable;
472 ULONGLONG ZeroPte = 0LL;
473 PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
474 for (i = 0; i < 512; i++)
475 {
476 if (PageTable[i] != 0LL)
477 {
478 DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
479 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
480 ASSERT(FALSE);
481 }
482 }
483 Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
484 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
485 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
486 }
487 else
488 {
489 PULONG PageTable;
490 PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
491 for (i = 0; i < 1024; i++)
492 {
493 if (PageTable[i] != 0)
494 {
495 DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
496 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
497 ASSERT(FALSE);
498 }
499 }
500 Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
501 *(ADDR_TO_PDE(Address)) = 0;
502 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
503 }
504
505 if (Address >= MmSystemRangeStart)
506 {
507 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
508 ASSERT(FALSE);
509 }
510 else
511 {
512 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
513 }
514 if (Process != NULL && Process != CurrentProcess)
515 {
516 KeDetachProcess();
517 }
518 }
519
520 static PULONGLONG
521 MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
522 {
523 NTSTATUS Status;
524 PFN_NUMBER Pfn;
525 ULONGLONG Entry;
526 ULONGLONG ZeroEntry = 0LL;
527 PULONGLONG Pt;
528 PULONGLONG PageDir;
529 PULONGLONG PageDirTable;
530
531 DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
532 Process, Address, Create);
533 if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000))
534 {
535 ASSERT(FALSE);
536 }
537 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
538 {
539 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
540 if (PageDirTable == NULL)
541 {
542 ASSERT(FALSE);
543 }
544 PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
545 MmDeleteHyperspaceMapping(PageDirTable);
546 if (PageDir == NULL)
547 {
548 ASSERT(FALSE);
549 }
550 PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
551 Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
552 if (Entry == 0LL)
553 {
554 if (Create == FALSE)
555 {
556 MmDeleteHyperspaceMapping(PageDir);
557 return NULL;
558 }
559 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
560 if (!NT_SUCCESS(Status))
561 {
562 ASSERT(FALSE);
563 }
564 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
565 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
566 if (Entry != 0LL)
567 {
568 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
569 Pfn = PAE_PTE_TO_PFN(Entry);
570 }
571 }
572 else
573 {
574 Pfn = PAE_PTE_TO_PFN(Entry);
575 }
576 MmDeleteHyperspaceMapping(PageDir);
577 Pt = MmCreateHyperspaceMapping(Pfn);
578 if (Pt == NULL)
579 {
580 ASSERT(FALSE);
581 }
582 return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
583 }
584 PageDir = PAE_ADDR_TO_PDE(Address);
585 if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
586 {
587 if (Address >= MmSystemRangeStart)
588 {
589 if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
590 {
591 if (Create == FALSE)
592 {
593 return NULL;
594 }
595 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
596 if (!NT_SUCCESS(Status))
597 {
598 ASSERT(FALSE);
599 }
600 Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
601 if (Ke386GlobalPagesEnabled)
602 {
603 Entry |= PA_GLOBAL;
604 }
605 if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
606 {
607 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
608 }
609 }
610 (void)ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
611 }
612 else
613 {
614 if (Create == FALSE)
615 {
616 return NULL;
617 }
618 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
619 if (!NT_SUCCESS(Status))
620 {
621 ASSERT(FALSE);
622 }
623 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
624 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
625 if (Entry != 0LL)
626 {
627 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
628 }
629 }
630 }
631 return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
632 }
633
634 static PULONG
635 MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
636 {
637 ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
638 NTSTATUS Status;
639 PFN_NUMBER Pfn;
640 ULONG Entry;
641 PULONG Pt, PageDir;
642
643 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
644 {
645 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.LowPart));
646 if (PageDir == NULL)
647 {
648 ASSERT(FALSE);
649 }
650 if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
651 {
652 if (Create == FALSE)
653 {
654 MmDeleteHyperspaceMapping(PageDir);
655 return NULL;
656 }
657 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
658 if (!NT_SUCCESS(Status) || Pfn == 0)
659 {
660 ASSERT(FALSE);
661 }
662 Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
663 if (Entry != 0)
664 {
665 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
666 Pfn = PTE_TO_PFN(Entry);
667 }
668 }
669 else
670 {
671 Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
672 }
673 MmDeleteHyperspaceMapping(PageDir);
674 Pt = MmCreateHyperspaceMapping(Pfn);
675 if (Pt == NULL)
676 {
677 ASSERT(FALSE);
678 }
679 return Pt + ADDR_TO_PTE_OFFSET(Address);
680 }
681 PageDir = ADDR_TO_PDE(Address);
682 if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
683 {
684 if (Address >= MmSystemRangeStart)
685 {
686 if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
687 {
688 if (Create == FALSE)
689 {
690 return NULL;
691 }
692 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
693 if (!NT_SUCCESS(Status) || Pfn == 0)
694 {
695 ASSERT(FALSE);
696 }
697 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
698 if (Ke386GlobalPagesEnabled)
699 {
700 Entry |= PA_GLOBAL;
701 }
702 if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
703 {
704 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
705 }
706 }
707 (void)InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
708 }
709 else
710 {
711 if (Create == FALSE)
712 {
713 return NULL;
714 }
715 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
716 if (!NT_SUCCESS(Status) || Pfn == 0)
717 {
718 ASSERT(FALSE);
719 }
720 Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
721 if (Entry != 0)
722 {
723 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
724 }
725 }
726 }
727 return (PULONG)ADDR_TO_PTE(Address);
728 }
729
730 BOOLEAN MmUnmapPageTable(PULONG Pt)
731 {
732 if (Ke386Pae)
733 {
734 if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
735 {
736 return TRUE;
737 }
738 }
739 else
740 {
741 if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
742 {
743 return TRUE;
744 }
745 }
746 if (Pt)
747 {
748 MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
749 }
750 return FALSE;
751 }
752
753 static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
754 {
755 ULONGLONG Pte;
756 PULONGLONG Pt;
757
758 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
759 if (Pt)
760 {
761 Pte = *Pt;
762 MmUnmapPageTable((PULONG)Pt);
763 return Pte;
764 }
765 return 0;
766 }
767
768 static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
769 {
770 ULONG Pte;
771 PULONG Pt;
772
773 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
774 if (Pt)
775 {
776 Pte = *Pt;
777 MmUnmapPageTable(Pt);
778 return Pte;
779 }
780 return 0;
781 }
782
783 PFN_NUMBER
784 NTAPI
785 MmGetPfnForProcess(PEPROCESS Process,
786 PVOID Address)
787 {
788
789 if (Ke386Pae)
790 {
791 ULONGLONG Entry;
792 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
793 if (!(Entry & PA_PRESENT))
794 {
795 return 0;
796 }
797 return(PAE_PTE_TO_PFN(Entry));
798 }
799 else
800 {
801 ULONG Entry;
802 Entry = MmGetPageEntryForProcess(Process, Address);
803 if (!(Entry & PA_PRESENT))
804 {
805 return 0;
806 }
807 return(PTE_TO_PFN(Entry));
808 }
809 }
810
811 VOID
812 NTAPI
813 MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage,
814 BOOLEAN* WasDirty, PPFN_NUMBER Page)
815 /*
816 * FUNCTION: Delete a virtual mapping
817 */
818 {
819 BOOLEAN WasValid = FALSE;
820 PFN_NUMBER Pfn;
821
822 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
823 Process, Address, FreePage, WasDirty, Page);
824 if (Ke386Pae)
825 {
826 ULONGLONG Pte;
827 PULONGLONG Pt;
828
829 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
830 if (Pt == NULL)
831 {
832 if (WasDirty != NULL)
833 {
834 *WasDirty = FALSE;
835 }
836 if (Page != NULL)
837 {
838 *Page = 0;
839 }
840 return;
841 }
842
843 /*
844 * Atomically set the entry to zero and get the old value.
845 */
846 Pte = 0LL;
847 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
848
849 MiFlushTlb((PULONG)Pt, Address);
850
851 WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
852 if (WasValid)
853 {
854 Pfn = PAE_PTE_TO_PFN(Pte);
855 MmMarkPageUnmapped(Pfn);
856 }
857 else
858 {
859 Pfn = 0;
860 }
861
862 if (FreePage && WasValid)
863 {
864 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
865 }
866
867 /*
868 * Return some information to the caller
869 */
870 if (WasDirty != NULL)
871 {
872 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
873 }
874 if (Page != NULL)
875 {
876 *Page = Pfn;
877 }
878 }
879 else
880 {
881 ULONG Pte;
882 PULONG Pt;
883
884 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
885
886 if (Pt == NULL)
887 {
888 if (WasDirty != NULL)
889 {
890 *WasDirty = FALSE;
891 }
892 if (Page != NULL)
893 {
894 *Page = 0;
895 }
896 return;
897 }
898
899 /*
900 * Atomically set the entry to zero and get the old value.
901 */
902 Pte = InterlockedExchangeUL(Pt, 0);
903
904 MiFlushTlb(Pt, Address);
905
906 WasValid = (PAGE_MASK(Pte) != 0);
907 if (WasValid)
908 {
909 Pfn = PTE_TO_PFN(Pte);
910 MmMarkPageUnmapped(Pfn);
911 }
912 else
913 {
914 Pfn = 0;
915 }
916
917 if (FreePage && WasValid)
918 {
919 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
920 }
921
922 /*
923 * Return some information to the caller
924 */
925 if (WasDirty != NULL)
926 {
927 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
928 }
929 if (Page != NULL)
930 {
931 *Page = Pfn;
932 }
933 }
934 /*
935 * Decrement the reference count for this page table.
936 */
937 if (Process != NULL && WasValid &&
938 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
939 Address < MmSystemRangeStart)
940 {
941 PUSHORT Ptrc;
942 ULONG Idx;
943
944 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
945 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
946
947 Ptrc[Idx]--;
948 if (Ptrc[Idx] == 0)
949 {
950 MmFreePageTable(Process, Address);
951 }
952 }
953 }
954
955 VOID
956 NTAPI
957 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
958 SWAPENTRY* SwapEntry)
959 /*
960 * FUNCTION: Delete a virtual mapping
961 */
962 {
963 if (Ke386Pae)
964 {
965 ULONGLONG Pte;
966 PULONGLONG Pt;
967
968 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
969 if (Pt == NULL)
970 {
971 *SwapEntry = 0;
972 return;
973 }
974
975 /*
976 * Atomically set the entry to zero and get the old value.
977 */
978 Pte = 0LL;
979 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
980
981 MiFlushTlb((PULONG)Pt, Address);
982
983 /*
984 * Decrement the reference count for this page table.
985 */
986 if (Process != NULL && Pte &&
987 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
988 Address < MmSystemRangeStart)
989 {
990 PUSHORT Ptrc;
991
992 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
993
994 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
995 if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
996 {
997 MmFreePageTable(Process, Address);
998 }
999 }
1000
1001
1002 /*
1003 * Return some information to the caller
1004 */
1005 *SwapEntry = Pte >> 1;
1006 }
1007 else
1008 {
1009 ULONG Pte;
1010 PULONG Pt;
1011
1012 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1013
1014 if (Pt == NULL)
1015 {
1016 *SwapEntry = 0;
1017 return;
1018 }
1019
1020 /*
1021 * Atomically set the entry to zero and get the old value.
1022 */
1023 Pte = InterlockedExchangeUL(Pt, 0);
1024
1025 MiFlushTlb(Pt, Address);
1026
1027 /*
1028 * Decrement the reference count for this page table.
1029 */
1030 if (Process != NULL && Pte &&
1031 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1032 Address < MmSystemRangeStart)
1033 {
1034 PUSHORT Ptrc;
1035
1036 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1037
1038 Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
1039 if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
1040 {
1041 MmFreePageTable(Process, Address);
1042 }
1043 }
1044
1045
1046 /*
1047 * Return some information to the caller
1048 */
1049 *SwapEntry = Pte >> 1;
1050 }
1051 }
1052
1053 BOOLEAN
1054 Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
1055 {
1056 if (Ke386Pae)
1057 {
1058 PULONGLONG Pt;
1059 PULONGLONG Pde;
1060 Pde = PAE_ADDR_TO_PDE(PAddress);
1061 if (*Pde == 0LL)
1062 {
1063 Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
1064 #if 0
1065 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1066 FLASH_TLB_ONE(PAddress);
1067 #endif
1068 if (Pt != NULL)
1069 {
1070 return TRUE;
1071 }
1072 }
1073 }
1074 else
1075 {
1076 PULONG Pt, Pde;
1077 Pde = ADDR_TO_PDE(PAddress);
1078 if (*Pde == 0)
1079 {
1080 Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
1081 #if 0
1082 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1083 FLASH_TLB_ONE(PAddress);
1084 #endif
1085 if (Pt != NULL)
1086 {
1087 return TRUE;
1088 }
1089 }
1090 }
1091 return(FALSE);
1092 }
1093
1094 BOOLEAN
1095 NTAPI
1096 MmIsDirtyPage(PEPROCESS Process, PVOID Address)
1097 {
1098 if (Ke386Pae)
1099 {
1100 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1101 }
1102 else
1103 {
1104 return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1105 }
1106 }
1107
1108 BOOLEAN
1109 NTAPI
1110 MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address)
1111 {
1112 if (Address < MmSystemRangeStart && Process == NULL)
1113 {
1114 DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1115 ASSERT(FALSE);
1116 }
1117 if (Ke386Pae)
1118 {
1119 PULONGLONG Pt;
1120 ULONGLONG Pte;
1121 ULONGLONG tmpPte;
1122
1123 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1124 if (Pt == NULL)
1125 {
1126 ASSERT(FALSE);
1127 }
1128
1129 do
1130 {
1131 Pte = *Pt;
1132 tmpPte = Pte & ~PA_ACCESSED;
1133 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1134
1135 if (Pte & PA_ACCESSED)
1136 {
1137 MiFlushTlb((PULONG)Pt, Address);
1138 return TRUE;
1139 }
1140 else
1141 {
1142 MmUnmapPageTable((PULONG)Pt);
1143 return FALSE;
1144 }
1145 }
1146 else
1147 {
1148 PULONG Pt;
1149 ULONG Pte;
1150
1151 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1152 if (Pt == NULL)
1153 {
1154 ASSERT(FALSE);
1155 }
1156
1157 do
1158 {
1159 Pte = *Pt;
1160 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte));
1161
1162 if (Pte & PA_ACCESSED)
1163 {
1164 MiFlushTlb(Pt, Address);
1165 return TRUE;
1166 }
1167 else
1168 {
1169 MmUnmapPageTable(Pt);
1170 return FALSE;
1171 }
1172 }
1173 }
1174
1175 VOID
1176 NTAPI
1177 MmSetCleanPage(PEPROCESS Process, PVOID Address)
1178 {
1179 if (Address < MmSystemRangeStart && Process == NULL)
1180 {
1181 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1182 ASSERT(FALSE);
1183 }
1184 if (Ke386Pae)
1185 {
1186 PULONGLONG Pt;
1187 ULONGLONG Pte;
1188 ULONGLONG tmpPte;
1189
1190 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1191
1192 if (Pt == NULL)
1193 {
1194 ASSERT(FALSE);
1195 }
1196
1197 do
1198 {
1199 Pte = *Pt;
1200 tmpPte = Pte & ~PA_DIRTY;
1201 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1202
1203 if (Pte & PA_DIRTY)
1204 {
1205 MiFlushTlb((PULONG)Pt, Address);
1206 }
1207 else
1208 {
1209 MmUnmapPageTable((PULONG)Pt);
1210 }
1211 }
1212 else
1213 {
1214 PULONG Pt;
1215 ULONG Pte;
1216
1217 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1218
1219 if (Pt == NULL)
1220 {
1221 ASSERT(FALSE);
1222 }
1223
1224 do
1225 {
1226 Pte = *Pt;
1227 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));
1228
1229 if (Pte & PA_DIRTY)
1230 {
1231 MiFlushTlb(Pt, Address);
1232 }
1233 else
1234 {
1235 MmUnmapPageTable(Pt);
1236 }
1237 }
1238 }
1239
1240 VOID
1241 NTAPI
1242 MmSetDirtyPage(PEPROCESS Process, PVOID Address)
1243 {
1244 if (Address < MmSystemRangeStart && Process == NULL)
1245 {
1246 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1247 ASSERT(FALSE);
1248 }
1249 if (Ke386Pae)
1250 {
1251 PULONGLONG Pt;
1252 ULONGLONG Pte;
1253 ULONGLONG tmpPte;
1254
1255 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1256 if (Pt == NULL)
1257 {
1258 ASSERT(FALSE);
1259 }
1260
1261 do
1262 {
1263 Pte = *Pt;
1264 tmpPte = Pte | PA_DIRTY;
1265 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1266 if (!(Pte & PA_DIRTY))
1267 {
1268 MiFlushTlb((PULONG)Pt, Address);
1269 }
1270 else
1271 {
1272 MmUnmapPageTable((PULONG)Pt);
1273 }
1274 }
1275 else
1276 {
1277 PULONG Pt;
1278 ULONG Pte;
1279
1280 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1281 if (Pt == NULL)
1282 {
1283 ASSERT(FALSE);
1284 }
1285
1286 do
1287 {
1288 Pte = *Pt;
1289 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
1290 if (!(Pte & PA_DIRTY))
1291 {
1292 MiFlushTlb(Pt, Address);
1293 }
1294 else
1295 {
1296 MmUnmapPageTable(Pt);
1297 }
1298 }
1299 }
1300
1301 VOID
1302 NTAPI
1303 MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
1304 {
1305 if (Ke386Pae)
1306 {
1307 PULONGLONG Pt;
1308 ULONGLONG Pte;
1309 ULONGLONG tmpPte;
1310
1311 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1312 if (Pt == NULL)
1313 {
1314 ASSERT(FALSE);
1315 }
1316
1317 do
1318 {
1319 Pte = *Pt;
1320 tmpPte = Pte | PA_PRESENT;
1321 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1322 if (!(Pte & PA_PRESENT))
1323 {
1324 MiFlushTlb((PULONG)Pt, Address);
1325 }
1326 else
1327 {
1328 MmUnmapPageTable((PULONG)Pt);
1329 }
1330 }
1331 else
1332 {
1333 PULONG Pt;
1334 ULONG Pte;
1335
1336 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1337 if (Pt == NULL)
1338 {
1339 ASSERT(FALSE);
1340 }
1341
1342 do
1343 {
1344 Pte = *Pt;
1345 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte));
1346 if (!(Pte & PA_PRESENT))
1347 {
1348 MiFlushTlb(Pt, Address);
1349 }
1350 else
1351 {
1352 MmUnmapPageTable(Pt);
1353 }
1354 }
1355 }
1356
1357 BOOLEAN
1358 NTAPI
1359 MmIsPagePresent(PEPROCESS Process, PVOID Address)
1360 {
1361 if (Ke386Pae)
1362 {
1363 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1364 }
1365 else
1366 {
1367 return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1368 }
1369 }
1370
1371 BOOLEAN
1372 NTAPI
1373 MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
1374 {
1375 if (Ke386Pae)
1376 {
1377 ULONGLONG Entry;
1378 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1379 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1380 }
1381 else
1382 {
1383 ULONG Entry;
1384 Entry = MmGetPageEntryForProcess(Process, Address);
1385 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1386 }
1387 }
1388
1389 NTSTATUS
1390 NTAPI
1391 MmCreateVirtualMappingForKernel(PVOID Address,
1392 ULONG flProtect,
1393 PPFN_NUMBER Pages,
1394 ULONG PageCount)
1395 {
1396 ULONG Attributes;
1397 ULONG i;
1398 PVOID Addr;
1399 ULONG PdeOffset, oldPdeOffset;
1400 BOOLEAN NoExecute = FALSE;
1401
1402 DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1403 Address, flProtect, Pages, PageCount);
1404
1405 if (Address < MmSystemRangeStart)
1406 {
1407 DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1408 ASSERT(FALSE);
1409 }
1410
1411 Attributes = ProtectToPTE(flProtect);
1412 if (Attributes & 0x80000000)
1413 {
1414 NoExecute = TRUE;
1415 }
1416 Attributes &= 0xfff;
1417 if (Ke386GlobalPagesEnabled)
1418 {
1419 Attributes |= PA_GLOBAL;
1420 }
1421
1422 Addr = Address;
1423
1424 if (Ke386Pae)
1425 {
1426 PULONGLONG Pt = NULL;
1427 ULONGLONG Pte;
1428
1429 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1430 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1431 {
1432 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1433 {
1434 DPRINT1("Setting physical address but not allowing access at address "
1435 "0x%.8X with attributes %x/%x.\n",
1436 Addr, Attributes, flProtect);
1437 ASSERT(FALSE);
1438 }
1439
1440 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1441 if (oldPdeOffset != PdeOffset)
1442 {
1443 Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE);
1444 if (Pt == NULL)
1445 {
1446 ASSERT(FALSE);
1447 }
1448 }
1449 else
1450 {
1451 Pt++;
1452 }
1453 oldPdeOffset = PdeOffset;
1454
1455 Pte = PFN_TO_PTE(Pages[i]) | Attributes;
1456 if (NoExecute)
1457 {
1458 Pte |= 0x8000000000000000LL;
1459 }
1460 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1461 if (Pte != 0LL)
1462 {
1463 ASSERT(FALSE);
1464 }
1465 }
1466 }
1467 else
1468 {
1469 PULONG Pt;
1470 ULONG Pte;
1471
1472 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1473 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1474 if (Pt == NULL)
1475 {
1476 ASSERT(FALSE);
1477 }
1478 Pt--;
1479
1480 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1481 {
1482 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1483 {
1484 DPRINT1("Setting physical address but not allowing access at address "
1485 "0x%.8X with attributes %x/%x.\n",
1486 Addr, Attributes, flProtect);
1487 ASSERT(FALSE);
1488 }
1489
1490 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1491 if (oldPdeOffset != PdeOffset)
1492 {
1493 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1494 if (Pt == NULL)
1495 {
1496 ASSERT(FALSE);
1497 }
1498 }
1499 else
1500 {
1501 Pt++;
1502 }
1503 oldPdeOffset = PdeOffset;
1504
1505 Pte = *Pt;
1506 if (Pte != 0)
1507 {
1508 ASSERT(FALSE);
1509 }
1510 (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1511 }
1512 }
1513
1514 return(STATUS_SUCCESS);
1515 }
1516
1517 NTSTATUS
1518 NTAPI
1519 MmCreatePageFileMapping(PEPROCESS Process,
1520 PVOID Address,
1521 SWAPENTRY SwapEntry)
1522 {
1523 if (Process == NULL && Address < MmSystemRangeStart)
1524 {
1525 DPRINT1("No process\n");
1526 ASSERT(FALSE);
1527 }
1528 if (Process != NULL && Address >= MmSystemRangeStart)
1529 {
1530 DPRINT1("Setting kernel address with process context\n");
1531 ASSERT(FALSE);
1532 }
1533 if (SwapEntry & (1 << 31))
1534 {
1535 ASSERT(FALSE);
1536 }
1537
1538 if (Ke386Pae)
1539 {
1540 PULONGLONG Pt;
1541 ULONGLONG Pte;
1542 ULONGLONG tmpPte;
1543
1544 Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
1545 if (Pt == NULL)
1546 {
1547 ASSERT(FALSE);
1548 }
1549 tmpPte = SwapEntry << 1;
1550 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1551 if (PAE_PAGE_MASK((Pte)) != 0)
1552 {
1553 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1554 }
1555
1556 if (Pte != 0)
1557 {
1558 MiFlushTlb((PULONG)Pt, Address);
1559 }
1560 else
1561 {
1562 MmUnmapPageTable((PULONG)Pt);
1563 }
1564 }
1565 else
1566 {
1567 PULONG Pt;
1568 ULONG Pte;
1569
1570 Pt = MmGetPageTableForProcess(Process, Address, TRUE);
1571 if (Pt == NULL)
1572 {
1573 ASSERT(FALSE);
1574 }
1575 Pte = *Pt;
1576 if (PAGE_MASK((Pte)) != 0)
1577 {
1578 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1579 }
1580 (void)InterlockedExchangeUL(Pt, SwapEntry << 1);
1581 if (Pte != 0)
1582 {
1583 MiFlushTlb(Pt, Address);
1584 }
1585 else
1586 {
1587 MmUnmapPageTable(Pt);
1588 }
1589 }
1590 if (Process != NULL &&
1591 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1592 Address < MmSystemRangeStart)
1593 {
1594 PUSHORT Ptrc;
1595 ULONG Idx;
1596
1597 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1598 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1599 Ptrc[Idx]++;
1600 }
1601 return(STATUS_SUCCESS);
1602 }
1603
1604
1605 NTSTATUS
1606 NTAPI
1607 MmCreateVirtualMappingUnsafe(PEPROCESS Process,
1608 PVOID Address,
1609 ULONG flProtect,
1610 PPFN_NUMBER Pages,
1611 ULONG PageCount)
1612 {
1613 ULONG Attributes;
1614 PVOID Addr;
1615 ULONG i;
1616 ULONG oldPdeOffset, PdeOffset;
1617 BOOLEAN NoExecute = FALSE;
1618
1619 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1620 Process, Address, flProtect, Pages, *Pages, PageCount);
1621
1622 if (Process == NULL)
1623 {
1624 if (Address < MmSystemRangeStart)
1625 {
1626 DPRINT1("No process\n");
1627 ASSERT(FALSE);
1628 }
1629 if (PageCount > 0x10000 ||
1630 (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
1631 {
1632 DPRINT1("Page count to large\n");
1633 ASSERT(FALSE);
1634 }
1635 }
1636 else
1637 {
1638 if (Address >= MmSystemRangeStart)
1639 {
1640 DPRINT1("Setting kernel address with process context\n");
1641 ASSERT(FALSE);
1642 }
1643 if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE ||
1644 (ULONG_PTR) Address / PAGE_SIZE + PageCount >
1645 (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE)
1646 {
1647 DPRINT1("Page Count to large\n");
1648 ASSERT(FALSE);
1649 }
1650 }
1651
1652 Attributes = ProtectToPTE(flProtect);
1653 if (Attributes & 0x80000000)
1654 {
1655 NoExecute = TRUE;
1656 }
1657 Attributes &= 0xfff;
1658 if (Address >= MmSystemRangeStart)
1659 {
1660 Attributes &= ~PA_USER;
1661 if (Ke386GlobalPagesEnabled)
1662 {
1663 Attributes |= PA_GLOBAL;
1664 }
1665 }
1666 else
1667 {
1668 Attributes |= PA_USER;
1669 }
1670
1671 Addr = Address;
1672
1673 if (Ke386Pae)
1674 {
1675 ULONGLONG Pte, tmpPte;
1676 PULONGLONG Pt = NULL;
1677
1678 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1679 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1680 {
1681 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1682 {
1683 DPRINT1("Setting physical address but not allowing access at address "
1684 "0x%.8X with attributes %x/%x.\n",
1685 Addr, Attributes, flProtect);
1686 ASSERT(FALSE);
1687 }
1688 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1689 if (oldPdeOffset != PdeOffset)
1690 {
1691 MmUnmapPageTable((PULONG)Pt);
1692 Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
1693 if (Pt == NULL)
1694 {
1695 ASSERT(FALSE);
1696 }
1697 }
1698 else
1699 {
1700 Pt++;
1701 }
1702 oldPdeOffset = PdeOffset;
1703
1704 MmMarkPageMapped(Pages[i]);
1705 tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
1706 if (NoExecute)
1707 {
1708 tmpPte |= 0x8000000000000000LL;
1709 }
1710 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1711 if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
1712 {
1713 ASSERT(FALSE);
1714 }
1715 if (PAE_PAGE_MASK((Pte)) != 0LL)
1716 {
1717 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1718 }
1719 if (Address < MmSystemRangeStart &&
1720 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1721 Attributes & PA_PRESENT)
1722 {
1723 PUSHORT Ptrc;
1724
1725 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1726
1727 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
1728 }
1729 if (Pte != 0LL)
1730 {
1731 if (Address > MmSystemRangeStart ||
1732 (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
1733 {
1734 MiFlushTlb((PULONG)Pt, Address);
1735 }
1736 }
1737 }
1738 if (Addr > Address)
1739 {
1740 MmUnmapPageTable((PULONG)Pt);
1741 }
1742 }
1743 else
1744 {
1745 PULONG Pt = NULL;
1746 ULONG Pte;
1747 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
1748 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1749 {
1750 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1751 {
1752 DPRINT1("Setting physical address but not allowing access at address "
1753 "0x%.8X with attributes %x/%x.\n",
1754 Addr, Attributes, flProtect);
1755 ASSERT(FALSE);
1756 }
1757 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1758 if (oldPdeOffset != PdeOffset)
1759 {
1760 MmUnmapPageTable(Pt);
1761 Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
1762 if (Pt == NULL)
1763 {
1764 ASSERT(FALSE);
1765 }
1766 }
1767 else
1768 {
1769 Pt++;
1770 }
1771 oldPdeOffset = PdeOffset;
1772
1773 Pte = *Pt;
1774 MmMarkPageMapped(Pages[i]);
1775 if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
1776 {
1777 ASSERT(FALSE);
1778 }
1779 if (PAGE_MASK((Pte)) != 0)
1780 {
1781 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1782 }
1783 (void)InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1784 if (Address < MmSystemRangeStart &&
1785 ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable != NULL &&
1786 Attributes & PA_PRESENT)
1787 {
1788 PUSHORT Ptrc;
1789
1790 Ptrc = ((PMADDRESS_SPACE)&Process->VadRoot)->PageTableRefCountTable;
1791
1792 Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
1793 }
1794 if (Pte != 0)
1795 {
1796 if (Address > MmSystemRangeStart ||
1797 (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
1798 {
1799 MiFlushTlb(Pt, Address);
1800 }
1801 }
1802 }
1803 if (Addr > Address)
1804 {
1805 MmUnmapPageTable(Pt);
1806 }
1807 }
1808 return(STATUS_SUCCESS);
1809 }
1810
1811 NTSTATUS
1812 NTAPI
1813 MmCreateVirtualMapping(PEPROCESS Process,
1814 PVOID Address,
1815 ULONG flProtect,
1816 PPFN_NUMBER Pages,
1817 ULONG PageCount)
1818 {
1819 ULONG i;
1820
1821 for (i = 0; i < PageCount; i++)
1822 {
1823 if (!MmIsPageInUse(Pages[i]))
1824 {
1825 DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages[i]));
1826 ASSERT(FALSE);
1827 }
1828 }
1829
1830 return(MmCreateVirtualMappingUnsafe(Process,
1831 Address,
1832 flProtect,
1833 Pages,
1834 PageCount));
1835 }
1836
1837 ULONG
1838 NTAPI
1839 MmGetPageProtect(PEPROCESS Process, PVOID Address)
1840 {
1841 ULONG Entry;
1842 ULONG Protect;
1843 if (Ke386Pae)
1844 {
1845 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1846 }
1847 else
1848 {
1849 Entry = MmGetPageEntryForProcess(Process, Address);
1850 }
1851
1852 if (!(Entry & PA_PRESENT))
1853 {
1854 Protect = PAGE_NOACCESS;
1855 }
1856 else
1857 {
1858 if (Entry & PA_READWRITE)
1859 {
1860 Protect = PAGE_READWRITE;
1861 }
1862 else
1863 {
1864 Protect = PAGE_EXECUTE_READ;
1865 }
1866 if (Entry & PA_CD)
1867 {
1868 Protect |= PAGE_NOCACHE;
1869 }
1870 if (Entry & PA_WT)
1871 {
1872 Protect |= PAGE_WRITETHROUGH;
1873 }
1874 if (!(Entry & PA_USER))
1875 {
1876 Protect |= PAGE_SYSTEM;
1877 }
1878
1879 }
1880 return(Protect);
1881 }
1882
1883 VOID
1884 NTAPI
1885 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
1886 {
1887 ULONG Attributes = 0;
1888 BOOLEAN NoExecute = FALSE;
1889
1890 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
1891 Process, Address, flProtect);
1892
1893 Attributes = ProtectToPTE(flProtect);
1894 if (Attributes & 0x80000000)
1895 {
1896 NoExecute = TRUE;
1897 }
1898 Attributes &= 0xfff;
1899 if (Address >= MmSystemRangeStart)
1900 {
1901 Attributes &= ~PA_USER;
1902 if (Ke386GlobalPagesEnabled)
1903 {
1904 Attributes |= PA_GLOBAL;
1905 }
1906 }
1907 else
1908 {
1909 Attributes |= PA_USER;
1910 }
1911 if (Ke386Pae)
1912 {
1913 PULONGLONG Pt;
1914 ULONGLONG tmpPte, Pte;
1915
1916 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1917 if (Pt == NULL)
1918 {
1919 DPRINT1("Address %x\n", Address);
1920 ASSERT(FALSE);
1921 }
1922 do
1923 {
1924 Pte = *Pt;
1925 tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
1926 if (NoExecute)
1927 {
1928 tmpPte |= 0x8000000000000000LL;
1929 }
1930 else
1931 {
1932 tmpPte &= ~0x8000000000000000LL;
1933 }
1934 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1935
1936 MiFlushTlb((PULONG)Pt, Address);
1937 }
1938 else
1939 {
1940 PULONG Pt;
1941
1942 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1943 if (Pt == NULL)
1944 {
1945 ASSERT(FALSE);
1946 }
1947 InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
1948 MiFlushTlb(Pt, Address);
1949 }
1950 }
1951
1952 /*
1953 * @implemented
1954 */
1955 PHYSICAL_ADDRESS NTAPI
1956 MmGetPhysicalAddress(PVOID vaddr)
1957 /*
1958 * FUNCTION: Returns the physical address corresponding to a virtual address
1959 */
1960 {
1961 PHYSICAL_ADDRESS p;
1962
1963 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr);
1964 if (Ke386Pae)
1965 {
1966 ULONGLONG Pte;
1967 Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr);
1968 if (Pte != 0 && Pte & PA_PRESENT)
1969 {
1970 p.QuadPart = PAE_PAGE_MASK(Pte);
1971 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
1972 }
1973 else
1974 {
1975 p.QuadPart = 0;
1976 }
1977 }
1978 else
1979 {
1980 ULONG Pte;
1981 Pte = MmGetPageEntryForProcess(NULL, vaddr);
1982 if (Pte != 0 && Pte & PA_PRESENT)
1983 {
1984 p.QuadPart = PAGE_MASK(Pte);
1985 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
1986 }
1987 else
1988 {
1989 p.QuadPart = 0;
1990 }
1991 }
1992 return p;
1993 }
1994
1995 PVOID
1996 NTAPI
1997 MmCreateHyperspaceMapping(PFN_NUMBER Page)
1998 {
1999 PVOID Address;
2000 ULONG i;
2001
2002 if (Ke386Pae)
2003 {
2004 ULONGLONG Entry;
2005 ULONGLONG ZeroEntry = 0LL;
2006 PULONGLONG Pte;
2007
2008 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2009 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2010
2011 if (Page & 1024)
2012 {
2013 for (i = Page %1024; i < 1024; i++, Pte++)
2014 {
2015 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2016 {
2017 break;
2018 }
2019 }
2020 if (i >= 1024)
2021 {
2022 Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
2023 for (i = 0; i < Page % 1024; i++, Pte++)
2024 {
2025 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2026 {
2027 break;
2028 }
2029 }
2030 if (i >= Page % 1024)
2031 {
2032 ASSERT(FALSE);
2033 }
2034 }
2035 }
2036 else
2037 {
2038 for (i = Page %1024; (LONG)i >= 0; i--, Pte--)
2039 {
2040 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2041 {
2042 break;
2043 }
2044 }
2045 if ((LONG)i < 0)
2046 {
2047 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
2048 for (i = 1023; i > Page % 1024; i--, Pte--)
2049 {
2050 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2051 {
2052 break;
2053 }
2054 }
2055 if (i <= Page % 1024)
2056 {
2057 ASSERT(FALSE);
2058 }
2059 }
2060 }
2061 }
2062 else
2063 {
2064 ULONG Entry;
2065 PULONG Pte;
2066 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2067 Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2068 if (Page & 1024)
2069 {
2070 for (i = Page % 1024; i < 1024; i++, Pte++)
2071 {
2072 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2073 {
2074 break;
2075 }
2076 }
2077 if (i >= 1024)
2078 {
2079 Pte = ADDR_TO_PTE(HYPERSPACE);
2080 for (i = 0; i < Page % 1024; i++, Pte++)
2081 {
2082 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2083 {
2084 break;
2085 }
2086 }
2087 if (i >= Page % 1024)
2088 {
2089 ASSERT(FALSE);
2090 }
2091 }
2092 }
2093 else
2094 {
2095 for (i = Page % 1024; (LONG)i >= 0; i--, Pte--)
2096 {
2097 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2098 {
2099 break;
2100 }
2101 }
2102 if ((LONG)i < 0)
2103 {
2104 Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
2105 for (i = 1023; i > Page % 1024; i--, Pte--)
2106 {
2107 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2108 {
2109 break;
2110 }
2111 }
2112 if (i <= Page % 1024)
2113 {
2114 ASSERT(FALSE);
2115 }
2116 }
2117 }
2118 }
2119 Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE);
2120 __invlpg(Address);
2121 return Address;
2122 }
2123
2124 PFN_NUMBER
2125 NTAPI
2126 MmChangeHyperspaceMapping(PVOID Address, PFN_NUMBER NewPage)
2127 {
2128 PFN_NUMBER Pfn;
2129 ASSERT (IS_HYPERSPACE(Address));
2130 if (Ke386Pae)
2131 {
2132 ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE;
2133 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2134 Pfn = PAE_PTE_TO_PFN(Entry);
2135 }
2136 else
2137 {
2138 ULONG Entry;
2139 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE);
2140 Pfn = PTE_TO_PFN(Entry);
2141 }
2142 __invlpg(Address);
2143 return Pfn;
2144 }
2145
2146 PFN_NUMBER
2147 NTAPI
2148 MmDeleteHyperspaceMapping(PVOID Address)
2149 {
2150 PFN_NUMBER Pfn;
2151 ASSERT (IS_HYPERSPACE(Address));
2152 if (Ke386Pae)
2153 {
2154 ULONGLONG Entry = 0LL;
2155 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2156 Pfn = PAE_PTE_TO_PFN(Entry);
2157 }
2158 else
2159 {
2160 ULONG Entry;
2161 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0);
2162 Pfn = PTE_TO_PFN(Entry);
2163 }
2164 __invlpg(Address);
2165 return Pfn;
2166 }
2167
2168 VOID
2169 NTAPI
2170 MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
2171 {
2172 ULONG StartOffset, EndOffset, Offset;
2173
2174 if (Address < MmSystemRangeStart)
2175 {
2176 ASSERT(FALSE);
2177 }
2178 if (Ke386Pae)
2179 {
2180 PULONGLONG PageDirTable;
2181 PULONGLONG Pde;
2182 ULONGLONG ZeroPde = 0LL;
2183 ULONG i;
2184
2185 for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++)
2186 {
2187 if (i == PAE_ADDR_TO_PDTE_OFFSET(Address))
2188 {
2189 StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
2190 }
2191 else
2192 {
2193 StartOffset = 0;
2194 }
2195 if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)))
2196 {
2197 EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2198 }
2199 else
2200 {
2201 EndOffset = 511;
2202 }
2203
2204 if (Process != NULL && Process != PsGetCurrentProcess())
2205 {
2206 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
2207 Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i]));
2208 MmDeleteHyperspaceMapping(PageDirTable);
2209 }
2210 else
2211 {
2212 Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512;
2213 }
2214
2215 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2216 {
2217 if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4)
2218 {
2219 (void)ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde);
2220 }
2221 }
2222 MmUnmapPageTable((PULONG)Pde);
2223 }
2224 }
2225 else
2226 {
2227 PULONG Pde;
2228 StartOffset = ADDR_TO_PDE_OFFSET(Address);
2229 EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2230
2231 if (Process != NULL && Process != PsGetCurrentProcess())
2232 {
2233 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
2234 }
2235 else
2236 {
2237 Pde = (PULONG)PAGEDIRECTORY_MAP;
2238 }
2239 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2240 {
2241 if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP))
2242 {
2243 (void)InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0);
2244 }
2245 }
2246 if (Pde != (PULONG)PAGEDIRECTORY_MAP)
2247 {
2248 MmDeleteHyperspaceMapping(Pde);
2249 }
2250 }
2251 }
2252
2253 VOID
2254 INIT_FUNCTION
2255 NTAPI
2256 MmInitGlobalKernelPageDirectory(VOID)
2257 {
2258 ULONG i;
2259
2260 DPRINT("MmInitGlobalKernelPageDirectory()\n");
2261
2262 if (Ke386Pae)
2263 {
2264 PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
2265 for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++)
2266 {
2267 if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
2268 !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
2269 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
2270 {
2271 (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
2272 if (Ke386GlobalPagesEnabled)
2273 {
2274 MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
2275 CurrentPageDirectory[i] |= PA_GLOBAL;
2276 }
2277 }
2278 }
2279 }
2280 else
2281 {
2282 PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
2283 for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++)
2284 {
2285 if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
2286 i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
2287 0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
2288 {
2289 MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
2290 if (Ke386GlobalPagesEnabled)
2291 {
2292 MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
2293 CurrentPageDirectory[i] |= PA_GLOBAL;
2294 }
2295 }
2296 }
2297 }
2298 }
2299
2300 ULONG
2301 NTAPI
2302 MiGetUserPageDirectoryCount(VOID)
2303 {
2304 return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart);
2305 }
2306
2307 VOID
2308 INIT_FUNCTION
2309 NTAPI
2310 MiInitPageDirectoryMap(VOID)
2311 {
2312 MEMORY_AREA* kernel_map_desc = NULL;
2313 MEMORY_AREA* hyperspace_desc = NULL;
2314 PVOID BaseAddress;
2315 NTSTATUS Status;
2316
2317 DPRINT("MiInitPageDirectoryMap()\n");
2318
2319 BaseAddress = (PVOID)PAGETABLE_MAP;
2320 Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
2321 MEMORY_AREA_SYSTEM,
2322 &BaseAddress,
2323 Ke386Pae ? 0x800000 : 0x400000,
2324 PAGE_READWRITE,
2325 &kernel_map_desc,
2326 TRUE,
2327 0,
2328 PAGE_SIZE);
2329 if (!NT_SUCCESS(Status))
2330 {
2331 ASSERT(FALSE);
2332 }
2333 BaseAddress = (PVOID)HYPERSPACE;
2334 Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
2335 MEMORY_AREA_SYSTEM,
2336 &BaseAddress,
2337 0x400000,
2338 PAGE_READWRITE,
2339 &hyperspace_desc,
2340 TRUE,
2341 0,
2342 PAGE_SIZE);
2343 if (!NT_SUCCESS(Status))
2344 {
2345 ASSERT(FALSE);
2346 }
2347 }
2348
2349 /* EOF */
A free Windows-compatible Operating System