projects / reactos.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
If we delete a page table, we have also to clear the entry in the page directory...
[reactos.git] / reactos / ntoskrnl / mm / i386 / page.c
1 /* $Id$
2 *
3 * COPYRIGHT: See COPYING in the top level directory
4 * PROJECT: ReactOS kernel
5 * FILE: ntoskrnl/mm/i386/page.c
6 * PURPOSE: Low level memory managment manipulation
7 *
8 * PROGRAMMERS: David Welch (welch@cwcom.net)
9 */
10
11 /* INCLUDES ***************************************************************/
12
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <internal/debug.h>
16
17 /* GLOBALS *****************************************************************/
18
19 #define PA_BIT_PRESENT (0)
20 #define PA_BIT_READWRITE (1)
21 #define PA_BIT_USER (2)
22 #define PA_BIT_WT (3)
23 #define PA_BIT_CD (4)
24 #define PA_BIT_ACCESSED (5)
25 #define PA_BIT_DIRTY (6)
26 #define PA_BIT_GLOBAL (8)
27
28 #define PA_PRESENT (1 << PA_BIT_PRESENT)
29 #define PA_READWRITE (1 << PA_BIT_READWRITE)
30 #define PA_USER (1 << PA_BIT_USER)
31 #define PA_DIRTY (1 << PA_BIT_DIRTY)
32 #define PA_WT (1 << PA_BIT_WT)
33 #define PA_CD (1 << PA_BIT_CD)
34 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
35 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
36
37 #define PAGETABLE_MAP (0xc0000000)
38 #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024)))
39
40 #define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512)))
41
42 #define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000)
43 #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
44
45 ULONG MmGlobalKernelPageDirectory[1024];
46 ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];
47
48 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
49 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
50
51 #define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
52 #define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
53
54 #if defined(__GNUC__)
55 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
56 #else
57 __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage)
58 {
59 LARGE_INTEGER dummy;
60 dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage));
61 return dummy;
62 }
63 #endif
64
65 extern BOOLEAN Ke386Pae;
66 extern BOOLEAN Ke386NoExecute;
67 extern BOOLEAN Ke386GlobalPagesEnabled;
68
69 /* FUNCTIONS ***************************************************************/
70
71 BOOLEAN MmUnmapPageTable(PULONG Pt);
72
73 VOID
74 STDCALL
75 MiFlushTlbIpiRoutine(PVOID Address)
76 {
77 if (Address == (PVOID)0xffffffff)
78 {
79 KeFlushCurrentTb();
80 }
81 else if (Address == (PVOID)0xfffffffe)
82 {
83 FLUSH_TLB;
84 }
85 else
86 {
87 FLUSH_TLB_ONE(Address);
88 }
89 }
90
91 VOID
92 MiFlushTlb(PULONG Pt, PVOID Address)
93 {
94 #ifdef CONFIG_SMP
95 if (Pt)
96 {
97 MmUnmapPageTable(Pt);
98 }
99 if (KeNumberProcessors>1)
100 {
101 KeIpiGenericCall(MiFlushTlbIpiRoutine, Address);
102 }
103 else
104 {
105 MiFlushTlbIpiRoutine(Address);
106 }
107 #else
108 if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart)
109 {
110 FLUSH_TLB_ONE(Address);
111 }
112 #endif
113 }
114
115
116
117 PULONG
118 MmGetPageDirectory(VOID)
119 {
120 unsigned int page_dir=0;
121 Ke386GetPageTableDirectory(page_dir);
122 return((PULONG)page_dir);
123 }
124
125 static ULONG
126 ProtectToPTE(ULONG flProtect)
127 {
128 ULONG Attributes = 0;
129
130 if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
131 {
132 Attributes = 0;
133 }
134 else if (flProtect & PAGE_IS_WRITABLE)
135 {
136 Attributes = PA_PRESENT | PA_READWRITE;
137 }
138 else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
139 {
140 Attributes = PA_PRESENT;
141 }
142 else
143 {
144 DPRINT1("Unknown main protection type.\n");
145 KEBUGCHECK(0);
146 }
147 if (Ke386NoExecute &&
148 !(flProtect & PAGE_IS_EXECUTABLE))
149 {
150 Attributes = Attributes | 0x80000000;
151 }
152
153 if (flProtect & PAGE_SYSTEM)
154 {
155 }
156 else
157 {
158 Attributes = Attributes | PA_USER;
159 }
160 if (flProtect & PAGE_NOCACHE)
161 {
162 Attributes = Attributes | PA_CD;
163 }
164 if (flProtect & PAGE_WRITETHROUGH)
165 {
166 Attributes = Attributes | PA_WT;
167 }
168 return(Attributes);
169 }
170
171 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
172
173 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
174 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
175 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
176
177 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
178
179 #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
180
181
182 #define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE))
183
184 #define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
185 ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
186 #define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
187
188
189 #define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
190
191 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
192
193 #define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
194
195 #define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
196
197
198 NTSTATUS Mmi386ReleaseMmInfo(PEPROCESS Process)
199 {
200 PUSHORT LdtDescriptor;
201 ULONG LdtBase;
202 ULONG i, j;
203
204 DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process);
205
206 LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor;
207 LdtBase = LdtDescriptor[1] |
208 ((LdtDescriptor[2] & 0xff) << 16) |
209 ((LdtDescriptor[3] & ~0xff) << 16);
210
211 DPRINT("LdtBase: %x\n", LdtBase);
212
213 if (LdtBase)
214 {
215 ExFreePool((PVOID) LdtBase);
216 }
217
218 if (Ke386Pae)
219 {
220 PULONGLONG PageDirTable;
221 PULONGLONG PageDir;
222 PULONGLONG Pde;
223 ULONG k;
224
225 PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
226 for (i = 0; i < 4; i++)
227 {
228 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i]));
229 if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart))
230 {
231 for (j = 0; j < 512; j++)
232 {
233 if (PageDir[j] != 0LL)
234 {
235 DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
236 Process->UniqueProcessId,
237 (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1,
238 Process->AddressSpace.PageTableRefCountTable[i*512 + j]);
239 Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j]));
240 for (k = 0; k < 512; k++)
241 {
242 if(Pde[k] != 0)
243 {
244 if (Pde[k] & PA_PRESENT)
245 {
246 DPRINT1("Page at %08x is not freed\n",
247 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
248 }
249 else
250 {
251 DPRINT1("Swapentry %x at %x is not freed\n",
252 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
253 }
254 }
255 }
256 MmDeleteHyperspaceMapping(Pde);
257 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j]));
258 }
259 }
260 }
261 if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE))
262 {
263 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)]));
264 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1]));
265 }
266 MmDeleteHyperspaceMapping(PageDir);
267 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i]));
268 }
269 MmDeleteHyperspaceMapping((PVOID)PageDirTable);
270 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
271 }
272 else
273 {
274 PULONG Pde;
275 PULONG PageDir;
276 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
277 for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++)
278 {
279 if (PageDir[i] != 0)
280 {
281 DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
282 i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1,
283 Process->AddressSpace.PageTableRefCountTable[i]);
284 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i]));
285 for (j = 0; j < 1024; j++)
286 {
287 if(Pde[j] != 0)
288 {
289 if (Pde[j] & PA_PRESENT)
290 {
291 DPRINT1("Page at %08x is not freed\n",
292 i * 4 * 1024 * 1024 + j * PAGE_SIZE);
293 }
294 else
295 {
296 DPRINT1("Swapentry %x at %x is not freed\n",
297 Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE);
298 }
299 }
300 }
301 MmDeleteHyperspaceMapping(Pde);
302 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i]));
303 }
304 }
305 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)]));
306 MmDeleteHyperspaceMapping(PageDir);
307 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
308 }
309
310 #if defined(__GNUC__)
311
312 Process->Pcb.DirectoryTableBase.QuadPart = 0LL;
313 #else
314
315 Process->Pcb.DirectoryTableBase.QuadPart = 0;
316 #endif
317
318 DPRINT("Finished Mmi386ReleaseMmInfo()\n");
319 return(STATUS_SUCCESS);
320 }
321
322 NTSTATUS
323 STDCALL
324 MmCopyMmInfo(PEPROCESS Src,
325 PEPROCESS Dest,
326 PPHYSICAL_ADDRESS DirectoryTableBase)
327 {
328 NTSTATUS Status;
329 ULONG i, j;
330 PFN_TYPE Pfn[7];
331 ULONG Count;
332
333 DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", Src, Dest);
334
335 Count = Ke386Pae ? 7 : 2;
336
337 for (i = 0; i < Count; i++)
338 {
339 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
340 if (!NT_SUCCESS(Status))
341 {
342 for (j = 0; j < i; j++)
343 {
344 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
345 }
346 return Status;
347 }
348 }
349
350 if (Ke386Pae)
351 {
352 PULONGLONG PageDirTable;
353 PULONGLONG PageDir;
354
355 PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
356 for (i = 0; i < 4; i++)
357 {
358 PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
359 }
360 MmDeleteHyperspaceMapping(PageDirTable);
361 for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++)
362 {
363 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
364 memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
365 if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
366 {
367 for (j = 0; j < 4; j++)
368 {
369 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
370 }
371 }
372 if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
373 {
374 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
375 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
376 }
377 MmDeleteHyperspaceMapping(PageDir);
378 }
379 }
380 else
381 {
382 PULONG PageDirectory;
383 PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);
384
385 memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
386 MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
387 (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG));
388
389 DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
390 PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
391 PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;
392
393 MmDeleteHyperspaceMapping(PageDirectory);
394 }
395
396 DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]);
397 DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart);
398 return(STATUS_SUCCESS);
399 }
400
401 VOID MmDeletePageTable(PEPROCESS Process, PVOID Address)
402 {
403 PEPROCESS CurrentProcess = PsGetCurrentProcess();
404
405 if (Process != NULL && Process != CurrentProcess)
406 {
407 KeAttachProcess(&Process->Pcb);
408 }
409
410 if (Ke386Pae)
411 {
412 ULONGLONG ZeroPde = 0LL;
413 ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde);
414 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
415 }
416 else
417 {
418 *(ADDR_TO_PDE(Address)) = 0;
419 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
420 }
421 if (Address >= MmSystemRangeStart)
422 {
423 KEBUGCHECK(0);
424 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
425 }
426 if (Process != NULL && Process != CurrentProcess)
427 {
428 KeDetachProcess();
429 }
430 }
431
432 VOID MmFreePageTable(PEPROCESS Process, PVOID Address)
433 {
434 PEPROCESS CurrentProcess = PsGetCurrentProcess();
435 ULONG i;
436 PFN_TYPE Pfn;
437
438 DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
439 if (Process != NULL && Process != CurrentProcess)
440 {
441 KeAttachProcess(&Process->Pcb);
442 }
443 if (Ke386Pae)
444 {
445 PULONGLONG PageTable;
446 ULONGLONG ZeroPte = 0LL;
447 PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
448 for (i = 0; i < 512; i++)
449 {
450 if (PageTable[i] != 0LL)
451 {
452 DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
453 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
454 KEBUGCHECK(0);
455 }
456 }
457 Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
458 ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
459 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
460 }
461 else
462 {
463 PULONG PageTable;
464 PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
465 for (i = 0; i < 1024; i++)
466 {
467 if (PageTable[i] != 0)
468 {
469 DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
470 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
471 KEBUGCHECK(0);
472 }
473 }
474 Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
475 *(ADDR_TO_PDE(Address)) = 0;
476 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
477 }
478
479 if (Address >= MmSystemRangeStart)
480 {
481 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
482 KEBUGCHECK(0);
483 }
484 else
485 {
486 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
487 }
488 if (Process != NULL && Process != CurrentProcess)
489 {
490 KeDetachProcess();
491 }
492 }
493
494 static PULONGLONG
495 MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
496 {
497 NTSTATUS Status;
498 PFN_TYPE Pfn;
499 ULONGLONG Entry;
500 ULONGLONG ZeroEntry = 0LL;
501 PULONGLONG Pt;
502 PULONGLONG PageDir;
503 PULONGLONG PageDirTable;
504
505 DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
506 Process, Address, Create);
507 if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000))
508 {
509 KEBUGCHECK(0);
510 }
511 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
512 {
513 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
514 if (PageDirTable == NULL)
515 {
516 KEBUGCHECK(0);
517 }
518 PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
519 MmDeleteHyperspaceMapping(PageDirTable);
520 if (PageDir == NULL)
521 {
522 KEBUGCHECK(0);
523 }
524 PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
525 Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
526 if (Entry == 0LL)
527 {
528 if (Create == FALSE)
529 {
530 MmDeleteHyperspaceMapping(PageDir);
531 return NULL;
532 }
533 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
534 if (!NT_SUCCESS(Status))
535 {
536 KEBUGCHECK(0);
537 }
538 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
539 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
540 if (Entry != 0LL)
541 {
542 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
543 Pfn = PAE_PTE_TO_PFN(Entry);
544 }
545 }
546 else
547 {
548 Pfn = PAE_PTE_TO_PFN(Entry);
549 }
550 MmDeleteHyperspaceMapping(PageDir);
551 Pt = MmCreateHyperspaceMapping(Pfn);
552 if (Pt == NULL)
553 {
554 KEBUGCHECK(0);
555 }
556 return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
557 }
558 PageDir = PAE_ADDR_TO_PDE(Address);
559 if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
560 {
561 if (Address >= MmSystemRangeStart)
562 {
563 if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
564 {
565 if (Create == FALSE)
566 {
567 return NULL;
568 }
569 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
570 if (!NT_SUCCESS(Status))
571 {
572 KEBUGCHECK(0);
573 }
574 Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
575 if (Ke386GlobalPagesEnabled)
576 {
577 Entry |= PA_GLOBAL;
578 }
579 if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
580 {
581 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
582 }
583 }
584 ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
585 }
586 else
587 {
588 if (Create == FALSE)
589 {
590 return NULL;
591 }
592 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
593 if (!NT_SUCCESS(Status))
594 {
595 KEBUGCHECK(0);
596 }
597 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
598 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
599 if (Entry != 0LL)
600 {
601 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
602 }
603 }
604 }
605 return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
606 }
607
608 static PULONG
609 MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
610 {
611 ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
612 NTSTATUS Status;
613 PFN_TYPE Pfn;
614 ULONG Entry;
615 PULONG Pt, PageDir;
616
617 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
618 {
619 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
620 if (PageDir == NULL)
621 {
622 KEBUGCHECK(0);
623 }
624 if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
625 {
626 if (Create == FALSE)
627 {
628 MmDeleteHyperspaceMapping(PageDir);
629 return NULL;
630 }
631 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
632 if (!NT_SUCCESS(Status) || Pfn == 0)
633 {
634 KEBUGCHECK(0);
635 }
636 Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
637 if (Entry != 0)
638 {
639 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
640 Pfn = PTE_TO_PFN(Entry);
641 }
642 }
643 else
644 {
645 Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
646 }
647 MmDeleteHyperspaceMapping(PageDir);
648 Pt = MmCreateHyperspaceMapping(Pfn);
649 if (Pt == NULL)
650 {
651 KEBUGCHECK(0);
652 }
653 return Pt + ADDR_TO_PTE_OFFSET(Address);
654 }
655 PageDir = ADDR_TO_PDE(Address);
656 if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
657 {
658 if (Address >= MmSystemRangeStart)
659 {
660 if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
661 {
662 if (Create == FALSE)
663 {
664 return NULL;
665 }
666 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
667 if (!NT_SUCCESS(Status) || Pfn == 0)
668 {
669 KEBUGCHECK(0);
670 }
671 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
672 if (Ke386GlobalPagesEnabled)
673 {
674 Entry |= PA_GLOBAL;
675 }
676 if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
677 {
678 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
679 }
680 }
681 InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
682 }
683 else
684 {
685 if (Create == FALSE)
686 {
687 return NULL;
688 }
689 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
690 if (!NT_SUCCESS(Status) || Pfn == 0)
691 {
692 KEBUGCHECK(0);
693 }
694 Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
695 if (Entry != 0)
696 {
697 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
698 }
699 }
700 }
701 return (PULONG)ADDR_TO_PTE(Address);
702 }
703
704 BOOLEAN MmUnmapPageTable(PULONG Pt)
705 {
706 if (Ke386Pae)
707 {
708 if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
709 {
710 return TRUE;
711 }
712 }
713 else
714 {
715 if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
716 {
717 return TRUE;
718 }
719 }
720 if (Pt)
721 {
722 MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
723 }
724 return FALSE;
725 }
726
727 static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
728 {
729 ULONGLONG Pte;
730 PULONGLONG Pt;
731
732 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
733 if (Pt)
734 {
735 Pte = *Pt;
736 MmUnmapPageTable((PULONG)Pt);
737 return Pte;
738 }
739 return 0;
740 }
741
742 static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
743 {
744 ULONG Pte;
745 PULONG Pt;
746
747 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
748 if (Pt)
749 {
750 Pte = *Pt;
751 MmUnmapPageTable(Pt);
752 return Pte;
753 }
754 return 0;
755 }
756
757 PFN_TYPE
758 MmGetPfnForProcess(PEPROCESS Process,
759 PVOID Address)
760 {
761
762 if (Ke386Pae)
763 {
764 ULONGLONG Entry;
765 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
766 if (!(Entry & PA_PRESENT))
767 {
768 return 0;
769 }
770 return(PAE_PTE_TO_PFN(Entry));
771 }
772 else
773 {
774 ULONG Entry;
775 Entry = MmGetPageEntryForProcess(Process, Address);
776 if (!(Entry & PA_PRESENT))
777 {
778 return 0;
779 }
780 return(PTE_TO_PFN(Entry));
781 }
782 }
783
784 VOID
785 MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOL* WasDirty, PPFN_TYPE Page)
786 /*
787 * FUNCTION: Delete a virtual mapping
788 */
789 {
790 BOOLEAN WasValid;
791 if (Ke386Pae)
792 {
793 ULONGLONG Pte;
794 ULONGLONG tmpPte;
795 PULONGLONG Pt;
796
797 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
798 if (Pt == NULL)
799 {
800 KEBUGCHECK(0);
801 }
802 /*
803 * Atomically disable the present bit and get the old value.
804 */
805 do
806 {
807 Pte = *Pt;
808 tmpPte = Pte & ~PA_PRESENT;
809 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
810
811 MiFlushTlb((PULONG)Pt, Address);
812 WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE;
813 if (!WasValid)
814 {
815 KEBUGCHECK(0);
816 }
817
818 /*
819 * Return some information to the caller
820 */
821 if (WasDirty != NULL)
822 {
823 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
824 }
825 if (Page != NULL)
826 {
827 *Page = PAE_PTE_TO_PFN(Pte);
828 }
829 }
830 else
831 {
832 ULONG Pte;
833 PULONG Pt;
834
835 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
836 if (Pt == NULL)
837 {
838 KEBUGCHECK(0);
839 }
840 /*
841 * Atomically disable the present bit and get the old value.
842 */
843 do
844 {
845 Pte = *Pt;
846 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte));
847
848 MiFlushTlb(Pt, Address);
849 WasValid = (PAGE_MASK(Pte) != 0);
850 if (!WasValid)
851 {
852 KEBUGCHECK(0);
853 }
854
855 /*
856 * Return some information to the caller
857 */
858 if (WasDirty != NULL)
859 {
860 *WasDirty = Pte & PA_DIRTY;
861 }
862 if (Page != NULL)
863 {
864 *Page = PTE_TO_PFN(Pte);
865 }
866 }
867 }
868
869 VOID
870 MmRawDeleteVirtualMapping(PVOID Address)
871 {
872 if (Ke386Pae)
873 {
874 PULONGLONG Pt;
875 ULONGLONG ZeroPte = 0LL;
876 Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE);
877 if (Pt)
878 {
879 /*
880 * Set the entry to zero
881 */
882 ExfpInterlockedExchange64UL(Pt, &ZeroPte);
883 MiFlushTlb((PULONG)Pt, Address);
884 }
885 }
886 else
887 {
888 PULONG Pt;
889
890 Pt = MmGetPageTableForProcess(NULL, Address, FALSE);
891 if (Pt && *Pt)
892 {
893 /*
894 * Set the entry to zero
895 */
896 InterlockedExchangeUL(Pt, 0);
897 MiFlushTlb(Pt, Address);
898 }
899 }
900 }
901
902 VOID
903 MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOL FreePage,
904 BOOL* WasDirty, PPFN_TYPE Page)
905 /*
906 * FUNCTION: Delete a virtual mapping
907 */
908 {
909 BOOLEAN WasValid = FALSE;
910 PFN_TYPE Pfn;
911
912 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
913 Process, Address, FreePage, WasDirty, Page);
914 if (Ke386Pae)
915 {
916 ULONGLONG Pte;
917 PULONGLONG Pt;
918
919 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
920 if (Pt == NULL)
921 {
922 if (WasDirty != NULL)
923 {
924 *WasDirty = FALSE;
925 }
926 if (Page != NULL)
927 {
928 *Page = 0;
929 }
930 return;
931 }
932
933 /*
934 * Atomically set the entry to zero and get the old value.
935 */
936 Pte = 0LL;
937 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
938
939 MiFlushTlb((PULONG)Pt, Address);
940
941 WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
942 if (WasValid)
943 {
944 Pfn = PAE_PTE_TO_PFN(Pte);
945 MmMarkPageUnmapped(Pfn);
946 }
947 else
948 {
949 Pfn = 0;
950 }
951
952 if (FreePage && WasValid)
953 {
954 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
955 }
956
957 /*
958 * Return some information to the caller
959 */
960 if (WasDirty != NULL)
961 {
962 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
963 }
964 if (Page != NULL)
965 {
966 *Page = Pfn;
967 }
968 }
969 else
970 {
971 ULONG Pte;
972 PULONG Pt;
973
974 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
975
976 if (Pt == NULL)
977 {
978 if (WasDirty != NULL)
979 {
980 *WasDirty = FALSE;
981 }
982 if (Page != NULL)
983 {
984 *Page = 0;
985 }
986 return;
987 }
988
989 /*
990 * Atomically set the entry to zero and get the old value.
991 */
992 Pte = InterlockedExchangeUL(Pt, 0);
993
994 MiFlushTlb(Pt, Address);
995
996 WasValid = (PAGE_MASK(Pte) != 0);
997 if (WasValid)
998 {
999 Pfn = PTE_TO_PFN(Pte);
1000 MmMarkPageUnmapped(Pfn);
1001 }
1002 else
1003 {
1004 Pfn = 0;
1005 }
1006
1007 if (FreePage && WasValid)
1008 {
1009 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
1010 }
1011
1012 /*
1013 * Return some information to the caller
1014 */
1015 if (WasDirty != NULL)
1016 {
1017 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
1018 }
1019 if (Page != NULL)
1020 {
1021 *Page = Pfn;
1022 }
1023 }
1024 /*
1025 * Decrement the reference count for this page table.
1026 */
1027 if (Process != NULL && WasValid &&
1028 Process->AddressSpace.PageTableRefCountTable != NULL &&
1029 Address < MmSystemRangeStart)
1030 {
1031 PUSHORT Ptrc;
1032 ULONG Idx;
1033
1034 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1035 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1036
1037 Ptrc[Idx]--;
1038 if (Ptrc[Idx] == 0)
1039 {
1040 MmFreePageTable(Process, Address);
1041 }
1042 }
1043 }
1044
1045 VOID
1046 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
1047 SWAPENTRY* SwapEntry)
1048 /*
1049 * FUNCTION: Delete a virtual mapping
1050 */
1051 {
1052 if (Ke386Pae)
1053 {
1054 ULONGLONG Pte;
1055 PULONGLONG Pt;
1056
1057 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1058 if (Pt == NULL)
1059 {
1060 *SwapEntry = 0;
1061 return;
1062 }
1063
1064 /*
1065 * Atomically set the entry to zero and get the old value.
1066 */
1067 Pte = 0LL;
1068 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1069
1070 MiFlushTlb((PULONG)Pt, Address);
1071
1072 /*
1073 * Decrement the reference count for this page table.
1074 */
1075 if (Process != NULL && Pte &&
1076 Process->AddressSpace.PageTableRefCountTable != NULL &&
1077 Address < MmSystemRangeStart)
1078 {
1079 PUSHORT Ptrc;
1080
1081 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1082
1083 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
1084 if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
1085 {
1086 MmFreePageTable(Process, Address);
1087 }
1088 }
1089
1090
1091 /*
1092 * Return some information to the caller
1093 */
1094 *SwapEntry = Pte >> 1;
1095 }
1096 else
1097 {
1098 ULONG Pte;
1099 PULONG Pt;
1100
1101 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1102
1103 if (Pt == NULL)
1104 {
1105 *SwapEntry = 0;
1106 return;
1107 }
1108
1109 /*
1110 * Atomically set the entry to zero and get the old value.
1111 */
1112 Pte = InterlockedExchangeUL(Pt, 0);
1113
1114 MiFlushTlb(Pt, Address);
1115
1116 /*
1117 * Decrement the reference count for this page table.
1118 */
1119 if (Process != NULL && Pte &&
1120 Process->AddressSpace.PageTableRefCountTable != NULL &&
1121 Address < MmSystemRangeStart)
1122 {
1123 PUSHORT Ptrc;
1124
1125 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1126
1127 Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
1128 if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
1129 {
1130 MmFreePageTable(Process, Address);
1131 }
1132 }
1133
1134
1135 /*
1136 * Return some information to the caller
1137 */
1138 *SwapEntry = Pte >> 1;
1139 }
1140 }
1141
1142 BOOLEAN
1143 Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
1144 {
1145 if (Ke386Pae)
1146 {
1147 PULONGLONG Pt;
1148 PULONGLONG Pde;
1149 Pde = PAE_ADDR_TO_PDE(PAddress);
1150 if (*Pde == 0LL)
1151 {
1152 Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
1153 #if 0
1154 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1155 FLASH_TLB_ONE(PAddress);
1156 #endif
1157 if (Pt != NULL)
1158 {
1159 return TRUE;
1160 }
1161 }
1162 }
1163 else
1164 {
1165 PULONG Pt, Pde;
1166 Pde = ADDR_TO_PDE(PAddress);
1167 if (*Pde == 0)
1168 {
1169 Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
1170 #if 0
1171 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1172 FLASH_TLB_ONE(PAddress);
1173 #endif
1174 if (Pt != NULL)
1175 {
1176 return TRUE;
1177 }
1178 }
1179 }
1180 return(FALSE);
1181 }
1182
1183 BOOLEAN MmIsDirtyPage(PEPROCESS Process, PVOID Address)
1184 {
1185 if (Ke386Pae)
1186 {
1187 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1188 }
1189 else
1190 {
1191 return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1192 }
1193 }
1194
1195 BOOLEAN
1196 MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address)
1197 {
1198 if (Address < MmSystemRangeStart && Process == NULL)
1199 {
1200 DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1201 KEBUGCHECK(0);
1202 }
1203 if (Ke386Pae)
1204 {
1205 PULONGLONG Pt;
1206 ULONGLONG Pte;
1207 ULONGLONG tmpPte;
1208
1209 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1210 if (Pt == NULL)
1211 {
1212 KEBUGCHECK(0);
1213 }
1214
1215 do
1216 {
1217 Pte = *Pt;
1218 tmpPte = Pte & ~PA_ACCESSED;
1219 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1220
1221 if (Pte & PA_ACCESSED)
1222 {
1223 MiFlushTlb((PULONG)Pt, Address);
1224 return TRUE;
1225 }
1226 else
1227 {
1228 MmUnmapPageTable((PULONG)Pt);
1229 return FALSE;
1230 }
1231 }
1232 else
1233 {
1234 PULONG Pt;
1235 ULONG Pte;
1236
1237 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1238 if (Pt == NULL)
1239 {
1240 KEBUGCHECK(0);
1241 }
1242
1243 do
1244 {
1245 Pte = *Pt;
1246 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte));
1247
1248 if (Pte & PA_ACCESSED)
1249 {
1250 MiFlushTlb(Pt, Address);
1251 return TRUE;
1252 }
1253 else
1254 {
1255 MmUnmapPageTable(Pt);
1256 return FALSE;
1257 }
1258 }
1259 }
1260
1261 VOID MmSetCleanPage(PEPROCESS Process, PVOID Address)
1262 {
1263 if (Address < MmSystemRangeStart && Process == NULL)
1264 {
1265 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1266 KEBUGCHECK(0);
1267 }
1268 if (Ke386Pae)
1269 {
1270 PULONGLONG Pt;
1271 ULONGLONG Pte;
1272 ULONGLONG tmpPte;
1273
1274 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1275
1276 if (Pt == NULL)
1277 {
1278 KEBUGCHECK(0);
1279 }
1280
1281 do
1282 {
1283 Pte = *Pt;
1284 tmpPte = Pte & ~PA_DIRTY;
1285 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1286
1287 if (Pte & PA_DIRTY)
1288 {
1289 MiFlushTlb((PULONG)Pt, Address);
1290 }
1291 else
1292 {
1293 MmUnmapPageTable((PULONG)Pt);
1294 }
1295 }
1296 else
1297 {
1298 PULONG Pt;
1299 ULONG Pte;
1300
1301 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1302
1303 if (Pt == NULL)
1304 {
1305 KEBUGCHECK(0);
1306 }
1307
1308 do
1309 {
1310 Pte = *Pt;
1311 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));
1312
1313 if (Pte & PA_DIRTY)
1314 {
1315 MiFlushTlb(Pt, Address);
1316 }
1317 else
1318 {
1319 MmUnmapPageTable(Pt);
1320 }
1321 }
1322 }
1323
1324 VOID MmSetDirtyPage(PEPROCESS Process, PVOID Address)
1325 {
1326 if (Address < MmSystemRangeStart && Process == NULL)
1327 {
1328 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1329 KEBUGCHECK(0);
1330 }
1331 if (Ke386Pae)
1332 {
1333 PULONGLONG Pt;
1334 ULONGLONG Pte;
1335 ULONGLONG tmpPte;
1336
1337 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1338 if (Pt == NULL)
1339 {
1340 KEBUGCHECK(0);
1341 }
1342
1343 do
1344 {
1345 Pte = *Pt;
1346 tmpPte = Pte | PA_DIRTY;
1347 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1348 if (!(Pte & PA_DIRTY))
1349 {
1350 MiFlushTlb((PULONG)Pt, Address);
1351 }
1352 else
1353 {
1354 MmUnmapPageTable((PULONG)Pt);
1355 }
1356 }
1357 else
1358 {
1359 PULONG Pt;
1360 ULONG Pte;
1361
1362 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1363 if (Pt == NULL)
1364 {
1365 KEBUGCHECK(0);
1366 }
1367
1368 do
1369 {
1370 Pte = *Pt;
1371 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
1372 if (!(Pte & PA_DIRTY))
1373 {
1374 MiFlushTlb(Pt, Address);
1375 }
1376 else
1377 {
1378 MmUnmapPageTable(Pt);
1379 }
1380 }
1381 }
1382
1383 VOID MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
1384 {
1385 if (Ke386Pae)
1386 {
1387 PULONGLONG Pt;
1388 ULONGLONG Pte;
1389 ULONGLONG tmpPte;
1390
1391 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1392 if (Pt == NULL)
1393 {
1394 KEBUGCHECK(0);
1395 }
1396
1397 do
1398 {
1399 Pte = *Pt;
1400 tmpPte = Pte | PA_PRESENT;
1401 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1402 if (!(Pte & PA_PRESENT))
1403 {
1404 MiFlushTlb((PULONG)Pt, Address);
1405 }
1406 else
1407 {
1408 MmUnmapPageTable((PULONG)Pt);
1409 }
1410 }
1411 else
1412 {
1413 PULONG Pt;
1414 ULONG Pte;
1415
1416 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1417 if (Pt == NULL)
1418 {
1419 KEBUGCHECK(0);
1420 }
1421
1422 do
1423 {
1424 Pte = *Pt;
1425 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte));
1426 if (!(Pte & PA_PRESENT))
1427 {
1428 MiFlushTlb(Pt, Address);
1429 }
1430 else
1431 {
1432 MmUnmapPageTable(Pt);
1433 }
1434 }
1435 }
1436
1437 BOOLEAN MmIsPagePresent(PEPROCESS Process, PVOID Address)
1438 {
1439 if (Ke386Pae)
1440 {
1441 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1442 }
1443 else
1444 {
1445 return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1446 }
1447 }
1448
1449 BOOLEAN MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
1450 {
1451 if (Ke386Pae)
1452 {
1453 ULONGLONG Entry;
1454 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1455 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1456 }
1457 else
1458 {
1459 ULONG Entry;
1460 Entry = MmGetPageEntryForProcess(Process, Address);
1461 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1462 }
1463 }
1464
1465 NTSTATUS
1466 MmCreateVirtualMappingForKernel(PVOID Address,
1467 ULONG flProtect,
1468 PPFN_TYPE Pages,
1469 ULONG PageCount)
1470 {
1471 ULONG Attributes;
1472 ULONG i;
1473 PVOID Addr;
1474 ULONG PdeOffset, oldPdeOffset;
1475 BOOLEAN NoExecute = FALSE;
1476
1477 DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1478 Address, flProtect, Pages, PageCount);
1479
1480 if (Address < MmSystemRangeStart)
1481 {
1482 DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1483 KEBUGCHECK(0);
1484 }
1485
1486 Attributes = ProtectToPTE(flProtect);
1487 if (Attributes & 0x80000000)
1488 {
1489 NoExecute = TRUE;
1490 }
1491 Attributes &= 0xfff;
1492 if (Ke386GlobalPagesEnabled)
1493 {
1494 Attributes |= PA_GLOBAL;
1495 }
1496
1497 Addr = Address;
1498
1499 if (Ke386Pae)
1500 {
1501 PULONGLONG Pt = NULL;
1502 ULONGLONG Pte;
1503
1504 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1505 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1506 {
1507 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1508 {
1509 DPRINT1("Setting physical address but not allowing access at address "
1510 "0x%.8X with attributes %x/%x.\n",
1511 Addr, Attributes, flProtect);
1512 KEBUGCHECK(0);
1513 }
1514
1515 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1516 if (oldPdeOffset != PdeOffset)
1517 {
1518 Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE);
1519 if (Pt == NULL)
1520 {
1521 KEBUGCHECK(0);
1522 }
1523 }
1524 else
1525 {
1526 Pt++;
1527 }
1528 oldPdeOffset = PdeOffset;
1529
1530 Pte = PFN_TO_PTE(Pages[i]) | Attributes;
1531 if (NoExecute)
1532 {
1533 Pte |= 0x8000000000000000LL;
1534 }
1535 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1536 if (Pte != 0LL)
1537 {
1538 KEBUGCHECK(0);
1539 }
1540 }
1541 }
1542 else
1543 {
1544 PULONG Pt;
1545 ULONG Pte;
1546
1547 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1548 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1549 if (Pt == NULL)
1550 {
1551 KEBUGCHECK(0);
1552 }
1553 Pt--;
1554
1555 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1556 {
1557 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1558 {
1559 DPRINT1("Setting physical address but not allowing access at address "
1560 "0x%.8X with attributes %x/%x.\n",
1561 Addr, Attributes, flProtect);
1562 KEBUGCHECK(0);
1563 }
1564
1565 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1566 if (oldPdeOffset != PdeOffset)
1567 {
1568 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1569 if (Pt == NULL)
1570 {
1571 KEBUGCHECK(0);
1572 }
1573 }
1574 else
1575 {
1576 Pt++;
1577 }
1578 oldPdeOffset = PdeOffset;
1579
1580 Pte = *Pt;
1581 if (Pte != 0)
1582 {
1583 KEBUGCHECK(0);
1584 }
1585 InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1586 }
1587 }
1588
1589 return(STATUS_SUCCESS);
1590 }
1591
1592 NTSTATUS
1593 MmCreatePageFileMapping(PEPROCESS Process,
1594 PVOID Address,
1595 SWAPENTRY SwapEntry)
1596 {
1597 if (Process == NULL && Address < MmSystemRangeStart)
1598 {
1599 DPRINT1("No process\n");
1600 KEBUGCHECK(0);
1601 }
1602 if (Process != NULL && Address >= MmSystemRangeStart)
1603 {
1604 DPRINT1("Setting kernel address with process context\n");
1605 KEBUGCHECK(0);
1606 }
1607 if (SwapEntry & (1 << 31))
1608 {
1609 KEBUGCHECK(0);
1610 }
1611
1612 if (Ke386Pae)
1613 {
1614 PULONGLONG Pt;
1615 ULONGLONG Pte;
1616 ULONGLONG tmpPte;
1617
1618 Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
1619 if (Pt == NULL)
1620 {
1621 KEBUGCHECK(0);
1622 }
1623 tmpPte = SwapEntry << 1;
1624 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1625 if (PAE_PAGE_MASK((Pte)) != 0)
1626 {
1627 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1628 }
1629
1630 if (Pte != 0)
1631 {
1632 MiFlushTlb((PULONG)Pt, Address);
1633 }
1634 else
1635 {
1636 MmUnmapPageTable((PULONG)Pt);
1637 }
1638 }
1639 else
1640 {
1641 PULONG Pt;
1642 ULONG Pte;
1643
1644 Pt = MmGetPageTableForProcess(Process, Address, TRUE);
1645 if (Pt == NULL)
1646 {
1647 KEBUGCHECK(0);
1648 }
1649 Pte = *Pt;
1650 if (PAGE_MASK((Pte)) != 0)
1651 {
1652 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1653 }
1654 InterlockedExchangeUL(Pt, SwapEntry << 1);
1655 if (Pte != 0)
1656 {
1657 MiFlushTlb(Pt, Address);
1658 }
1659 else
1660 {
1661 MmUnmapPageTable(Pt);
1662 }
1663 }
1664 if (Process != NULL &&
1665 Process->AddressSpace.PageTableRefCountTable != NULL &&
1666 Address < MmSystemRangeStart)
1667 {
1668 PUSHORT Ptrc;
1669 ULONG Idx;
1670
1671 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1672 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1673 Ptrc[Idx]++;
1674 }
1675 return(STATUS_SUCCESS);
1676 }
1677
1678
1679 NTSTATUS
1680 MmCreateVirtualMappingUnsafe(PEPROCESS Process,
1681 PVOID Address,
1682 ULONG flProtect,
1683 PPFN_TYPE Pages,
1684 ULONG PageCount)
1685 {
1686 ULONG Attributes;
1687 PVOID Addr;
1688 ULONG i;
1689 ULONG oldPdeOffset, PdeOffset;
1690 BOOLEAN NoExecute = FALSE;
1691
1692 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1693 Process, Address, flProtect, Pages, *Pages, PageCount);
1694
1695 if (Process == NULL)
1696 {
1697 if (Address < MmSystemRangeStart)
1698 {
1699 DPRINT1("No process\n");
1700 KEBUGCHECK(0);
1701 }
1702 if (PageCount > 0x10000 ||
1703 (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
1704 {
1705 DPRINT1("Page count to large\n");
1706 KEBUGCHECK(0);
1707 }
1708 }
1709 else
1710 {
1711 if (Address >= MmSystemRangeStart)
1712 {
1713 DPRINT1("Setting kernel address with process context\n");
1714 KEBUGCHECK(0);
1715 }
1716 if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE ||
1717 (ULONG_PTR) Address / PAGE_SIZE + PageCount >
1718 (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE)
1719 {
1720 DPRINT1("Page Count to large\n");
1721 KEBUGCHECK(0);
1722 }
1723 }
1724
1725 Attributes = ProtectToPTE(flProtect);
1726 if (Attributes & 0x80000000)
1727 {
1728 NoExecute = TRUE;
1729 }
1730 Attributes &= 0xfff;
1731 if (Address >= MmSystemRangeStart)
1732 {
1733 Attributes &= ~PA_USER;
1734 if (Ke386GlobalPagesEnabled)
1735 {
1736 Attributes |= PA_GLOBAL;
1737 }
1738 }
1739 else
1740 {
1741 Attributes |= PA_USER;
1742 }
1743
1744 Addr = Address;
1745
1746 if (Ke386Pae)
1747 {
1748 ULONGLONG Pte, tmpPte;
1749 PULONGLONG Pt = NULL;
1750
1751 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1752 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1753 {
1754 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1755 {
1756 DPRINT1("Setting physical address but not allowing access at address "
1757 "0x%.8X with attributes %x/%x.\n",
1758 Addr, Attributes, flProtect);
1759 KEBUGCHECK(0);
1760 }
1761 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1762 if (oldPdeOffset != PdeOffset)
1763 {
1764 MmUnmapPageTable((PULONG)Pt);
1765 Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
1766 if (Pt == NULL)
1767 {
1768 KEBUGCHECK(0);
1769 }
1770 }
1771 else
1772 {
1773 Pt++;
1774 }
1775 oldPdeOffset = PdeOffset;
1776
1777 MmMarkPageMapped(Pages[i]);
1778 tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
1779 if (NoExecute)
1780 {
1781 tmpPte |= 0x8000000000000000LL;
1782 }
1783 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1784 if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
1785 {
1786 KEBUGCHECK(0);
1787 }
1788 if (PAE_PAGE_MASK((Pte)) != 0LL)
1789 {
1790 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1791 }
1792 if (Address < MmSystemRangeStart &&
1793 Process->AddressSpace.PageTableRefCountTable != NULL &&
1794 Attributes & PA_PRESENT)
1795 {
1796 PUSHORT Ptrc;
1797
1798 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1799
1800 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
1801 }
1802 if (Pte != 0LL)
1803 {
1804 if (Address > MmSystemRangeStart ||
1805 (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
1806 {
1807 MiFlushTlb((PULONG)Pt, Address);
1808 }
1809 }
1810 }
1811 if (Addr > Address)
1812 {
1813 MmUnmapPageTable((PULONG)Pt);
1814 }
1815 }
1816 else
1817 {
1818 PULONG Pt = NULL;
1819 ULONG Pte;
1820 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
1821 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1822 {
1823 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1824 {
1825 DPRINT1("Setting physical address but not allowing access at address "
1826 "0x%.8X with attributes %x/%x.\n",
1827 Addr, Attributes, flProtect);
1828 KEBUGCHECK(0);
1829 }
1830 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1831 if (oldPdeOffset != PdeOffset)
1832 {
1833 MmUnmapPageTable(Pt);
1834 Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
1835 if (Pt == NULL)
1836 {
1837 KEBUGCHECK(0);
1838 }
1839 }
1840 else
1841 {
1842 Pt++;
1843 }
1844 oldPdeOffset = PdeOffset;
1845
1846 Pte = *Pt;
1847 MmMarkPageMapped(Pages[i]);
1848 if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
1849 {
1850 KEBUGCHECK(0);
1851 }
1852 if (PAGE_MASK((Pte)) != 0)
1853 {
1854 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1855 }
1856 InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1857 if (Address < MmSystemRangeStart &&
1858 Process->AddressSpace.PageTableRefCountTable != NULL &&
1859 Attributes & PA_PRESENT)
1860 {
1861 PUSHORT Ptrc;
1862
1863 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1864
1865 Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
1866 }
1867 if (Pte != 0)
1868 {
1869 if (Address > MmSystemRangeStart ||
1870 (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
1871 {
1872 MiFlushTlb(Pt, Address);
1873 }
1874 }
1875 }
1876 if (Addr > Address)
1877 {
1878 MmUnmapPageTable(Pt);
1879 }
1880 }
1881 return(STATUS_SUCCESS);
1882 }
1883
1884 NTSTATUS
1885 MmCreateVirtualMapping(PEPROCESS Process,
1886 PVOID Address,
1887 ULONG flProtect,
1888 PPFN_TYPE Pages,
1889 ULONG PageCount)
1890 {
1891 ULONG i;
1892
1893 for (i = 0; i < PageCount; i++)
1894 {
1895 if (!MmIsUsablePage(Pages[i]))
1896 {
1897 DPRINT1("Page at address %x not usable\n", PFN_TO_PTE(Pages[i]));
1898 KEBUGCHECK(0);
1899 }
1900 }
1901
1902 return(MmCreateVirtualMappingUnsafe(Process,
1903 Address,
1904 flProtect,
1905 Pages,
1906 PageCount));
1907 }
1908
1909 ULONG
1910 MmGetPageProtect(PEPROCESS Process, PVOID Address)
1911 {
1912 ULONG Entry;
1913 ULONG Protect;
1914 if (Ke386Pae)
1915 {
1916 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1917 }
1918 else
1919 {
1920 Entry = MmGetPageEntryForProcess(Process, Address);
1921 }
1922
1923 if (!(Entry & PA_PRESENT))
1924 {
1925 Protect = PAGE_NOACCESS;
1926 }
1927 else
1928 {
1929 if (Entry & PA_READWRITE)
1930 {
1931 Protect = PAGE_READWRITE;
1932 }
1933 else
1934 {
1935 Protect = PAGE_EXECUTE_READ;
1936 }
1937 if (Entry & PA_CD)
1938 {
1939 Protect |= PAGE_NOCACHE;
1940 }
1941 if (Entry & PA_WT)
1942 {
1943 Protect |= PAGE_WRITETHROUGH;
1944 }
1945 if (!(Entry & PA_USER))
1946 {
1947 Protect |= PAGE_SYSTEM;
1948 }
1949
1950 }
1951 return(Protect);
1952 }
1953
1954 VOID
1955 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
1956 {
1957 ULONG Attributes = 0;
1958 BOOLEAN NoExecute = FALSE;
1959
1960 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
1961 Process, Address, flProtect);
1962
1963 Attributes = ProtectToPTE(flProtect);
1964 if (Attributes & 0x80000000)
1965 {
1966 NoExecute = TRUE;
1967 }
1968 Attributes &= 0xfff;
1969 if (Address >= MmSystemRangeStart)
1970 {
1971 Attributes &= ~PA_USER;
1972 if (Ke386GlobalPagesEnabled)
1973 {
1974 Attributes |= PA_GLOBAL;
1975 }
1976 }
1977 else
1978 {
1979 Attributes |= PA_USER;
1980 }
1981 if (Ke386Pae)
1982 {
1983 PULONGLONG Pt;
1984 ULONGLONG tmpPte, Pte;
1985
1986 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1987 if (Pt == NULL)
1988 {
1989 DPRINT1("Address %x\n", Address);
1990 KEBUGCHECK(0);
1991 }
1992 do
1993 {
1994 Pte = *Pt;
1995 tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
1996 if (NoExecute)
1997 {
1998 tmpPte |= 0x8000000000000000LL;
1999 }
2000 else
2001 {
2002 tmpPte &= ~0x8000000000000000LL;
2003 }
2004 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
2005
2006 MiFlushTlb((PULONG)Pt, Address);
2007 }
2008 else
2009 {
2010 PULONG Pt;
2011
2012 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
2013 if (Pt == NULL)
2014 {
2015 KEBUGCHECK(0);
2016 }
2017 InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
2018 MiFlushTlb(Pt, Address);
2019 }
2020 }
2021
2022 /*
2023 * @implemented
2024 */
2025 PHYSICAL_ADDRESS STDCALL
2026 MmGetPhysicalAddress(PVOID vaddr)
2027 /*
2028 * FUNCTION: Returns the physical address corresponding to a virtual address
2029 */
2030 {
2031 PHYSICAL_ADDRESS p;
2032
2033 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr);
2034 if (Ke386Pae)
2035 {
2036 ULONGLONG Pte;
2037 Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr);
2038 if (Pte != 0 && Pte & PA_PRESENT)
2039 {
2040 p.QuadPart = PAE_PAGE_MASK(Pte);
2041 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2042 }
2043 else
2044 {
2045 p.QuadPart = 0;
2046 }
2047 }
2048 else
2049 {
2050 ULONG Pte;
2051 Pte = MmGetPageEntryForProcess(NULL, vaddr);
2052 if (Pte != 0 && Pte & PA_PRESENT)
2053 {
2054 p.QuadPart = PAGE_MASK(Pte);
2055 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2056 }
2057 else
2058 {
2059 p.QuadPart = 0;
2060 }
2061 }
2062 return p;
2063 }
2064
2065 PVOID
2066 MmCreateHyperspaceMapping(PFN_TYPE Page)
2067 {
2068 PVOID Address;
2069 ULONG i;
2070
2071 if (Ke386Pae)
2072 {
2073 ULONGLONG Entry;
2074 ULONGLONG ZeroEntry = 0LL;
2075 PULONGLONG Pte;
2076
2077 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2078 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2079
2080 if (Page & 1024)
2081 {
2082 for (i = Page %1024; i < 1024; i++, Pte++)
2083 {
2084 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2085 {
2086 break;
2087 }
2088 }
2089 if (i >= 1024)
2090 {
2091 Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
2092 for (i = 0; i < Page % 1024; i++, Pte++)
2093 {
2094 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2095 {
2096 break;
2097 }
2098 }
2099 if (i >= Page % 1024)
2100 {
2101 KEBUGCHECK(0);
2102 }
2103 }
2104 }
2105 else
2106 {
2107 for (i = Page %1024; (LONG)i >= 0; i--, Pte--)
2108 {
2109 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2110 {
2111 break;
2112 }
2113 }
2114 if ((LONG)i < 0)
2115 {
2116 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
2117 for (i = 1023; i > Page % 1024; i--, Pte--)
2118 {
2119 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2120 {
2121 break;
2122 }
2123 }
2124 if (i <= Page % 1024)
2125 {
2126 KEBUGCHECK(0);
2127 }
2128 }
2129 }
2130 }
2131 else
2132 {
2133 ULONG Entry;
2134 PULONG Pte;
2135 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2136 Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2137 if (Page & 1024)
2138 {
2139 for (i = Page % 1024; i < 1024; i++, Pte++)
2140 {
2141 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2142 {
2143 break;
2144 }
2145 }
2146 if (i >= 1024)
2147 {
2148 Pte = ADDR_TO_PTE(HYPERSPACE);
2149 for (i = 0; i < Page % 1024; i++, Pte++)
2150 {
2151 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2152 {
2153 break;
2154 }
2155 }
2156 if (i >= Page % 1024)
2157 {
2158 KEBUGCHECK(0);
2159 }
2160 }
2161 }
2162 else
2163 {
2164 for (i = Page % 1024; (LONG)i >= 0; i--, Pte--)
2165 {
2166 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2167 {
2168 break;
2169 }
2170 }
2171 if ((LONG)i < 0)
2172 {
2173 Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
2174 for (i = 1023; i > Page % 1024; i--, Pte--)
2175 {
2176 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2177 {
2178 break;
2179 }
2180 }
2181 if (i <= Page % 1024)
2182 {
2183 KEBUGCHECK(0);
2184 }
2185 }
2186 }
2187 }
2188 Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE);
2189 FLUSH_TLB_ONE(Address);
2190 return Address;
2191 }
2192
2193 PFN_TYPE
2194 MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage)
2195 {
2196 PFN_TYPE Pfn;
2197 ASSERT (IS_HYPERSPACE(Address));
2198 if (Ke386Pae)
2199 {
2200 ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE;
2201 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2202 Pfn = PAE_PTE_TO_PFN(Entry);
2203 }
2204 else
2205 {
2206 ULONG Entry;
2207 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE);
2208 Pfn = PTE_TO_PFN(Entry);
2209 }
2210 FLUSH_TLB_ONE(Address);
2211 return Pfn;
2212 }
2213
2214 PFN_TYPE
2215 MmDeleteHyperspaceMapping(PVOID Address)
2216 {
2217 PFN_TYPE Pfn;
2218 ASSERT (IS_HYPERSPACE(Address));
2219 if (Ke386Pae)
2220 {
2221 ULONGLONG Entry = 0LL;
2222 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2223 Pfn = PAE_PTE_TO_PFN(Entry);
2224 }
2225 else
2226 {
2227 ULONG Entry;
2228 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0);
2229 Pfn = PTE_TO_PFN(Entry);
2230 }
2231 FLUSH_TLB_ONE(Address);
2232 return Pfn;
2233 }
2234
2235 VOID MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
2236 {
2237 ULONG StartOffset, EndOffset, Offset;
2238
2239 if (Address < MmSystemRangeStart)
2240 {
2241 KEBUGCHECK(0);
2242 }
2243 if (Ke386Pae)
2244 {
2245 PULONGLONG PageDirTable;
2246 PULONGLONG Pde;
2247 ULONGLONG ZeroPde = 0LL;
2248 ULONG i;
2249
2250 for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++)
2251 {
2252 if (i == PAE_ADDR_TO_PDTE_OFFSET(Address))
2253 {
2254 StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
2255 }
2256 else
2257 {
2258 StartOffset = 0;
2259 }
2260 if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)))
2261 {
2262 EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2263 }
2264 else
2265 {
2266 EndOffset = 511;
2267 }
2268
2269 if (Process != NULL && Process != PsGetCurrentProcess())
2270 {
2271 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
2272 Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i]));
2273 MmDeleteHyperspaceMapping(PageDirTable);
2274 }
2275 else
2276 {
2277 Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512;
2278 }
2279
2280 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2281 {
2282 if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4)
2283 {
2284 ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde);
2285 }
2286 }
2287 MmUnmapPageTable((PULONG)Pde);
2288 }
2289 }
2290 else
2291 {
2292 PULONG Pde;
2293 StartOffset = ADDR_TO_PDE_OFFSET(Address);
2294 EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2295
2296 if (Process != NULL && Process != PsGetCurrentProcess())
2297 {
2298 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
2299 }
2300 else
2301 {
2302 Pde = (PULONG)PAGEDIRECTORY_MAP;
2303 }
2304 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2305 {
2306 if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP))
2307 {
2308 InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0);
2309 }
2310 }
2311 if (Pde != (PULONG)PAGEDIRECTORY_MAP)
2312 {
2313 MmDeleteHyperspaceMapping(Pde);
2314 }
2315 }
2316 }
2317
2318 VOID INIT_FUNCTION
2319 MmInitGlobalKernelPageDirectory(VOID)
2320 {
2321 ULONG i;
2322
2323 DPRINT("MmInitGlobalKernelPageDirectory()\n");
2324
2325 if (Ke386Pae)
2326 {
2327 PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
2328 for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++)
2329 {
2330 if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
2331 !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
2332 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
2333 {
2334 ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
2335 if (Ke386GlobalPagesEnabled)
2336 {
2337 MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
2338 CurrentPageDirectory[i] |= PA_GLOBAL;
2339 }
2340 }
2341 }
2342 }
2343 else
2344 {
2345 PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
2346 for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++)
2347 {
2348 if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
2349 i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
2350 0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
2351 {
2352 MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
2353 if (Ke386GlobalPagesEnabled)
2354 {
2355 MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
2356 CurrentPageDirectory[i] |= PA_GLOBAL;
2357 }
2358 }
2359 }
2360 }
2361 }
2362
2363 ULONG
2364 MiGetUserPageDirectoryCount(VOID)
2365 {
2366 return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart);
2367 }
2368
2369 VOID INIT_FUNCTION
2370 MiInitPageDirectoryMap(VOID)
2371 {
2372 MEMORY_AREA* kernel_map_desc = NULL;
2373 MEMORY_AREA* hyperspace_desc = NULL;
2374 PHYSICAL_ADDRESS BoundaryAddressMultiple;
2375 PVOID BaseAddress;
2376 NTSTATUS Status;
2377
2378 DPRINT("MiInitPageDirectoryMap()\n");
2379
2380 BoundaryAddressMultiple.QuadPart = 0;
2381 BaseAddress = (PVOID)PAGETABLE_MAP;
2382 Status = MmCreateMemoryArea(NULL,
2383 MmGetKernelAddressSpace(),
2384 MEMORY_AREA_SYSTEM,
2385 &BaseAddress,
2386 Ke386Pae ? 0x800000 : 0x400000,
2387 0,
2388 &kernel_map_desc,
2389 TRUE,
2390 FALSE,
2391 BoundaryAddressMultiple);
2392 if (!NT_SUCCESS(Status))
2393 {
2394 KEBUGCHECK(0);
2395 }
2396 BaseAddress = (PVOID)HYPERSPACE;
2397 Status = MmCreateMemoryArea(NULL,
2398 MmGetKernelAddressSpace(),
2399 MEMORY_AREA_SYSTEM,
2400 &BaseAddress,
2401 0x400000,
2402 0,
2403 &hyperspace_desc,
2404 TRUE,
2405 FALSE,
2406 BoundaryAddressMultiple);
2407 if (!NT_SUCCESS(Status))
2408 {
2409 KEBUGCHECK(0);
2410 }
2411 }
2412
2413 /* EOF */
A free Windows-compatible Operating System