projects / reactos.git / blob
? search:


b774ee527aed80a57f1ebc77f52aef6b488f64ac
[reactos.git] / reactos / ntoskrnl / mm / i386 / page.c
1 /* $Id$
2 *
3 * COPYRIGHT: See COPYING in the top level directory
4 * PROJECT: ReactOS kernel
5 * FILE: ntoskrnl/mm/i386/page.c
6 * PURPOSE: Low level memory managment manipulation
7 *
8 * PROGRAMMERS: David Welch (welch@cwcom.net)
9 */
10
11 /* INCLUDES ***************************************************************/
12
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <internal/debug.h>
16
17 #if defined (ALLOC_PRAGMA)
18 #pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
19 #pragma alloc_text(INIT, MiInitPageDirectoryMap)
20 #endif
21
22
23 /* GLOBALS *****************************************************************/
24
25 #define PA_BIT_PRESENT (0)
26 #define PA_BIT_READWRITE (1)
27 #define PA_BIT_USER (2)
28 #define PA_BIT_WT (3)
29 #define PA_BIT_CD (4)
30 #define PA_BIT_ACCESSED (5)
31 #define PA_BIT_DIRTY (6)
32 #define PA_BIT_GLOBAL (8)
33
34 #define PA_PRESENT (1 << PA_BIT_PRESENT)
35 #define PA_READWRITE (1 << PA_BIT_READWRITE)
36 #define PA_USER (1 << PA_BIT_USER)
37 #define PA_DIRTY (1 << PA_BIT_DIRTY)
38 #define PA_WT (1 << PA_BIT_WT)
39 #define PA_CD (1 << PA_BIT_CD)
40 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
41 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
42
43 #define PAGETABLE_MAP (0xc0000000)
44 #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024)))
45
46 #define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512)))
47
48 #define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000)
49 #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
50
51 ULONG MmGlobalKernelPageDirectory[1024];
52 ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];
53
54 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
55 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
56
57 #define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
58 #define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
59
60 #if defined(__GNUC__)
61 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
62 #else
63 __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage)
64 {
65 LARGE_INTEGER dummy;
66 dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage));
67 return dummy;
68 }
69 #endif
70
71 extern BOOLEAN Ke386Pae;
72 extern BOOLEAN Ke386NoExecute;
73 extern BOOLEAN Ke386GlobalPagesEnabled;
74
75 /* FUNCTIONS ***************************************************************/
76
77 BOOLEAN MmUnmapPageTable(PULONG Pt);
78
79 VOID
80 STDCALL
81 MiFlushTlbIpiRoutine(PVOID Address)
82 {
83 if (Address == (PVOID)0xffffffff)
84 {
85 KeFlushCurrentTb();
86 }
87 else if (Address == (PVOID)0xfffffffe)
88 {
89 FLUSH_TLB;
90 }
91 else
92 {
93 FLUSH_TLB_ONE(Address);
94 }
95 }
96
97 VOID
98 MiFlushTlb(PULONG Pt, PVOID Address)
99 {
100 #ifdef CONFIG_SMP
101 if (Pt)
102 {
103 MmUnmapPageTable(Pt);
104 }
105 if (KeNumberProcessors>1)
106 {
107 KeIpiGenericCall(MiFlushTlbIpiRoutine, Address);
108 }
109 else
110 {
111 MiFlushTlbIpiRoutine(Address);
112 }
113 #else
114 if ((Pt && MmUnmapPageTable(Pt)) || Address >= MmSystemRangeStart)
115 {
116 FLUSH_TLB_ONE(Address);
117 }
118 #endif
119 }
120
121
122
123 PULONG
124 MmGetPageDirectory(VOID)
125 {
126 unsigned int page_dir=0;
127 Ke386GetPageTableDirectory(page_dir);
128 return((PULONG)page_dir);
129 }
130
131 static ULONG
132 ProtectToPTE(ULONG flProtect)
133 {
134 ULONG Attributes = 0;
135
136 if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
137 {
138 Attributes = 0;
139 }
140 else if (flProtect & PAGE_IS_WRITABLE)
141 {
142 Attributes = PA_PRESENT | PA_READWRITE;
143 }
144 else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
145 {
146 Attributes = PA_PRESENT;
147 }
148 else
149 {
150 DPRINT1("Unknown main protection type.\n");
151 KEBUGCHECK(0);
152 }
153 if (Ke386NoExecute &&
154 !(flProtect & PAGE_IS_EXECUTABLE))
155 {
156 Attributes = Attributes | 0x80000000;
157 }
158
159 if (flProtect & PAGE_SYSTEM)
160 {
161 }
162 else
163 {
164 Attributes = Attributes | PA_USER;
165 }
166 if (flProtect & PAGE_NOCACHE)
167 {
168 Attributes = Attributes | PA_CD;
169 }
170 if (flProtect & PAGE_WRITETHROUGH)
171 {
172 Attributes = Attributes | PA_WT;
173 }
174 return(Attributes);
175 }
176
177 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
178
179 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
180 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
181 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
182
183 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
184
185 #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
186
187
188 #define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE))
189
190 #define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
191 ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
192 #define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
193
194
195 #define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
196
197 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
198
199 #define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
200
201 #define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
202
203
204 NTSTATUS
205 NTAPI
206 Mmi386ReleaseMmInfo(PEPROCESS Process)
207 {
208 PUSHORT LdtDescriptor;
209 ULONG LdtBase;
210 ULONG i, j;
211
212 DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process);
213
214 LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor;
215 LdtBase = LdtDescriptor[1] |
216 ((LdtDescriptor[2] & 0xff) << 16) |
217 ((LdtDescriptor[3] & ~0xff) << 16);
218
219 DPRINT("LdtBase: %x\n", LdtBase);
220
221 if (LdtBase)
222 {
223 ExFreePool((PVOID) LdtBase);
224 }
225
226 if (Ke386Pae)
227 {
228 PULONGLONG PageDirTable;
229 PULONGLONG PageDir;
230 PULONGLONG Pde;
231 ULONG k;
232
233 PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
234 for (i = 0; i < 4; i++)
235 {
236 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i]));
237 if (i < PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart))
238 {
239 for (j = 0; j < 512; j++)
240 {
241 if (PageDir[j] != 0LL)
242 {
243 DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
244 Process->UniqueProcessId,
245 (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1,
246 Process->AddressSpace.PageTableRefCountTable[i*512 + j]);
247 Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j]));
248 for (k = 0; k < 512; k++)
249 {
250 if(Pde[k] != 0)
251 {
252 if (Pde[k] & PA_PRESENT)
253 {
254 DPRINT1("Page at %08x is not freed\n",
255 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
256 }
257 else
258 {
259 DPRINT1("Swapentry %x at %x is not freed\n",
260 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
261 }
262 }
263 }
264 MmDeleteHyperspaceMapping(Pde);
265 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j]));
266 }
267 }
268 }
269 if (i == PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE))
270 {
271 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)]));
272 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1]));
273 }
274 MmDeleteHyperspaceMapping(PageDir);
275 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i]));
276 }
277 MmDeleteHyperspaceMapping((PVOID)PageDirTable);
278 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
279 }
280 else
281 {
282 PULONG Pde;
283 PULONG PageDir;
284 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
285 for (i = 0; i < ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i++)
286 {
287 if (PageDir[i] != 0)
288 {
289 DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
290 i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1,
291 Process->AddressSpace.PageTableRefCountTable[i]);
292 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i]));
293 for (j = 0; j < 1024; j++)
294 {
295 if(Pde[j] != 0)
296 {
297 if (Pde[j] & PA_PRESENT)
298 {
299 DPRINT1("Page at %08x is not freed\n",
300 i * 4 * 1024 * 1024 + j * PAGE_SIZE);
301 }
302 else
303 {
304 DPRINT1("Swapentry %x at %x is not freed\n",
305 Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE);
306 }
307 }
308 }
309 MmDeleteHyperspaceMapping(Pde);
310 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i]));
311 }
312 }
313 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)]));
314 MmDeleteHyperspaceMapping(PageDir);
315 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
316 }
317
318 #if defined(__GNUC__)
319
320 Process->Pcb.DirectoryTableBase.QuadPart = 0LL;
321 #else
322
323 Process->Pcb.DirectoryTableBase.QuadPart = 0;
324 #endif
325
326 DPRINT("Finished Mmi386ReleaseMmInfo()\n");
327 return(STATUS_SUCCESS);
328 }
329
330 NTSTATUS
331 STDCALL
332 MmCopyMmInfo(PEPROCESS Src,
333 PEPROCESS Dest,
334 PPHYSICAL_ADDRESS DirectoryTableBase)
335 {
336 NTSTATUS Status;
337 ULONG i, j;
338 PFN_TYPE Pfn[7];
339 ULONG Count;
340
341 DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", Src, Dest);
342
343 Count = Ke386Pae ? 7 : 2;
344
345 for (i = 0; i < Count; i++)
346 {
347 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
348 if (!NT_SUCCESS(Status))
349 {
350 for (j = 0; j < i; j++)
351 {
352 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
353 }
354 return Status;
355 }
356 }
357
358 if (Ke386Pae)
359 {
360 PULONGLONG PageDirTable;
361 PULONGLONG PageDir;
362
363 PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
364 for (i = 0; i < 4; i++)
365 {
366 PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
367 }
368 MmDeleteHyperspaceMapping(PageDirTable);
369 for (i = PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart); i < 4; i++)
370 {
371 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
372 memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
373 if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
374 {
375 for (j = 0; j < 4; j++)
376 {
377 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
378 }
379 }
380 if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
381 {
382 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
383 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
384 }
385 MmDeleteHyperspaceMapping(PageDir);
386 }
387 }
388 else
389 {
390 PULONG PageDirectory;
391 PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);
392
393 memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
394 MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(MmSystemRangeStart),
395 (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart)) * sizeof(ULONG));
396
397 DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
398 PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
399 PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;
400
401 MmDeleteHyperspaceMapping(PageDirectory);
402 }
403
404 DirectoryTableBase->QuadPart = PFN_TO_PTE(Pfn[0]);
405 DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase->QuadPart);
406 return(STATUS_SUCCESS);
407 }
408
409 VOID
410 NTAPI
411 MmDeletePageTable(PEPROCESS Process, PVOID Address)
412 {
413 PEPROCESS CurrentProcess = PsGetCurrentProcess();
414
415 if (Process != NULL && Process != CurrentProcess)
416 {
417 KeAttachProcess(&Process->Pcb);
418 }
419
420 if (Ke386Pae)
421 {
422 ULONGLONG ZeroPde = 0LL;
423 ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde);
424 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
425 }
426 else
427 {
428 *(ADDR_TO_PDE(Address)) = 0;
429 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
430 }
431 if (Address >= MmSystemRangeStart)
432 {
433 KEBUGCHECK(0);
434 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
435 }
436 if (Process != NULL && Process != CurrentProcess)
437 {
438 KeDetachProcess();
439 }
440 }
441
442 VOID
443 NTAPI
444 MmFreePageTable(PEPROCESS Process, PVOID Address)
445 {
446 PEPROCESS CurrentProcess = PsGetCurrentProcess();
447 ULONG i;
448 PFN_TYPE Pfn;
449
450 DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
451 if (Process != NULL && Process != CurrentProcess)
452 {
453 KeAttachProcess(&Process->Pcb);
454 }
455 if (Ke386Pae)
456 {
457 PULONGLONG PageTable;
458 ULONGLONG ZeroPte = 0LL;
459 PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
460 for (i = 0; i < 512; i++)
461 {
462 if (PageTable[i] != 0LL)
463 {
464 DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
465 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
466 KEBUGCHECK(0);
467 }
468 }
469 Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
470 ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
471 MiFlushTlb((PULONG)PAE_ADDR_TO_PDE(Address), PAE_ADDR_TO_PTE(Address));
472 }
473 else
474 {
475 PULONG PageTable;
476 PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
477 for (i = 0; i < 1024; i++)
478 {
479 if (PageTable[i] != 0)
480 {
481 DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
482 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
483 KEBUGCHECK(0);
484 }
485 }
486 Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
487 *(ADDR_TO_PDE(Address)) = 0;
488 MiFlushTlb(ADDR_TO_PDE(Address), ADDR_TO_PTE(Address));
489 }
490
491 if (Address >= MmSystemRangeStart)
492 {
493 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
494 KEBUGCHECK(0);
495 }
496 else
497 {
498 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
499 }
500 if (Process != NULL && Process != CurrentProcess)
501 {
502 KeDetachProcess();
503 }
504 }
505
506 static PULONGLONG
507 MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
508 {
509 NTSTATUS Status;
510 PFN_TYPE Pfn;
511 ULONGLONG Entry;
512 ULONGLONG ZeroEntry = 0LL;
513 PULONGLONG Pt;
514 PULONGLONG PageDir;
515 PULONGLONG PageDirTable;
516
517 DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
518 Process, Address, Create);
519 if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)((ULONG_PTR)PAGETABLE_MAP + 0x800000))
520 {
521 KEBUGCHECK(0);
522 }
523 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
524 {
525 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
526 if (PageDirTable == NULL)
527 {
528 KEBUGCHECK(0);
529 }
530 PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
531 MmDeleteHyperspaceMapping(PageDirTable);
532 if (PageDir == NULL)
533 {
534 KEBUGCHECK(0);
535 }
536 PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
537 Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
538 if (Entry == 0LL)
539 {
540 if (Create == FALSE)
541 {
542 MmDeleteHyperspaceMapping(PageDir);
543 return NULL;
544 }
545 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
546 if (!NT_SUCCESS(Status))
547 {
548 KEBUGCHECK(0);
549 }
550 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
551 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
552 if (Entry != 0LL)
553 {
554 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
555 Pfn = PAE_PTE_TO_PFN(Entry);
556 }
557 }
558 else
559 {
560 Pfn = PAE_PTE_TO_PFN(Entry);
561 }
562 MmDeleteHyperspaceMapping(PageDir);
563 Pt = MmCreateHyperspaceMapping(Pfn);
564 if (Pt == NULL)
565 {
566 KEBUGCHECK(0);
567 }
568 return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
569 }
570 PageDir = PAE_ADDR_TO_PDE(Address);
571 if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
572 {
573 if (Address >= MmSystemRangeStart)
574 {
575 if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
576 {
577 if (Create == FALSE)
578 {
579 return NULL;
580 }
581 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
582 if (!NT_SUCCESS(Status))
583 {
584 KEBUGCHECK(0);
585 }
586 Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
587 if (Ke386GlobalPagesEnabled)
588 {
589 Entry |= PA_GLOBAL;
590 }
591 if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
592 {
593 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
594 }
595 }
596 ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
597 }
598 else
599 {
600 if (Create == FALSE)
601 {
602 return NULL;
603 }
604 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
605 if (!NT_SUCCESS(Status))
606 {
607 KEBUGCHECK(0);
608 }
609 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
610 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
611 if (Entry != 0LL)
612 {
613 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
614 }
615 }
616 }
617 return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
618 }
619
620 static PULONG
621 MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
622 {
623 ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
624 NTSTATUS Status;
625 PFN_TYPE Pfn;
626 ULONG Entry;
627 PULONG Pt, PageDir;
628
629 if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess())
630 {
631 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
632 if (PageDir == NULL)
633 {
634 KEBUGCHECK(0);
635 }
636 if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
637 {
638 if (Create == FALSE)
639 {
640 MmDeleteHyperspaceMapping(PageDir);
641 return NULL;
642 }
643 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
644 if (!NT_SUCCESS(Status) || Pfn == 0)
645 {
646 KEBUGCHECK(0);
647 }
648 Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
649 if (Entry != 0)
650 {
651 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
652 Pfn = PTE_TO_PFN(Entry);
653 }
654 }
655 else
656 {
657 Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
658 }
659 MmDeleteHyperspaceMapping(PageDir);
660 Pt = MmCreateHyperspaceMapping(Pfn);
661 if (Pt == NULL)
662 {
663 KEBUGCHECK(0);
664 }
665 return Pt + ADDR_TO_PTE_OFFSET(Address);
666 }
667 PageDir = ADDR_TO_PDE(Address);
668 if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
669 {
670 if (Address >= MmSystemRangeStart)
671 {
672 if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
673 {
674 if (Create == FALSE)
675 {
676 return NULL;
677 }
678 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
679 if (!NT_SUCCESS(Status) || Pfn == 0)
680 {
681 KEBUGCHECK(0);
682 }
683 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
684 if (Ke386GlobalPagesEnabled)
685 {
686 Entry |= PA_GLOBAL;
687 }
688 if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
689 {
690 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
691 }
692 }
693 InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
694 }
695 else
696 {
697 if (Create == FALSE)
698 {
699 return NULL;
700 }
701 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
702 if (!NT_SUCCESS(Status) || Pfn == 0)
703 {
704 KEBUGCHECK(0);
705 }
706 Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
707 if (Entry != 0)
708 {
709 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
710 }
711 }
712 }
713 return (PULONG)ADDR_TO_PTE(Address);
714 }
715
716 BOOLEAN MmUnmapPageTable(PULONG Pt)
717 {
718 if (Ke386Pae)
719 {
720 if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
721 {
722 return TRUE;
723 }
724 }
725 else
726 {
727 if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
728 {
729 return TRUE;
730 }
731 }
732 if (Pt)
733 {
734 MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
735 }
736 return FALSE;
737 }
738
739 static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
740 {
741 ULONGLONG Pte;
742 PULONGLONG Pt;
743
744 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
745 if (Pt)
746 {
747 Pte = *Pt;
748 MmUnmapPageTable((PULONG)Pt);
749 return Pte;
750 }
751 return 0;
752 }
753
754 static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
755 {
756 ULONG Pte;
757 PULONG Pt;
758
759 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
760 if (Pt)
761 {
762 Pte = *Pt;
763 MmUnmapPageTable(Pt);
764 return Pte;
765 }
766 return 0;
767 }
768
769 PFN_TYPE
770 NTAPI
771 MmGetPfnForProcess(PEPROCESS Process,
772 PVOID Address)
773 {
774
775 if (Ke386Pae)
776 {
777 ULONGLONG Entry;
778 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
779 if (!(Entry & PA_PRESENT))
780 {
781 return 0;
782 }
783 return(PAE_PTE_TO_PFN(Entry));
784 }
785 else
786 {
787 ULONG Entry;
788 Entry = MmGetPageEntryForProcess(Process, Address);
789 if (!(Entry & PA_PRESENT))
790 {
791 return 0;
792 }
793 return(PTE_TO_PFN(Entry));
794 }
795 }
796
797 VOID
798 NTAPI
799 MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN* WasDirty, PPFN_TYPE Page)
800 /*
801 * FUNCTION: Delete a virtual mapping
802 */
803 {
804 BOOLEAN WasValid;
805 if (Ke386Pae)
806 {
807 ULONGLONG Pte;
808 ULONGLONG tmpPte;
809 PULONGLONG Pt;
810
811 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
812 if (Pt == NULL)
813 {
814 KEBUGCHECK(0);
815 }
816 /*
817 * Atomically disable the present bit and get the old value.
818 */
819 do
820 {
821 Pte = *Pt;
822 tmpPte = Pte & ~PA_PRESENT;
823 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
824
825 MiFlushTlb((PULONG)Pt, Address);
826 WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE;
827 if (!WasValid)
828 {
829 KEBUGCHECK(0);
830 }
831
832 /*
833 * Return some information to the caller
834 */
835 if (WasDirty != NULL)
836 {
837 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
838 }
839 if (Page != NULL)
840 {
841 *Page = PAE_PTE_TO_PFN(Pte);
842 }
843 }
844 else
845 {
846 ULONG Pte;
847 PULONG Pt;
848
849 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
850 if (Pt == NULL)
851 {
852 KEBUGCHECK(0);
853 }
854 /*
855 * Atomically disable the present bit and get the old value.
856 */
857 do
858 {
859 Pte = *Pt;
860 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte));
861
862 MiFlushTlb(Pt, Address);
863 WasValid = (PAGE_MASK(Pte) != 0);
864 if (!WasValid)
865 {
866 KEBUGCHECK(0);
867 }
868
869 /*
870 * Return some information to the caller
871 */
872 if (WasDirty != NULL)
873 {
874 *WasDirty = Pte & PA_DIRTY;
875 }
876 if (Page != NULL)
877 {
878 *Page = PTE_TO_PFN(Pte);
879 }
880 }
881 }
882
883 VOID
884 NTAPI
885 MmRawDeleteVirtualMapping(PVOID Address)
886 {
887 if (Ke386Pae)
888 {
889 PULONGLONG Pt;
890 ULONGLONG ZeroPte = 0LL;
891 Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE);
892 if (Pt)
893 {
894 /*
895 * Set the entry to zero
896 */
897 ExfpInterlockedExchange64UL(Pt, &ZeroPte);
898 MiFlushTlb((PULONG)Pt, Address);
899 }
900 }
901 else
902 {
903 PULONG Pt;
904
905 Pt = MmGetPageTableForProcess(NULL, Address, FALSE);
906 if (Pt && *Pt)
907 {
908 /*
909 * Set the entry to zero
910 */
911 InterlockedExchangeUL(Pt, 0);
912 MiFlushTlb(Pt, Address);
913 }
914 }
915 }
916
917 VOID
918 NTAPI
919 MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOLEAN FreePage,
920 BOOLEAN* WasDirty, PPFN_TYPE Page)
921 /*
922 * FUNCTION: Delete a virtual mapping
923 */
924 {
925 BOOLEAN WasValid = FALSE;
926 PFN_TYPE Pfn;
927
928 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
929 Process, Address, FreePage, WasDirty, Page);
930 if (Ke386Pae)
931 {
932 ULONGLONG Pte;
933 PULONGLONG Pt;
934
935 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
936 if (Pt == NULL)
937 {
938 if (WasDirty != NULL)
939 {
940 *WasDirty = FALSE;
941 }
942 if (Page != NULL)
943 {
944 *Page = 0;
945 }
946 return;
947 }
948
949 /*
950 * Atomically set the entry to zero and get the old value.
951 */
952 Pte = 0LL;
953 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
954
955 MiFlushTlb((PULONG)Pt, Address);
956
957 WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
958 if (WasValid)
959 {
960 Pfn = PAE_PTE_TO_PFN(Pte);
961 MmMarkPageUnmapped(Pfn);
962 }
963 else
964 {
965 Pfn = 0;
966 }
967
968 if (FreePage && WasValid)
969 {
970 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
971 }
972
973 /*
974 * Return some information to the caller
975 */
976 if (WasDirty != NULL)
977 {
978 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
979 }
980 if (Page != NULL)
981 {
982 *Page = Pfn;
983 }
984 }
985 else
986 {
987 ULONG Pte;
988 PULONG Pt;
989
990 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
991
992 if (Pt == NULL)
993 {
994 if (WasDirty != NULL)
995 {
996 *WasDirty = FALSE;
997 }
998 if (Page != NULL)
999 {
1000 *Page = 0;
1001 }
1002 return;
1003 }
1004
1005 /*
1006 * Atomically set the entry to zero and get the old value.
1007 */
1008 Pte = InterlockedExchangeUL(Pt, 0);
1009
1010 MiFlushTlb(Pt, Address);
1011
1012 WasValid = (PAGE_MASK(Pte) != 0);
1013 if (WasValid)
1014 {
1015 Pfn = PTE_TO_PFN(Pte);
1016 MmMarkPageUnmapped(Pfn);
1017 }
1018 else
1019 {
1020 Pfn = 0;
1021 }
1022
1023 if (FreePage && WasValid)
1024 {
1025 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
1026 }
1027
1028 /*
1029 * Return some information to the caller
1030 */
1031 if (WasDirty != NULL)
1032 {
1033 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
1034 }
1035 if (Page != NULL)
1036 {
1037 *Page = Pfn;
1038 }
1039 }
1040 /*
1041 * Decrement the reference count for this page table.
1042 */
1043 if (Process != NULL && WasValid &&
1044 Process->AddressSpace.PageTableRefCountTable != NULL &&
1045 Address < MmSystemRangeStart)
1046 {
1047 PUSHORT Ptrc;
1048 ULONG Idx;
1049
1050 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1051 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1052
1053 Ptrc[Idx]--;
1054 if (Ptrc[Idx] == 0)
1055 {
1056 MmFreePageTable(Process, Address);
1057 }
1058 }
1059 }
1060
1061 VOID
1062 NTAPI
1063 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
1064 SWAPENTRY* SwapEntry)
1065 /*
1066 * FUNCTION: Delete a virtual mapping
1067 */
1068 {
1069 if (Ke386Pae)
1070 {
1071 ULONGLONG Pte;
1072 PULONGLONG Pt;
1073
1074 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1075 if (Pt == NULL)
1076 {
1077 *SwapEntry = 0;
1078 return;
1079 }
1080
1081 /*
1082 * Atomically set the entry to zero and get the old value.
1083 */
1084 Pte = 0LL;
1085 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1086
1087 MiFlushTlb((PULONG)Pt, Address);
1088
1089 /*
1090 * Decrement the reference count for this page table.
1091 */
1092 if (Process != NULL && Pte &&
1093 Process->AddressSpace.PageTableRefCountTable != NULL &&
1094 Address < MmSystemRangeStart)
1095 {
1096 PUSHORT Ptrc;
1097
1098 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1099
1100 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
1101 if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
1102 {
1103 MmFreePageTable(Process, Address);
1104 }
1105 }
1106
1107
1108 /*
1109 * Return some information to the caller
1110 */
1111 *SwapEntry = Pte >> 1;
1112 }
1113 else
1114 {
1115 ULONG Pte;
1116 PULONG Pt;
1117
1118 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1119
1120 if (Pt == NULL)
1121 {
1122 *SwapEntry = 0;
1123 return;
1124 }
1125
1126 /*
1127 * Atomically set the entry to zero and get the old value.
1128 */
1129 Pte = InterlockedExchangeUL(Pt, 0);
1130
1131 MiFlushTlb(Pt, Address);
1132
1133 /*
1134 * Decrement the reference count for this page table.
1135 */
1136 if (Process != NULL && Pte &&
1137 Process->AddressSpace.PageTableRefCountTable != NULL &&
1138 Address < MmSystemRangeStart)
1139 {
1140 PUSHORT Ptrc;
1141
1142 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1143
1144 Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
1145 if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
1146 {
1147 MmFreePageTable(Process, Address);
1148 }
1149 }
1150
1151
1152 /*
1153 * Return some information to the caller
1154 */
1155 *SwapEntry = Pte >> 1;
1156 }
1157 }
1158
1159 BOOLEAN
1160 Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
1161 {
1162 if (Ke386Pae)
1163 {
1164 PULONGLONG Pt;
1165 PULONGLONG Pde;
1166 Pde = PAE_ADDR_TO_PDE(PAddress);
1167 if (*Pde == 0LL)
1168 {
1169 Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
1170 #if 0
1171 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1172 FLASH_TLB_ONE(PAddress);
1173 #endif
1174 if (Pt != NULL)
1175 {
1176 return TRUE;
1177 }
1178 }
1179 }
1180 else
1181 {
1182 PULONG Pt, Pde;
1183 Pde = ADDR_TO_PDE(PAddress);
1184 if (*Pde == 0)
1185 {
1186 Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
1187 #if 0
1188 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1189 FLASH_TLB_ONE(PAddress);
1190 #endif
1191 if (Pt != NULL)
1192 {
1193 return TRUE;
1194 }
1195 }
1196 }
1197 return(FALSE);
1198 }
1199
1200 BOOLEAN
1201 NTAPI
1202 MmIsDirtyPage(PEPROCESS Process, PVOID Address)
1203 {
1204 if (Ke386Pae)
1205 {
1206 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1207 }
1208 else
1209 {
1210 return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1211 }
1212 }
1213
1214 BOOLEAN
1215 NTAPI
1216 MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address)
1217 {
1218 if (Address < MmSystemRangeStart && Process == NULL)
1219 {
1220 DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1221 KEBUGCHECK(0);
1222 }
1223 if (Ke386Pae)
1224 {
1225 PULONGLONG Pt;
1226 ULONGLONG Pte;
1227 ULONGLONG tmpPte;
1228
1229 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1230 if (Pt == NULL)
1231 {
1232 KEBUGCHECK(0);
1233 }
1234
1235 do
1236 {
1237 Pte = *Pt;
1238 tmpPte = Pte & ~PA_ACCESSED;
1239 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1240
1241 if (Pte & PA_ACCESSED)
1242 {
1243 MiFlushTlb((PULONG)Pt, Address);
1244 return TRUE;
1245 }
1246 else
1247 {
1248 MmUnmapPageTable((PULONG)Pt);
1249 return FALSE;
1250 }
1251 }
1252 else
1253 {
1254 PULONG Pt;
1255 ULONG Pte;
1256
1257 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1258 if (Pt == NULL)
1259 {
1260 KEBUGCHECK(0);
1261 }
1262
1263 do
1264 {
1265 Pte = *Pt;
1266 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte));
1267
1268 if (Pte & PA_ACCESSED)
1269 {
1270 MiFlushTlb(Pt, Address);
1271 return TRUE;
1272 }
1273 else
1274 {
1275 MmUnmapPageTable(Pt);
1276 return FALSE;
1277 }
1278 }
1279 }
1280
1281 VOID
1282 NTAPI
1283 MmSetCleanPage(PEPROCESS Process, PVOID Address)
1284 {
1285 if (Address < MmSystemRangeStart && Process == NULL)
1286 {
1287 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1288 KEBUGCHECK(0);
1289 }
1290 if (Ke386Pae)
1291 {
1292 PULONGLONG Pt;
1293 ULONGLONG Pte;
1294 ULONGLONG tmpPte;
1295
1296 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1297
1298 if (Pt == NULL)
1299 {
1300 KEBUGCHECK(0);
1301 }
1302
1303 do
1304 {
1305 Pte = *Pt;
1306 tmpPte = Pte & ~PA_DIRTY;
1307 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1308
1309 if (Pte & PA_DIRTY)
1310 {
1311 MiFlushTlb((PULONG)Pt, Address);
1312 }
1313 else
1314 {
1315 MmUnmapPageTable((PULONG)Pt);
1316 }
1317 }
1318 else
1319 {
1320 PULONG Pt;
1321 ULONG Pte;
1322
1323 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1324
1325 if (Pt == NULL)
1326 {
1327 KEBUGCHECK(0);
1328 }
1329
1330 do
1331 {
1332 Pte = *Pt;
1333 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));
1334
1335 if (Pte & PA_DIRTY)
1336 {
1337 MiFlushTlb(Pt, Address);
1338 }
1339 else
1340 {
1341 MmUnmapPageTable(Pt);
1342 }
1343 }
1344 }
1345
1346 VOID
1347 NTAPI
1348 MmSetDirtyPage(PEPROCESS Process, PVOID Address)
1349 {
1350 if (Address < MmSystemRangeStart && Process == NULL)
1351 {
1352 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1353 KEBUGCHECK(0);
1354 }
1355 if (Ke386Pae)
1356 {
1357 PULONGLONG Pt;
1358 ULONGLONG Pte;
1359 ULONGLONG tmpPte;
1360
1361 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1362 if (Pt == NULL)
1363 {
1364 KEBUGCHECK(0);
1365 }
1366
1367 do
1368 {
1369 Pte = *Pt;
1370 tmpPte = Pte | PA_DIRTY;
1371 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1372 if (!(Pte & PA_DIRTY))
1373 {
1374 MiFlushTlb((PULONG)Pt, Address);
1375 }
1376 else
1377 {
1378 MmUnmapPageTable((PULONG)Pt);
1379 }
1380 }
1381 else
1382 {
1383 PULONG Pt;
1384 ULONG Pte;
1385
1386 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1387 if (Pt == NULL)
1388 {
1389 KEBUGCHECK(0);
1390 }
1391
1392 do
1393 {
1394 Pte = *Pt;
1395 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
1396 if (!(Pte & PA_DIRTY))
1397 {
1398 MiFlushTlb(Pt, Address);
1399 }
1400 else
1401 {
1402 MmUnmapPageTable(Pt);
1403 }
1404 }
1405 }
1406
1407 VOID
1408 NTAPI
1409 MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
1410 {
1411 if (Ke386Pae)
1412 {
1413 PULONGLONG Pt;
1414 ULONGLONG Pte;
1415 ULONGLONG tmpPte;
1416
1417 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1418 if (Pt == NULL)
1419 {
1420 KEBUGCHECK(0);
1421 }
1422
1423 do
1424 {
1425 Pte = *Pt;
1426 tmpPte = Pte | PA_PRESENT;
1427 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1428 if (!(Pte & PA_PRESENT))
1429 {
1430 MiFlushTlb((PULONG)Pt, Address);
1431 }
1432 else
1433 {
1434 MmUnmapPageTable((PULONG)Pt);
1435 }
1436 }
1437 else
1438 {
1439 PULONG Pt;
1440 ULONG Pte;
1441
1442 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1443 if (Pt == NULL)
1444 {
1445 KEBUGCHECK(0);
1446 }
1447
1448 do
1449 {
1450 Pte = *Pt;
1451 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte));
1452 if (!(Pte & PA_PRESENT))
1453 {
1454 MiFlushTlb(Pt, Address);
1455 }
1456 else
1457 {
1458 MmUnmapPageTable(Pt);
1459 }
1460 }
1461 }
1462
1463 BOOLEAN
1464 NTAPI
1465 MmIsPagePresent(PEPROCESS Process, PVOID Address)
1466 {
1467 if (Ke386Pae)
1468 {
1469 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1470 }
1471 else
1472 {
1473 return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1474 }
1475 }
1476
1477 BOOLEAN
1478 NTAPI
1479 MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
1480 {
1481 if (Ke386Pae)
1482 {
1483 ULONGLONG Entry;
1484 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1485 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1486 }
1487 else
1488 {
1489 ULONG Entry;
1490 Entry = MmGetPageEntryForProcess(Process, Address);
1491 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1492 }
1493 }
1494
1495 NTSTATUS
1496 NTAPI
1497 MmCreateVirtualMappingForKernel(PVOID Address,
1498 ULONG flProtect,
1499 PPFN_TYPE Pages,
1500 ULONG PageCount)
1501 {
1502 ULONG Attributes;
1503 ULONG i;
1504 PVOID Addr;
1505 ULONG PdeOffset, oldPdeOffset;
1506 BOOLEAN NoExecute = FALSE;
1507
1508 DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1509 Address, flProtect, Pages, PageCount);
1510
1511 if (Address < MmSystemRangeStart)
1512 {
1513 DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1514 KEBUGCHECK(0);
1515 }
1516
1517 Attributes = ProtectToPTE(flProtect);
1518 if (Attributes & 0x80000000)
1519 {
1520 NoExecute = TRUE;
1521 }
1522 Attributes &= 0xfff;
1523 if (Ke386GlobalPagesEnabled)
1524 {
1525 Attributes |= PA_GLOBAL;
1526 }
1527
1528 Addr = Address;
1529
1530 if (Ke386Pae)
1531 {
1532 PULONGLONG Pt = NULL;
1533 ULONGLONG Pte;
1534
1535 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1536 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1537 {
1538 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1539 {
1540 DPRINT1("Setting physical address but not allowing access at address "
1541 "0x%.8X with attributes %x/%x.\n",
1542 Addr, Attributes, flProtect);
1543 KEBUGCHECK(0);
1544 }
1545
1546 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1547 if (oldPdeOffset != PdeOffset)
1548 {
1549 Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE);
1550 if (Pt == NULL)
1551 {
1552 KEBUGCHECK(0);
1553 }
1554 }
1555 else
1556 {
1557 Pt++;
1558 }
1559 oldPdeOffset = PdeOffset;
1560
1561 Pte = PFN_TO_PTE(Pages[i]) | Attributes;
1562 if (NoExecute)
1563 {
1564 Pte |= 0x8000000000000000LL;
1565 }
1566 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1567 if (Pte != 0LL)
1568 {
1569 KEBUGCHECK(0);
1570 }
1571 }
1572 }
1573 else
1574 {
1575 PULONG Pt;
1576 ULONG Pte;
1577
1578 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1579 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1580 if (Pt == NULL)
1581 {
1582 KEBUGCHECK(0);
1583 }
1584 Pt--;
1585
1586 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1587 {
1588 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1589 {
1590 DPRINT1("Setting physical address but not allowing access at address "
1591 "0x%.8X with attributes %x/%x.\n",
1592 Addr, Attributes, flProtect);
1593 KEBUGCHECK(0);
1594 }
1595
1596 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1597 if (oldPdeOffset != PdeOffset)
1598 {
1599 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1600 if (Pt == NULL)
1601 {
1602 KEBUGCHECK(0);
1603 }
1604 }
1605 else
1606 {
1607 Pt++;
1608 }
1609 oldPdeOffset = PdeOffset;
1610
1611 Pte = *Pt;
1612 if (Pte != 0)
1613 {
1614 KEBUGCHECK(0);
1615 }
1616 InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1617 }
1618 }
1619
1620 return(STATUS_SUCCESS);
1621 }
1622
1623 NTSTATUS
1624 NTAPI
1625 MmCreatePageFileMapping(PEPROCESS Process,
1626 PVOID Address,
1627 SWAPENTRY SwapEntry)
1628 {
1629 if (Process == NULL && Address < MmSystemRangeStart)
1630 {
1631 DPRINT1("No process\n");
1632 KEBUGCHECK(0);
1633 }
1634 if (Process != NULL && Address >= MmSystemRangeStart)
1635 {
1636 DPRINT1("Setting kernel address with process context\n");
1637 KEBUGCHECK(0);
1638 }
1639 if (SwapEntry & (1 << 31))
1640 {
1641 KEBUGCHECK(0);
1642 }
1643
1644 if (Ke386Pae)
1645 {
1646 PULONGLONG Pt;
1647 ULONGLONG Pte;
1648 ULONGLONG tmpPte;
1649
1650 Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
1651 if (Pt == NULL)
1652 {
1653 KEBUGCHECK(0);
1654 }
1655 tmpPte = SwapEntry << 1;
1656 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1657 if (PAE_PAGE_MASK((Pte)) != 0)
1658 {
1659 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1660 }
1661
1662 if (Pte != 0)
1663 {
1664 MiFlushTlb((PULONG)Pt, Address);
1665 }
1666 else
1667 {
1668 MmUnmapPageTable((PULONG)Pt);
1669 }
1670 }
1671 else
1672 {
1673 PULONG Pt;
1674 ULONG Pte;
1675
1676 Pt = MmGetPageTableForProcess(Process, Address, TRUE);
1677 if (Pt == NULL)
1678 {
1679 KEBUGCHECK(0);
1680 }
1681 Pte = *Pt;
1682 if (PAGE_MASK((Pte)) != 0)
1683 {
1684 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1685 }
1686 InterlockedExchangeUL(Pt, SwapEntry << 1);
1687 if (Pte != 0)
1688 {
1689 MiFlushTlb(Pt, Address);
1690 }
1691 else
1692 {
1693 MmUnmapPageTable(Pt);
1694 }
1695 }
1696 if (Process != NULL &&
1697 Process->AddressSpace.PageTableRefCountTable != NULL &&
1698 Address < MmSystemRangeStart)
1699 {
1700 PUSHORT Ptrc;
1701 ULONG Idx;
1702
1703 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1704 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1705 Ptrc[Idx]++;
1706 }
1707 return(STATUS_SUCCESS);
1708 }
1709
1710
1711 NTSTATUS
1712 NTAPI
1713 MmCreateVirtualMappingUnsafe(PEPROCESS Process,
1714 PVOID Address,
1715 ULONG flProtect,
1716 PPFN_TYPE Pages,
1717 ULONG PageCount)
1718 {
1719 ULONG Attributes;
1720 PVOID Addr;
1721 ULONG i;
1722 ULONG oldPdeOffset, PdeOffset;
1723 BOOLEAN NoExecute = FALSE;
1724
1725 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1726 Process, Address, flProtect, Pages, *Pages, PageCount);
1727
1728 if (Process == NULL)
1729 {
1730 if (Address < MmSystemRangeStart)
1731 {
1732 DPRINT1("No process\n");
1733 KEBUGCHECK(0);
1734 }
1735 if (PageCount > 0x10000 ||
1736 (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
1737 {
1738 DPRINT1("Page count to large\n");
1739 KEBUGCHECK(0);
1740 }
1741 }
1742 else
1743 {
1744 if (Address >= MmSystemRangeStart)
1745 {
1746 DPRINT1("Setting kernel address with process context\n");
1747 KEBUGCHECK(0);
1748 }
1749 if (PageCount > (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE ||
1750 (ULONG_PTR) Address / PAGE_SIZE + PageCount >
1751 (ULONG_PTR)MmSystemRangeStart / PAGE_SIZE)
1752 {
1753 DPRINT1("Page Count to large\n");
1754 KEBUGCHECK(0);
1755 }
1756 }
1757
1758 Attributes = ProtectToPTE(flProtect);
1759 if (Attributes & 0x80000000)
1760 {
1761 NoExecute = TRUE;
1762 }
1763 Attributes &= 0xfff;
1764 if (Address >= MmSystemRangeStart)
1765 {
1766 Attributes &= ~PA_USER;
1767 if (Ke386GlobalPagesEnabled)
1768 {
1769 Attributes |= PA_GLOBAL;
1770 }
1771 }
1772 else
1773 {
1774 Attributes |= PA_USER;
1775 }
1776
1777 Addr = Address;
1778
1779 if (Ke386Pae)
1780 {
1781 ULONGLONG Pte, tmpPte;
1782 PULONGLONG Pt = NULL;
1783
1784 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1785 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1786 {
1787 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1788 {
1789 DPRINT1("Setting physical address but not allowing access at address "
1790 "0x%.8X with attributes %x/%x.\n",
1791 Addr, Attributes, flProtect);
1792 KEBUGCHECK(0);
1793 }
1794 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1795 if (oldPdeOffset != PdeOffset)
1796 {
1797 MmUnmapPageTable((PULONG)Pt);
1798 Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
1799 if (Pt == NULL)
1800 {
1801 KEBUGCHECK(0);
1802 }
1803 }
1804 else
1805 {
1806 Pt++;
1807 }
1808 oldPdeOffset = PdeOffset;
1809
1810 MmMarkPageMapped(Pages[i]);
1811 tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
1812 if (NoExecute)
1813 {
1814 tmpPte |= 0x8000000000000000LL;
1815 }
1816 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1817 if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
1818 {
1819 KEBUGCHECK(0);
1820 }
1821 if (PAE_PAGE_MASK((Pte)) != 0LL)
1822 {
1823 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1824 }
1825 if (Address < MmSystemRangeStart &&
1826 Process->AddressSpace.PageTableRefCountTable != NULL &&
1827 Attributes & PA_PRESENT)
1828 {
1829 PUSHORT Ptrc;
1830
1831 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1832
1833 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
1834 }
1835 if (Pte != 0LL)
1836 {
1837 if (Address > MmSystemRangeStart ||
1838 (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
1839 {
1840 MiFlushTlb((PULONG)Pt, Address);
1841 }
1842 }
1843 }
1844 if (Addr > Address)
1845 {
1846 MmUnmapPageTable((PULONG)Pt);
1847 }
1848 }
1849 else
1850 {
1851 PULONG Pt = NULL;
1852 ULONG Pte;
1853 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
1854 for (i = 0; i < PageCount; i++, Addr = (PVOID)((ULONG_PTR)Addr + PAGE_SIZE))
1855 {
1856 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1857 {
1858 DPRINT1("Setting physical address but not allowing access at address "
1859 "0x%.8X with attributes %x/%x.\n",
1860 Addr, Attributes, flProtect);
1861 KEBUGCHECK(0);
1862 }
1863 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1864 if (oldPdeOffset != PdeOffset)
1865 {
1866 MmUnmapPageTable(Pt);
1867 Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
1868 if (Pt == NULL)
1869 {
1870 KEBUGCHECK(0);
1871 }
1872 }
1873 else
1874 {
1875 Pt++;
1876 }
1877 oldPdeOffset = PdeOffset;
1878
1879 Pte = *Pt;
1880 MmMarkPageMapped(Pages[i]);
1881 if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
1882 {
1883 KEBUGCHECK(0);
1884 }
1885 if (PAGE_MASK((Pte)) != 0)
1886 {
1887 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1888 }
1889 InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1890 if (Address < MmSystemRangeStart &&
1891 Process->AddressSpace.PageTableRefCountTable != NULL &&
1892 Attributes & PA_PRESENT)
1893 {
1894 PUSHORT Ptrc;
1895
1896 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1897
1898 Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
1899 }
1900 if (Pte != 0)
1901 {
1902 if (Address > MmSystemRangeStart ||
1903 (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
1904 {
1905 MiFlushTlb(Pt, Address);
1906 }
1907 }
1908 }
1909 if (Addr > Address)
1910 {
1911 MmUnmapPageTable(Pt);
1912 }
1913 }
1914 return(STATUS_SUCCESS);
1915 }
1916
1917 NTSTATUS
1918 NTAPI
1919 MmCreateVirtualMapping(PEPROCESS Process,
1920 PVOID Address,
1921 ULONG flProtect,
1922 PPFN_TYPE Pages,
1923 ULONG PageCount)
1924 {
1925 ULONG i;
1926
1927 for (i = 0; i < PageCount; i++)
1928 {
1929 if (!MmIsUsablePage(Pages[i]))
1930 {
1931 DPRINT1("Page at address %x not usable\n", PFN_TO_PTE(Pages[i]));
1932 KEBUGCHECK(0);
1933 }
1934 }
1935
1936 return(MmCreateVirtualMappingUnsafe(Process,
1937 Address,
1938 flProtect,
1939 Pages,
1940 PageCount));
1941 }
1942
1943 ULONG
1944 NTAPI
1945 MmGetPageProtect(PEPROCESS Process, PVOID Address)
1946 {
1947 ULONG Entry;
1948 ULONG Protect;
1949 if (Ke386Pae)
1950 {
1951 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1952 }
1953 else
1954 {
1955 Entry = MmGetPageEntryForProcess(Process, Address);
1956 }
1957
1958 if (!(Entry & PA_PRESENT))
1959 {
1960 Protect = PAGE_NOACCESS;
1961 }
1962 else
1963 {
1964 if (Entry & PA_READWRITE)
1965 {
1966 Protect = PAGE_READWRITE;
1967 }
1968 else
1969 {
1970 Protect = PAGE_EXECUTE_READ;
1971 }
1972 if (Entry & PA_CD)
1973 {
1974 Protect |= PAGE_NOCACHE;
1975 }
1976 if (Entry & PA_WT)
1977 {
1978 Protect |= PAGE_WRITETHROUGH;
1979 }
1980 if (!(Entry & PA_USER))
1981 {
1982 Protect |= PAGE_SYSTEM;
1983 }
1984
1985 }
1986 return(Protect);
1987 }
1988
1989 VOID
1990 NTAPI
1991 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
1992 {
1993 ULONG Attributes = 0;
1994 BOOLEAN NoExecute = FALSE;
1995
1996 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
1997 Process, Address, flProtect);
1998
1999 Attributes = ProtectToPTE(flProtect);
2000 if (Attributes & 0x80000000)
2001 {
2002 NoExecute = TRUE;
2003 }
2004 Attributes &= 0xfff;
2005 if (Address >= MmSystemRangeStart)
2006 {
2007 Attributes &= ~PA_USER;
2008 if (Ke386GlobalPagesEnabled)
2009 {
2010 Attributes |= PA_GLOBAL;
2011 }
2012 }
2013 else
2014 {
2015 Attributes |= PA_USER;
2016 }
2017 if (Ke386Pae)
2018 {
2019 PULONGLONG Pt;
2020 ULONGLONG tmpPte, Pte;
2021
2022 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
2023 if (Pt == NULL)
2024 {
2025 DPRINT1("Address %x\n", Address);
2026 KEBUGCHECK(0);
2027 }
2028 do
2029 {
2030 Pte = *Pt;
2031 tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
2032 if (NoExecute)
2033 {
2034 tmpPte |= 0x8000000000000000LL;
2035 }
2036 else
2037 {
2038 tmpPte &= ~0x8000000000000000LL;
2039 }
2040 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
2041
2042 MiFlushTlb((PULONG)Pt, Address);
2043 }
2044 else
2045 {
2046 PULONG Pt;
2047
2048 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
2049 if (Pt == NULL)
2050 {
2051 KEBUGCHECK(0);
2052 }
2053 InterlockedExchange((PLONG)Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
2054 MiFlushTlb(Pt, Address);
2055 }
2056 }
2057
2058 /*
2059 * @implemented
2060 */
2061 PHYSICAL_ADDRESS STDCALL
2062 MmGetPhysicalAddress(PVOID vaddr)
2063 /*
2064 * FUNCTION: Returns the physical address corresponding to a virtual address
2065 */
2066 {
2067 PHYSICAL_ADDRESS p;
2068
2069 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr);
2070 if (Ke386Pae)
2071 {
2072 ULONGLONG Pte;
2073 Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr);
2074 if (Pte != 0 && Pte & PA_PRESENT)
2075 {
2076 p.QuadPart = PAE_PAGE_MASK(Pte);
2077 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2078 }
2079 else
2080 {
2081 p.QuadPart = 0;
2082 }
2083 }
2084 else
2085 {
2086 ULONG Pte;
2087 Pte = MmGetPageEntryForProcess(NULL, vaddr);
2088 if (Pte != 0 && Pte & PA_PRESENT)
2089 {
2090 p.QuadPart = PAGE_MASK(Pte);
2091 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2092 }
2093 else
2094 {
2095 p.QuadPart = 0;
2096 }
2097 }
2098 return p;
2099 }
2100
2101 PVOID
2102 NTAPI
2103 MmCreateHyperspaceMapping(PFN_TYPE Page)
2104 {
2105 PVOID Address;
2106 ULONG i;
2107
2108 if (Ke386Pae)
2109 {
2110 ULONGLONG Entry;
2111 ULONGLONG ZeroEntry = 0LL;
2112 PULONGLONG Pte;
2113
2114 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2115 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2116
2117 if (Page & 1024)
2118 {
2119 for (i = Page %1024; i < 1024; i++, Pte++)
2120 {
2121 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2122 {
2123 break;
2124 }
2125 }
2126 if (i >= 1024)
2127 {
2128 Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
2129 for (i = 0; i < Page % 1024; i++, Pte++)
2130 {
2131 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2132 {
2133 break;
2134 }
2135 }
2136 if (i >= Page % 1024)
2137 {
2138 KEBUGCHECK(0);
2139 }
2140 }
2141 }
2142 else
2143 {
2144 for (i = Page %1024; (LONG)i >= 0; i--, Pte--)
2145 {
2146 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2147 {
2148 break;
2149 }
2150 }
2151 if ((LONG)i < 0)
2152 {
2153 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
2154 for (i = 1023; i > Page % 1024; i--, Pte--)
2155 {
2156 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2157 {
2158 break;
2159 }
2160 }
2161 if (i <= Page % 1024)
2162 {
2163 KEBUGCHECK(0);
2164 }
2165 }
2166 }
2167 }
2168 else
2169 {
2170 ULONG Entry;
2171 PULONG Pte;
2172 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2173 Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2174 if (Page & 1024)
2175 {
2176 for (i = Page % 1024; i < 1024; i++, Pte++)
2177 {
2178 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2179 {
2180 break;
2181 }
2182 }
2183 if (i >= 1024)
2184 {
2185 Pte = ADDR_TO_PTE(HYPERSPACE);
2186 for (i = 0; i < Page % 1024; i++, Pte++)
2187 {
2188 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2189 {
2190 break;
2191 }
2192 }
2193 if (i >= Page % 1024)
2194 {
2195 KEBUGCHECK(0);
2196 }
2197 }
2198 }
2199 else
2200 {
2201 for (i = Page % 1024; (LONG)i >= 0; i--, Pte--)
2202 {
2203 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2204 {
2205 break;
2206 }
2207 }
2208 if ((LONG)i < 0)
2209 {
2210 Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
2211 for (i = 1023; i > Page % 1024; i--, Pte--)
2212 {
2213 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2214 {
2215 break;
2216 }
2217 }
2218 if (i <= Page % 1024)
2219 {
2220 KEBUGCHECK(0);
2221 }
2222 }
2223 }
2224 }
2225 Address = (PVOID)((ULONG_PTR)HYPERSPACE + i * PAGE_SIZE);
2226 FLUSH_TLB_ONE(Address);
2227 return Address;
2228 }
2229
2230 PFN_TYPE
2231 NTAPI
2232 MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage)
2233 {
2234 PFN_TYPE Pfn;
2235 ASSERT (IS_HYPERSPACE(Address));
2236 if (Ke386Pae)
2237 {
2238 ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE;
2239 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2240 Pfn = PAE_PTE_TO_PFN(Entry);
2241 }
2242 else
2243 {
2244 ULONG Entry;
2245 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE);
2246 Pfn = PTE_TO_PFN(Entry);
2247 }
2248 FLUSH_TLB_ONE(Address);
2249 return Pfn;
2250 }
2251
2252 PFN_TYPE
2253 NTAPI
2254 MmDeleteHyperspaceMapping(PVOID Address)
2255 {
2256 PFN_TYPE Pfn;
2257 ASSERT (IS_HYPERSPACE(Address));
2258 if (Ke386Pae)
2259 {
2260 ULONGLONG Entry = 0LL;
2261 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2262 Pfn = PAE_PTE_TO_PFN(Entry);
2263 }
2264 else
2265 {
2266 ULONG Entry;
2267 Entry = InterlockedExchange((PLONG)ADDR_TO_PTE(Address), 0);
2268 Pfn = PTE_TO_PFN(Entry);
2269 }
2270 FLUSH_TLB_ONE(Address);
2271 return Pfn;
2272 }
2273
2274 VOID
2275 NTAPI
2276 MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
2277 {
2278 ULONG StartOffset, EndOffset, Offset;
2279
2280 if (Address < MmSystemRangeStart)
2281 {
2282 KEBUGCHECK(0);
2283 }
2284 if (Ke386Pae)
2285 {
2286 PULONGLONG PageDirTable;
2287 PULONGLONG Pde;
2288 ULONGLONG ZeroPde = 0LL;
2289 ULONG i;
2290
2291 for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)); i++)
2292 {
2293 if (i == PAE_ADDR_TO_PDTE_OFFSET(Address))
2294 {
2295 StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
2296 }
2297 else
2298 {
2299 StartOffset = 0;
2300 }
2301 if (i == PAE_ADDR_TO_PDTE_OFFSET((PVOID)((ULONG_PTR)Address + Size)))
2302 {
2303 EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2304 }
2305 else
2306 {
2307 EndOffset = 511;
2308 }
2309
2310 if (Process != NULL && Process != PsGetCurrentProcess())
2311 {
2312 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
2313 Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i]));
2314 MmDeleteHyperspaceMapping(PageDirTable);
2315 }
2316 else
2317 {
2318 Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512;
2319 }
2320
2321 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2322 {
2323 if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4)
2324 {
2325 ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde);
2326 }
2327 }
2328 MmUnmapPageTable((PULONG)Pde);
2329 }
2330 }
2331 else
2332 {
2333 PULONG Pde;
2334 StartOffset = ADDR_TO_PDE_OFFSET(Address);
2335 EndOffset = ADDR_TO_PDE_OFFSET((PVOID)((ULONG_PTR)Address + Size));
2336
2337 if (Process != NULL && Process != PsGetCurrentProcess())
2338 {
2339 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
2340 }
2341 else
2342 {
2343 Pde = (PULONG)PAGEDIRECTORY_MAP;
2344 }
2345 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2346 {
2347 if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP))
2348 {
2349 InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0);
2350 }
2351 }
2352 if (Pde != (PULONG)PAGEDIRECTORY_MAP)
2353 {
2354 MmDeleteHyperspaceMapping(Pde);
2355 }
2356 }
2357 }
2358
2359 VOID
2360 INIT_FUNCTION
2361 NTAPI
2362 MmInitGlobalKernelPageDirectory(VOID)
2363 {
2364 ULONG i;
2365
2366 DPRINT("MmInitGlobalKernelPageDirectory()\n");
2367
2368 if (Ke386Pae)
2369 {
2370 PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
2371 for (i = PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 4 * 512; i++)
2372 {
2373 if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
2374 !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
2375 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
2376 {
2377 ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
2378 if (Ke386GlobalPagesEnabled)
2379 {
2380 MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
2381 CurrentPageDirectory[i] |= PA_GLOBAL;
2382 }
2383 }
2384 }
2385 }
2386 else
2387 {
2388 PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
2389 for (i = ADDR_TO_PDE_OFFSET(MmSystemRangeStart); i < 1024; i++)
2390 {
2391 if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
2392 i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
2393 0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
2394 {
2395 MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
2396 if (Ke386GlobalPagesEnabled)
2397 {
2398 MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
2399 CurrentPageDirectory[i] |= PA_GLOBAL;
2400 }
2401 }
2402 }
2403 }
2404 }
2405
2406 ULONG
2407 NTAPI
2408 MiGetUserPageDirectoryCount(VOID)
2409 {
2410 return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart);
2411 }
2412
2413 VOID
2414 INIT_FUNCTION
2415 NTAPI
2416 MiInitPageDirectoryMap(VOID)
2417 {
2418 MEMORY_AREA* kernel_map_desc = NULL;
2419 MEMORY_AREA* hyperspace_desc = NULL;
2420 PHYSICAL_ADDRESS BoundaryAddressMultiple;
2421 PVOID BaseAddress;
2422 NTSTATUS Status;
2423
2424 DPRINT("MiInitPageDirectoryMap()\n");
2425
2426 BoundaryAddressMultiple.QuadPart = 0;
2427 BaseAddress = (PVOID)PAGETABLE_MAP;
2428 Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
2429 MEMORY_AREA_SYSTEM,
2430 &BaseAddress,
2431 Ke386Pae ? 0x800000 : 0x400000,
2432 PAGE_READWRITE,
2433 &kernel_map_desc,
2434 TRUE,
2435 0,
2436 BoundaryAddressMultiple);
2437 if (!NT_SUCCESS(Status))
2438 {
2439 KEBUGCHECK(0);
2440 }
2441 BaseAddress = (PVOID)HYPERSPACE;
2442 Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
2443 MEMORY_AREA_SYSTEM,
2444 &BaseAddress,
2445 0x400000,
2446 PAGE_READWRITE,
2447 &hyperspace_desc,
2448 TRUE,
2449 0,
2450 BoundaryAddressMultiple);
2451 if (!NT_SUCCESS(Status))
2452 {
2453 KEBUGCHECK(0);
2454 }
2455 }
2456
2457 /* EOF */
A free Windows-compatible Operating System