2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/i386/page.c
5 * PURPOSE: Low level memory managment manipulation
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
10 /* INCLUDES ***************************************************************/
16 #if defined (ALLOC_PRAGMA)
17 #pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
18 #pragma alloc_text(INIT, MiInitPageDirectoryMap)
22 /* GLOBALS *****************************************************************/
24 #define PA_BIT_PRESENT (0)
25 #define PA_BIT_READWRITE (1)
26 #define PA_BIT_USER (2)
29 #define PA_BIT_ACCESSED (5)
30 #define PA_BIT_DIRTY (6)
31 #define PA_BIT_GLOBAL (8)
33 #define PA_PRESENT (1 << PA_BIT_PRESENT)
34 #define PA_READWRITE (1 << PA_BIT_READWRITE)
35 #define PA_USER (1 << PA_BIT_USER)
36 #define PA_DIRTY (1 << PA_BIT_DIRTY)
37 #define PA_WT (1 << PA_BIT_WT)
38 #define PA_CD (1 << PA_BIT_CD)
39 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
40 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
42 #define PAGETABLE_MAP (0xc0000000)
43 #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024)))
45 #define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512)))
47 #define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000)
48 #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
50 ULONG MmGlobalKernelPageDirectory
[1024];
51 ULONGLONG MmGlobalKernelPageDirectoryForPAE
[2048];
53 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
54 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
56 #define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
57 #define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
60 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
62 __inline LARGE_INTEGER
PTE_TO_PAGE(ULONG npage
)
65 dummy
.QuadPart
= (LONGLONG
)(PAGE_MASK(npage
));
70 extern BOOLEAN Ke386Pae
;
71 extern BOOLEAN Ke386NoExecute
;
73 /* FUNCTIONS ***************************************************************/
75 BOOLEAN
MmUnmapPageTable(PULONG Pt
);
79 MiFlushTlbIpiRoutine(ULONG_PTR Address
)
81 if (Address
== (ULONGLONG
)-1)
85 else if (Address
== (ULONGLONG
)-2)
91 __invlpg((PVOID
)Address
);
97 MiFlushTlb(PULONG Pt
, PVOID Address
)
102 MmUnmapPageTable(Pt
);
104 if (KeNumberProcessors
> 1)
106 KeIpiGenericCall(MiFlushTlbIpiRoutine
, (ULONG_PTR
)Address
);
110 MiFlushTlbIpiRoutine((ULONG_PTR
)Address
);
113 if ((Pt
&& MmUnmapPageTable(Pt
)) || Address
>= MmSystemRangeStart
)
123 MmGetPageDirectory(VOID
)
125 return (PULONG
)__readcr3();
129 ProtectToPTE(ULONG flProtect
)
131 ULONG Attributes
= 0;
133 if (flProtect
& (PAGE_NOACCESS
|PAGE_GUARD
))
137 else if (flProtect
& PAGE_IS_WRITABLE
)
139 Attributes
= PA_PRESENT
| PA_READWRITE
;
141 else if (flProtect
& (PAGE_IS_READABLE
| PAGE_IS_EXECUTABLE
))
143 Attributes
= PA_PRESENT
;
147 DPRINT1("Unknown main protection type.\n");
150 if (Ke386NoExecute
&&
151 !(flProtect
& PAGE_IS_EXECUTABLE
))
153 Attributes
= Attributes
| 0x80000000;
156 if (flProtect
& PAGE_SYSTEM
)
161 Attributes
= Attributes
| PA_USER
;
163 if (flProtect
& PAGE_NOCACHE
)
165 Attributes
= Attributes
| PA_CD
;
167 if (flProtect
& PAGE_WRITETHROUGH
)
169 Attributes
= Attributes
| PA_WT
;
174 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
176 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
177 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
178 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
180 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
182 #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
185 #define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE))
187 #define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
188 ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
189 #define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
192 #define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
194 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
196 #define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
198 #define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
203 Mmi386ReleaseMmInfo(PEPROCESS Process
)
205 PUSHORT LdtDescriptor
;
209 DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process
);
211 LdtDescriptor
= (PUSHORT
) &Process
->Pcb
.LdtDescriptor
;
212 LdtBase
= LdtDescriptor
[1] |
213 ((LdtDescriptor
[2] & 0xff) << 16) |
214 ((LdtDescriptor
[3] & ~0xff) << 16);
216 DPRINT("LdtBase: %x\n", LdtBase
);
220 ExFreePool((PVOID
) LdtBase
);
225 PULONGLONG PageDirTable
;
230 PageDirTable
= (PULONGLONG
)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
231 for (i
= 0; i
< 4; i
++)
233 PageDir
= (PULONGLONG
)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable
[i
]));
234 if (i
< PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart
))
236 for (j
= 0; j
< 512; j
++)
238 if (PageDir
[j
] != 0LL)
240 DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
241 Process
->UniqueProcessId
,
242 (i
* 512 + j
) * 512 * PAGE_SIZE
, (i
* 512 + j
+ 1) * 512 * PAGE_SIZE
- 1,
243 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
[i
*512 + j
]);
244 Pde
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir
[j
]));
245 for (k
= 0; k
< 512; k
++)
249 if (Pde
[k
] & PA_PRESENT
)
251 DPRINT1("Page at %08x is not freed\n",
252 (i
* 512 + j
) * 512 * PAGE_SIZE
+ k
* PAGE_SIZE
);
256 DPRINT1("Swapentry %x at %x is not freed\n",
257 (i
* 512 + j
) * 512 * PAGE_SIZE
+ k
* PAGE_SIZE
);
261 MmDeleteHyperspaceMapping(Pde
);
262 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDir
[j
]));
266 if (i
== PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE
))
268 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)]));
269 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)+1]));
271 MmDeleteHyperspaceMapping(PageDir
);
272 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDirTable
[i
]));
274 MmDeleteHyperspaceMapping((PVOID
)PageDirTable
);
275 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
281 PageDir
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.u
.LowPart
));
282 for (i
= 0; i
< ADDR_TO_PDE_OFFSET(MmSystemRangeStart
); i
++)
286 DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
287 i
* 4 * 1024 * 1024, (i
+ 1) * 4 * 1024 * 1024 - 1,
288 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
[i
]);
289 Pde
= MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir
[i
]));
290 for (j
= 0; j
< 1024; j
++)
294 if (Pde
[j
] & PA_PRESENT
)
296 DPRINT1("Page at %08x is not freed\n",
297 i
* 4 * 1024 * 1024 + j
* PAGE_SIZE
);
301 DPRINT1("Swapentry %x at %x is not freed\n",
302 Pde
[j
], i
* 4 * 1024 * 1024 + j
* PAGE_SIZE
);
306 MmDeleteHyperspaceMapping(Pde
);
307 MmReleasePageMemoryConsumer(MC_NPPOOL
, PTE_TO_PFN(PageDir
[i
]));
310 MmReleasePageMemoryConsumer(MC_NPPOOL
, PTE_TO_PFN(PageDir
[ADDR_TO_PDE_OFFSET(HYPERSPACE
)]));
311 MmDeleteHyperspaceMapping(PageDir
);
312 MmReleasePageMemoryConsumer(MC_NPPOOL
, PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.u
.LowPart
));
315 #if defined(__GNUC__)
317 Process
->Pcb
.DirectoryTableBase
.QuadPart
= 0LL;
320 Process
->Pcb
.DirectoryTableBase
.QuadPart
= 0;
323 DPRINT("Finished Mmi386ReleaseMmInfo()\n");
324 return(STATUS_SUCCESS
);
329 MmInitializeHandBuiltProcess(IN PEPROCESS Process
,
330 IN PLARGE_INTEGER DirectoryTableBase
)
332 /* Share the directory base with the idle process */
333 *DirectoryTableBase
= PsGetCurrentProcess()->Pcb
.DirectoryTableBase
;
335 /* Initialize the Addresss Space */
336 MmInitializeAddressSpace(Process
, (PMADDRESS_SPACE
)&Process
->VadRoot
);
338 /* The process now has an address space */
339 Process
->HasAddressSpace
= TRUE
;
340 return STATUS_SUCCESS
;
345 MmCreateProcessAddressSpace(IN ULONG MinWs
,
346 IN PEPROCESS Process
,
347 IN PLARGE_INTEGER DirectoryTableBase
)
354 DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs
, Process
);
356 Count
= Ke386Pae
? 7 : 2;
358 for (i
= 0; i
< Count
; i
++)
360 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
[i
]);
361 if (!NT_SUCCESS(Status
))
363 for (j
= 0; j
< i
; j
++)
365 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
[j
]);
374 PULONGLONG PageDirTable
;
377 PageDirTable
= MmCreateHyperspaceMapping(Pfn
[0]);
378 for (i
= 0; i
< 4; i
++)
380 PageDirTable
[i
] = PAE_PFN_TO_PTE(Pfn
[1+i
]) | PA_PRESENT
;
382 MmDeleteHyperspaceMapping(PageDirTable
);
383 for (i
= PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart
); i
< 4; i
++)
385 PageDir
= (PULONGLONG
)MmCreateHyperspaceMapping(Pfn
[i
+1]);
386 memcpy(PageDir
, &MmGlobalKernelPageDirectoryForPAE
[i
* 512], 512 * sizeof(ULONGLONG
));
387 if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP
) == i
)
389 for (j
= 0; j
< 4; j
++)
391 PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP
) + j
] = PAE_PFN_TO_PTE(Pfn
[1+j
]) | PA_PRESENT
| PA_READWRITE
;
394 if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE
) == i
)
396 PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)] = PAE_PFN_TO_PTE(Pfn
[5]) | PA_PRESENT
| PA_READWRITE
;
397 PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)+1] = PAE_PFN_TO_PTE(Pfn
[6]) | PA_PRESENT
| PA_READWRITE
;
399 MmDeleteHyperspaceMapping(PageDir
);
404 PULONG PageDirectory
;
405 PageDirectory
= MmCreateHyperspaceMapping(Pfn
[0]);
407 memcpy(PageDirectory
+ ADDR_TO_PDE_OFFSET(MmSystemRangeStart
),
408 MmGlobalKernelPageDirectory
+ ADDR_TO_PDE_OFFSET(MmSystemRangeStart
),
409 (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart
)) * sizeof(ULONG
));
411 DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
));
412 PageDirectory
[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
)] = PFN_TO_PTE(Pfn
[0]) | PA_PRESENT
| PA_READWRITE
;
413 PageDirectory
[ADDR_TO_PDE_OFFSET(HYPERSPACE
)] = PFN_TO_PTE(Pfn
[1]) | PA_PRESENT
| PA_READWRITE
;
415 MmDeleteHyperspaceMapping(PageDirectory
);
418 DirectoryTableBase
->QuadPart
= PFN_TO_PTE(Pfn
[0]);
419 DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase
->QuadPart
);
425 MmDeletePageTable(PEPROCESS Process
, PVOID Address
)
427 PEPROCESS CurrentProcess
= PsGetCurrentProcess();
429 if (Process
!= NULL
&& Process
!= CurrentProcess
)
431 KeAttachProcess(&Process
->Pcb
);
436 ULONGLONG ZeroPde
= 0LL;
437 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address
), &ZeroPde
);
438 MiFlushTlb((PULONG
)PAE_ADDR_TO_PDE(Address
), PAE_ADDR_TO_PTE(Address
));
442 *(ADDR_TO_PDE(Address
)) = 0;
443 MiFlushTlb(ADDR_TO_PDE(Address
), ADDR_TO_PTE(Address
));
445 if (Address
>= MmSystemRangeStart
)
448 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
450 if (Process
!= NULL
&& Process
!= CurrentProcess
)
458 MmFreePageTable(PEPROCESS Process
, PVOID Address
)
460 PEPROCESS CurrentProcess
= PsGetCurrentProcess();
464 DPRINT("ProcessId %d, Address %x\n", Process
->UniqueProcessId
, Address
);
465 if (Process
!= NULL
&& Process
!= CurrentProcess
)
467 KeAttachProcess(&Process
->Pcb
);
471 PULONGLONG PageTable
;
472 ULONGLONG ZeroPte
= 0LL;
473 PageTable
= (PULONGLONG
)PAGE_ROUND_DOWN((PVOID
)PAE_ADDR_TO_PTE(Address
));
474 for (i
= 0; i
< 512; i
++)
476 if (PageTable
[i
] != 0LL)
478 DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
479 ((ULONG
)Address
/ (4*1024*1024)), i
, PageTable
[i
]);
483 Pfn
= PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address
)));
484 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address
), &ZeroPte
);
485 MiFlushTlb((PULONG
)PAE_ADDR_TO_PDE(Address
), PAE_ADDR_TO_PTE(Address
));
490 PageTable
= (PULONG
)PAGE_ROUND_DOWN((PVOID
)ADDR_TO_PTE(Address
));
491 for (i
= 0; i
< 1024; i
++)
493 if (PageTable
[i
] != 0)
495 DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
496 ((ULONG
)Address
/ (4*1024*1024)), i
, PageTable
[i
]);
500 Pfn
= PTE_TO_PFN(*(ADDR_TO_PDE(Address
)));
501 *(ADDR_TO_PDE(Address
)) = 0;
502 MiFlushTlb(ADDR_TO_PDE(Address
), ADDR_TO_PTE(Address
));
505 if (Address
>= MmSystemRangeStart
)
507 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
512 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
514 if (Process
!= NULL
&& Process
!= CurrentProcess
)
521 MmGetPageTableForProcessForPAE(PEPROCESS Process
, PVOID Address
, BOOLEAN Create
)
526 ULONGLONG ZeroEntry
= 0LL;
529 PULONGLONG PageDirTable
;
531 DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
532 Process
, Address
, Create
);
533 if (Address
>= (PVOID
)PAGETABLE_MAP
&& Address
< (PVOID
)((ULONG_PTR
)PAGETABLE_MAP
+ 0x800000))
537 if (Address
< MmSystemRangeStart
&& Process
&& Process
!= PsGetCurrentProcess())
539 PageDirTable
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
540 if (PageDirTable
== NULL
)
544 PageDir
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable
[PAE_ADDR_TO_PDTE_OFFSET(Address
)]));
545 MmDeleteHyperspaceMapping(PageDirTable
);
550 PageDir
+= PAE_ADDR_TO_PDE_PAGE_OFFSET(Address
);
551 Entry
= ExfInterlockedCompareExchange64UL(PageDir
, &ZeroEntry
, &ZeroEntry
);
556 MmDeleteHyperspaceMapping(PageDir
);
559 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
560 if (!NT_SUCCESS(Status
))
564 Entry
= PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
;
565 Entry
= ExfInterlockedCompareExchange64UL(PageDir
, &Entry
, &ZeroEntry
);
568 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
569 Pfn
= PAE_PTE_TO_PFN(Entry
);
574 Pfn
= PAE_PTE_TO_PFN(Entry
);
576 MmDeleteHyperspaceMapping(PageDir
);
577 Pt
= MmCreateHyperspaceMapping(Pfn
);
582 return Pt
+ PAE_ADDR_TO_PTE_OFFSET(Address
);
584 PageDir
= PAE_ADDR_TO_PDE(Address
);
585 if (0LL == ExfInterlockedCompareExchange64UL(PageDir
, &ZeroEntry
, &ZeroEntry
))
587 if (Address
>= MmSystemRangeStart
)
589 if (MmGlobalKernelPageDirectoryForPAE
[PAE_ADDR_TO_PDE_OFFSET(Address
)] == 0LL)
595 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
596 if (!NT_SUCCESS(Status
))
600 Entry
= PAE_PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
;
601 if (Ke386GlobalPagesEnabled
)
605 if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE
[PAE_ADDR_TO_PDE_OFFSET(Address
)], &Entry
, &ZeroEntry
))
607 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
610 (void)ExfInterlockedCompareExchange64UL(PageDir
, &MmGlobalKernelPageDirectoryForPAE
[PAE_ADDR_TO_PDE_OFFSET(Address
)], &ZeroEntry
);
618 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
619 if (!NT_SUCCESS(Status
))
623 Entry
= PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
;
624 Entry
= ExfInterlockedCompareExchange64UL(PageDir
, &Entry
, &ZeroEntry
);
627 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
631 return (PULONGLONG
)PAE_ADDR_TO_PTE(Address
);
635 MmGetPageTableForProcess(PEPROCESS Process
, PVOID Address
, BOOLEAN Create
)
637 ULONG PdeOffset
= ADDR_TO_PDE_OFFSET(Address
);
643 if (Address
< MmSystemRangeStart
&& Process
&& Process
!= PsGetCurrentProcess())
645 PageDir
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.LowPart
));
650 if (0 == InterlockedCompareExchangeUL(&PageDir
[PdeOffset
], 0, 0))
654 MmDeleteHyperspaceMapping(PageDir
);
657 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
658 if (!NT_SUCCESS(Status
) || Pfn
== 0)
662 Entry
= InterlockedCompareExchangeUL(&PageDir
[PdeOffset
], PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
, 0);
665 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
666 Pfn
= PTE_TO_PFN(Entry
);
671 Pfn
= PTE_TO_PFN(PageDir
[PdeOffset
]);
673 MmDeleteHyperspaceMapping(PageDir
);
674 Pt
= MmCreateHyperspaceMapping(Pfn
);
679 return Pt
+ ADDR_TO_PTE_OFFSET(Address
);
681 PageDir
= ADDR_TO_PDE(Address
);
682 if (0 == InterlockedCompareExchangeUL(PageDir
, 0, 0))
684 if (Address
>= MmSystemRangeStart
)
686 if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory
[PdeOffset
], 0, 0))
692 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
693 if (!NT_SUCCESS(Status
) || Pfn
== 0)
697 Entry
= PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
;
698 if (Ke386GlobalPagesEnabled
)
702 if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory
[PdeOffset
], Entry
, 0))
704 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
707 (void)InterlockedExchangeUL(PageDir
, MmGlobalKernelPageDirectory
[PdeOffset
]);
715 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
716 if (!NT_SUCCESS(Status
) || Pfn
== 0)
720 Entry
= InterlockedCompareExchangeUL(PageDir
, PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
, 0);
723 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
727 return (PULONG
)ADDR_TO_PTE(Address
);
730 BOOLEAN
MmUnmapPageTable(PULONG Pt
)
734 if ((PULONGLONG
)Pt
>= (PULONGLONG
)PAGETABLE_MAP
&& (PULONGLONG
)Pt
< (PULONGLONG
)PAGETABLE_MAP
+ 4*512*512)
741 if (Pt
>= (PULONG
)PAGETABLE_MAP
&& Pt
< (PULONG
)PAGETABLE_MAP
+ 1024*1024)
748 MmDeleteHyperspaceMapping((PVOID
)PAGE_ROUND_DOWN(Pt
));
753 static ULONGLONG
MmGetPageEntryForProcessForPAE(PEPROCESS Process
, PVOID Address
)
758 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
762 MmUnmapPageTable((PULONG
)Pt
);
768 static ULONG
MmGetPageEntryForProcess(PEPROCESS Process
, PVOID Address
)
773 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
777 MmUnmapPageTable(Pt
);
785 MmGetPfnForProcess(PEPROCESS Process
,
792 Entry
= MmGetPageEntryForProcessForPAE(Process
, Address
);
793 if (!(Entry
& PA_PRESENT
))
797 return(PAE_PTE_TO_PFN(Entry
));
802 Entry
= MmGetPageEntryForProcess(Process
, Address
);
803 if (!(Entry
& PA_PRESENT
))
807 return(PTE_TO_PFN(Entry
));
813 MmDeleteVirtualMapping(PEPROCESS Process
, PVOID Address
, BOOLEAN FreePage
,
814 BOOLEAN
* WasDirty
, PPFN_NUMBER Page
)
816 * FUNCTION: Delete a virtual mapping
819 BOOLEAN WasValid
= FALSE
;
822 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
823 Process
, Address
, FreePage
, WasDirty
, Page
);
829 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
832 if (WasDirty
!= NULL
)
844 * Atomically set the entry to zero and get the old value.
847 Pte
= ExfpInterlockedExchange64UL(Pt
, &Pte
);
849 MiFlushTlb((PULONG
)Pt
, Address
);
851 WasValid
= PAE_PAGE_MASK(Pte
) != 0 ? TRUE
: FALSE
;
854 Pfn
= PAE_PTE_TO_PFN(Pte
);
855 MmMarkPageUnmapped(Pfn
);
862 if (FreePage
&& WasValid
)
864 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
868 * Return some information to the caller
870 if (WasDirty
!= NULL
)
872 *WasDirty
= Pte
& PA_DIRTY
? TRUE
: FALSE
;
884 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
888 if (WasDirty
!= NULL
)
900 * Atomically set the entry to zero and get the old value.
902 Pte
= InterlockedExchangeUL(Pt
, 0);
904 MiFlushTlb(Pt
, Address
);
906 WasValid
= (PAGE_MASK(Pte
) != 0);
909 Pfn
= PTE_TO_PFN(Pte
);
910 MmMarkPageUnmapped(Pfn
);
917 if (FreePage
&& WasValid
)
919 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
923 * Return some information to the caller
925 if (WasDirty
!= NULL
)
927 *WasDirty
= Pte
& PA_DIRTY
? TRUE
: FALSE
;
935 * Decrement the reference count for this page table.
937 if (Process
!= NULL
&& WasValid
&&
938 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
939 Address
< MmSystemRangeStart
)
944 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
945 Idx
= Ke386Pae
? PAE_ADDR_TO_PAGE_TABLE(Address
) : ADDR_TO_PAGE_TABLE(Address
);
950 MmFreePageTable(Process
, Address
);
957 MmDeletePageFileMapping(PEPROCESS Process
, PVOID Address
,
958 SWAPENTRY
* SwapEntry
)
960 * FUNCTION: Delete a virtual mapping
968 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
976 * Atomically set the entry to zero and get the old value.
979 Pte
= ExfpInterlockedExchange64UL(Pt
, &Pte
);
981 MiFlushTlb((PULONG
)Pt
, Address
);
984 * Decrement the reference count for this page table.
986 if (Process
!= NULL
&& Pte
&&
987 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
988 Address
< MmSystemRangeStart
)
992 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
994 Ptrc
[PAE_ADDR_TO_PAGE_TABLE(Address
)]--;
995 if (Ptrc
[PAE_ADDR_TO_PAGE_TABLE(Address
)] == 0)
997 MmFreePageTable(Process
, Address
);
1003 * Return some information to the caller
1005 *SwapEntry
= Pte
>> 1;
1012 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1021 * Atomically set the entry to zero and get the old value.
1023 Pte
= InterlockedExchangeUL(Pt
, 0);
1025 MiFlushTlb(Pt
, Address
);
1028 * Decrement the reference count for this page table.
1030 if (Process
!= NULL
&& Pte
&&
1031 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1032 Address
< MmSystemRangeStart
)
1036 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1038 Ptrc
[ADDR_TO_PAGE_TABLE(Address
)]--;
1039 if (Ptrc
[ADDR_TO_PAGE_TABLE(Address
)] == 0)
1041 MmFreePageTable(Process
, Address
);
1047 * Return some information to the caller
1049 *SwapEntry
= Pte
>> 1;
1054 Mmi386MakeKernelPageTableGlobal(PVOID PAddress
)
1060 Pde
= PAE_ADDR_TO_PDE(PAddress
);
1063 Pt
= MmGetPageTableForProcessForPAE(NULL
, PAddress
, FALSE
);
1065 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1066 FLASH_TLB_ONE(PAddress
);
1077 Pde
= ADDR_TO_PDE(PAddress
);
1080 Pt
= MmGetPageTableForProcess(NULL
, PAddress
, FALSE
);
1082 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1083 FLASH_TLB_ONE(PAddress
);
1096 MmIsDirtyPage(PEPROCESS Process
, PVOID Address
)
1100 return MmGetPageEntryForProcessForPAE(Process
, Address
) & PA_DIRTY
? TRUE
: FALSE
;
1104 return MmGetPageEntryForProcess(Process
, Address
) & PA_DIRTY
? TRUE
: FALSE
;
1110 MmIsAccessedAndResetAccessPage(PEPROCESS Process
, PVOID Address
)
1112 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
1114 DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1123 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1132 tmpPte
= Pte
& ~PA_ACCESSED
;
1133 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1135 if (Pte
& PA_ACCESSED
)
1137 MiFlushTlb((PULONG
)Pt
, Address
);
1142 MmUnmapPageTable((PULONG
)Pt
);
1151 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1160 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
& ~PA_ACCESSED
, Pte
));
1162 if (Pte
& PA_ACCESSED
)
1164 MiFlushTlb(Pt
, Address
);
1169 MmUnmapPageTable(Pt
);
1177 MmSetCleanPage(PEPROCESS Process
, PVOID Address
)
1179 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
1181 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1190 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1200 tmpPte
= Pte
& ~PA_DIRTY
;
1201 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1205 MiFlushTlb((PULONG
)Pt
, Address
);
1209 MmUnmapPageTable((PULONG
)Pt
);
1217 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1227 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
& ~PA_DIRTY
, Pte
));
1231 MiFlushTlb(Pt
, Address
);
1235 MmUnmapPageTable(Pt
);
1242 MmSetDirtyPage(PEPROCESS Process
, PVOID Address
)
1244 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
1246 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1255 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1264 tmpPte
= Pte
| PA_DIRTY
;
1265 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1266 if (!(Pte
& PA_DIRTY
))
1268 MiFlushTlb((PULONG
)Pt
, Address
);
1272 MmUnmapPageTable((PULONG
)Pt
);
1280 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1289 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
| PA_DIRTY
, Pte
));
1290 if (!(Pte
& PA_DIRTY
))
1292 MiFlushTlb(Pt
, Address
);
1296 MmUnmapPageTable(Pt
);
1303 MmEnableVirtualMapping(PEPROCESS Process
, PVOID Address
)
1311 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1320 tmpPte
= Pte
| PA_PRESENT
;
1321 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1322 if (!(Pte
& PA_PRESENT
))
1324 MiFlushTlb((PULONG
)Pt
, Address
);
1328 MmUnmapPageTable((PULONG
)Pt
);
1336 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1345 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
| PA_PRESENT
, Pte
));
1346 if (!(Pte
& PA_PRESENT
))
1348 MiFlushTlb(Pt
, Address
);
1352 MmUnmapPageTable(Pt
);
1359 MmIsPagePresent(PEPROCESS Process
, PVOID Address
)
1363 return MmGetPageEntryForProcessForPAE(Process
, Address
) & PA_PRESENT
? TRUE
: FALSE
;
1367 return MmGetPageEntryForProcess(Process
, Address
) & PA_PRESENT
? TRUE
: FALSE
;
1373 MmIsPageSwapEntry(PEPROCESS Process
, PVOID Address
)
1378 Entry
= MmGetPageEntryForProcessForPAE(Process
, Address
);
1379 return !(Entry
& PA_PRESENT
) && Entry
!= 0 ? TRUE
: FALSE
;
1384 Entry
= MmGetPageEntryForProcess(Process
, Address
);
1385 return !(Entry
& PA_PRESENT
) && Entry
!= 0 ? TRUE
: FALSE
;
1391 MmCreateVirtualMappingForKernel(PVOID Address
,
1399 ULONG PdeOffset
, oldPdeOffset
;
1400 BOOLEAN NoExecute
= FALSE
;
1402 DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1403 Address
, flProtect
, Pages
, PageCount
);
1405 if (Address
< MmSystemRangeStart
)
1407 DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1411 Attributes
= ProtectToPTE(flProtect
);
1412 if (Attributes
& 0x80000000)
1416 Attributes
&= 0xfff;
1417 if (Ke386GlobalPagesEnabled
)
1419 Attributes
|= PA_GLOBAL
;
1426 PULONGLONG Pt
= NULL
;
1429 oldPdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
) + 1;
1430 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1432 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1434 DPRINT1("Setting physical address but not allowing access at address "
1435 "0x%.8X with attributes %x/%x.\n",
1436 Addr
, Attributes
, flProtect
);
1440 PdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
);
1441 if (oldPdeOffset
!= PdeOffset
)
1443 Pt
= MmGetPageTableForProcessForPAE(NULL
, Addr
, TRUE
);
1453 oldPdeOffset
= PdeOffset
;
1455 Pte
= PFN_TO_PTE(Pages
[i
]) | Attributes
;
1458 Pte
|= 0x8000000000000000LL
;
1460 Pte
= ExfpInterlockedExchange64UL(Pt
, &Pte
);
1472 oldPdeOffset
= ADDR_TO_PDE_OFFSET(Addr
);
1473 Pt
= MmGetPageTableForProcess(NULL
, Addr
, TRUE
);
1480 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1482 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1484 DPRINT1("Setting physical address but not allowing access at address "
1485 "0x%.8X with attributes %x/%x.\n",
1486 Addr
, Attributes
, flProtect
);
1490 PdeOffset
= ADDR_TO_PDE_OFFSET(Addr
);
1491 if (oldPdeOffset
!= PdeOffset
)
1493 Pt
= MmGetPageTableForProcess(NULL
, Addr
, TRUE
);
1503 oldPdeOffset
= PdeOffset
;
1510 (void)InterlockedExchangeUL(Pt
, PFN_TO_PTE(Pages
[i
]) | Attributes
);
1514 return(STATUS_SUCCESS
);
1519 MmCreatePageFileMapping(PEPROCESS Process
,
1521 SWAPENTRY SwapEntry
)
1523 if (Process
== NULL
&& Address
< MmSystemRangeStart
)
1525 DPRINT1("No process\n");
1528 if (Process
!= NULL
&& Address
>= MmSystemRangeStart
)
1530 DPRINT1("Setting kernel address with process context\n");
1533 if (SwapEntry
& (1 << 31))
1544 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, TRUE
);
1549 tmpPte
= SwapEntry
<< 1;
1550 Pte
= ExfpInterlockedExchange64UL(Pt
, &tmpPte
);
1551 if (PAE_PAGE_MASK((Pte
)) != 0)
1553 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte
)));
1558 MiFlushTlb((PULONG
)Pt
, Address
);
1562 MmUnmapPageTable((PULONG
)Pt
);
1570 Pt
= MmGetPageTableForProcess(Process
, Address
, TRUE
);
1576 if (PAGE_MASK((Pte
)) != 0)
1578 MmMarkPageUnmapped(PTE_TO_PFN((Pte
)));
1580 (void)InterlockedExchangeUL(Pt
, SwapEntry
<< 1);
1583 MiFlushTlb(Pt
, Address
);
1587 MmUnmapPageTable(Pt
);
1590 if (Process
!= NULL
&&
1591 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1592 Address
< MmSystemRangeStart
)
1597 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1598 Idx
= Ke386Pae
? PAE_ADDR_TO_PAGE_TABLE(Address
) : ADDR_TO_PAGE_TABLE(Address
);
1601 return(STATUS_SUCCESS
);
1607 MmCreateVirtualMappingUnsafe(PEPROCESS Process
,
1616 ULONG oldPdeOffset
, PdeOffset
;
1617 BOOLEAN NoExecute
= FALSE
;
1619 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1620 Process
, Address
, flProtect
, Pages
, *Pages
, PageCount
);
1622 if (Process
== NULL
)
1624 if (Address
< MmSystemRangeStart
)
1626 DPRINT1("No process\n");
1629 if (PageCount
> 0x10000 ||
1630 (ULONG_PTR
) Address
/ PAGE_SIZE
+ PageCount
> 0x100000)
1632 DPRINT1("Page count to large\n");
1638 if (Address
>= MmSystemRangeStart
)
1640 DPRINT1("Setting kernel address with process context\n");
1643 if (PageCount
> (ULONG_PTR
)MmSystemRangeStart
/ PAGE_SIZE
||
1644 (ULONG_PTR
) Address
/ PAGE_SIZE
+ PageCount
>
1645 (ULONG_PTR
)MmSystemRangeStart
/ PAGE_SIZE
)
1647 DPRINT1("Page Count to large\n");
1652 Attributes
= ProtectToPTE(flProtect
);
1653 if (Attributes
& 0x80000000)
1657 Attributes
&= 0xfff;
1658 if (Address
>= MmSystemRangeStart
)
1660 Attributes
&= ~PA_USER
;
1661 if (Ke386GlobalPagesEnabled
)
1663 Attributes
|= PA_GLOBAL
;
1668 Attributes
|= PA_USER
;
1675 ULONGLONG Pte
, tmpPte
;
1676 PULONGLONG Pt
= NULL
;
1678 oldPdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
) + 1;
1679 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1681 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1683 DPRINT1("Setting physical address but not allowing access at address "
1684 "0x%.8X with attributes %x/%x.\n",
1685 Addr
, Attributes
, flProtect
);
1688 PdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
);
1689 if (oldPdeOffset
!= PdeOffset
)
1691 MmUnmapPageTable((PULONG
)Pt
);
1692 Pt
= MmGetPageTableForProcessForPAE(Process
, Addr
, TRUE
);
1702 oldPdeOffset
= PdeOffset
;
1704 MmMarkPageMapped(Pages
[i
]);
1705 tmpPte
= PAE_PFN_TO_PTE(Pages
[i
]) | Attributes
;
1708 tmpPte
|= 0x8000000000000000LL
;
1710 Pte
= ExfpInterlockedExchange64UL(Pt
, &tmpPte
);
1711 if (PAE_PAGE_MASK((Pte
)) != 0LL && !((Pte
) & PA_PRESENT
))
1715 if (PAE_PAGE_MASK((Pte
)) != 0LL)
1717 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte
)));
1719 if (Address
< MmSystemRangeStart
&&
1720 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1721 Attributes
& PA_PRESENT
)
1725 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1727 Ptrc
[PAE_ADDR_TO_PAGE_TABLE(Addr
)]++;
1731 if (Address
> MmSystemRangeStart
||
1732 (Pt
>= (PULONGLONG
)PAGETABLE_MAP
&& Pt
< (PULONGLONG
)PAGETABLE_MAP
+ 4*512*512))
1734 MiFlushTlb((PULONG
)Pt
, Address
);
1740 MmUnmapPageTable((PULONG
)Pt
);
1747 oldPdeOffset
= ADDR_TO_PDE_OFFSET(Addr
) + 1;
1748 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1750 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1752 DPRINT1("Setting physical address but not allowing access at address "
1753 "0x%.8X with attributes %x/%x.\n",
1754 Addr
, Attributes
, flProtect
);
1757 PdeOffset
= ADDR_TO_PDE_OFFSET(Addr
);
1758 if (oldPdeOffset
!= PdeOffset
)
1760 MmUnmapPageTable(Pt
);
1761 Pt
= MmGetPageTableForProcess(Process
, Addr
, TRUE
);
1771 oldPdeOffset
= PdeOffset
;
1774 MmMarkPageMapped(Pages
[i
]);
1775 if (PAGE_MASK((Pte
)) != 0 && !((Pte
) & PA_PRESENT
))
1779 if (PAGE_MASK((Pte
)) != 0)
1781 MmMarkPageUnmapped(PTE_TO_PFN((Pte
)));
1783 (void)InterlockedExchangeUL(Pt
, PFN_TO_PTE(Pages
[i
]) | Attributes
);
1784 if (Address
< MmSystemRangeStart
&&
1785 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1786 Attributes
& PA_PRESENT
)
1790 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1792 Ptrc
[ADDR_TO_PAGE_TABLE(Addr
)]++;
1796 if (Address
> MmSystemRangeStart
||
1797 (Pt
>= (PULONG
)PAGETABLE_MAP
&& Pt
< (PULONG
)PAGETABLE_MAP
+ 1024*1024))
1799 MiFlushTlb(Pt
, Address
);
1805 MmUnmapPageTable(Pt
);
1808 return(STATUS_SUCCESS
);
1813 MmCreateVirtualMapping(PEPROCESS Process
,
1821 for (i
= 0; i
< PageCount
; i
++)
1823 if (!MmIsPageInUse(Pages
[i
]))
1825 DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages
[i
]));
1830 return(MmCreateVirtualMappingUnsafe(Process
,
1839 MmGetPageProtect(PEPROCESS Process
, PVOID Address
)
1845 Entry
= MmGetPageEntryForProcessForPAE(Process
, Address
);
1849 Entry
= MmGetPageEntryForProcess(Process
, Address
);
1852 if (!(Entry
& PA_PRESENT
))
1854 Protect
= PAGE_NOACCESS
;
1858 if (Entry
& PA_READWRITE
)
1860 Protect
= PAGE_READWRITE
;
1864 Protect
= PAGE_EXECUTE_READ
;
1868 Protect
|= PAGE_NOCACHE
;
1872 Protect
|= PAGE_WRITETHROUGH
;
1874 if (!(Entry
& PA_USER
))
1876 Protect
|= PAGE_SYSTEM
;
1885 MmSetPageProtect(PEPROCESS Process
, PVOID Address
, ULONG flProtect
)
1887 ULONG Attributes
= 0;
1888 BOOLEAN NoExecute
= FALSE
;
1890 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
1891 Process
, Address
, flProtect
);
1893 Attributes
= ProtectToPTE(flProtect
);
1894 if (Attributes
& 0x80000000)
1898 Attributes
&= 0xfff;
1899 if (Address
>= MmSystemRangeStart
)
1901 Attributes
&= ~PA_USER
;
1902 if (Ke386GlobalPagesEnabled
)
1904 Attributes
|= PA_GLOBAL
;
1909 Attributes
|= PA_USER
;
1914 ULONGLONG tmpPte
, Pte
;
1916 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1919 DPRINT1("Address %x\n", Address
);
1925 tmpPte
= PAE_PAGE_MASK(Pte
) | Attributes
| (Pte
& (PA_ACCESSED
|PA_DIRTY
));
1928 tmpPte
|= 0x8000000000000000LL
;
1932 tmpPte
&= ~0x8000000000000000LL
;
1934 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1936 MiFlushTlb((PULONG
)Pt
, Address
);
1942 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1947 InterlockedExchange((PLONG
)Pt
, PAGE_MASK(*Pt
) | Attributes
| (*Pt
& (PA_ACCESSED
|PA_DIRTY
)));
1948 MiFlushTlb(Pt
, Address
);
1955 PHYSICAL_ADDRESS NTAPI
1956 MmGetPhysicalAddress(PVOID vaddr
)
1958 * FUNCTION: Returns the physical address corresponding to a virtual address
1963 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr
);
1967 Pte
= MmGetPageEntryForProcessForPAE(NULL
, vaddr
);
1968 if (Pte
!= 0 && Pte
& PA_PRESENT
)
1970 p
.QuadPart
= PAE_PAGE_MASK(Pte
);
1971 p
.u
.LowPart
|= (ULONG_PTR
)vaddr
& (PAGE_SIZE
- 1);
1981 Pte
= MmGetPageEntryForProcess(NULL
, vaddr
);
1982 if (Pte
!= 0 && Pte
& PA_PRESENT
)
1984 p
.QuadPart
= PAGE_MASK(Pte
);
1985 p
.u
.LowPart
|= (ULONG_PTR
)vaddr
& (PAGE_SIZE
- 1);
1997 MmCreateHyperspaceMapping(PFN_NUMBER Page
)
2005 ULONGLONG ZeroEntry
= 0LL;
2008 Entry
= PFN_TO_PTE(Page
) | PA_PRESENT
| PA_READWRITE
;
2009 Pte
= PAE_ADDR_TO_PTE(HYPERSPACE
) + Page
% 1024;
2013 for (i
= Page
%1024; i
< 1024; i
++, Pte
++)
2015 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2022 Pte
= PAE_ADDR_TO_PTE(HYPERSPACE
);
2023 for (i
= 0; i
< Page
% 1024; i
++, Pte
++)
2025 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2030 if (i
>= Page
% 1024)
2038 for (i
= Page
%1024; (LONG
)i
>= 0; i
--, Pte
--)
2040 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2047 Pte
= PAE_ADDR_TO_PTE(HYPERSPACE
) + 1023;
2048 for (i
= 1023; i
> Page
% 1024; i
--, Pte
--)
2050 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2055 if (i
<= Page
% 1024)
2066 Entry
= PFN_TO_PTE(Page
) | PA_PRESENT
| PA_READWRITE
;
2067 Pte
= ADDR_TO_PTE(HYPERSPACE
) + Page
% 1024;
2070 for (i
= Page
% 1024; i
< 1024; i
++, Pte
++)
2072 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2079 Pte
= ADDR_TO_PTE(HYPERSPACE
);
2080 for (i
= 0; i
< Page
% 1024; i
++, Pte
++)
2082 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2087 if (i
>= Page
% 1024)
2095 for (i
= Page
% 1024; (LONG
)i
>= 0; i
--, Pte
--)
2097 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2104 Pte
= ADDR_TO_PTE(HYPERSPACE
) + 1023;
2105 for (i
= 1023; i
> Page
% 1024; i
--, Pte
--)
2107 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2112 if (i
<= Page
% 1024)
2119 Address
= (PVOID
)((ULONG_PTR
)HYPERSPACE
+ i
* PAGE_SIZE
);
2126 MmChangeHyperspaceMapping(PVOID Address
, PFN_NUMBER NewPage
)
2129 ASSERT (IS_HYPERSPACE(Address
));
2132 ULONGLONG Entry
= PAE_PFN_TO_PTE(NewPage
) | PA_PRESENT
| PA_READWRITE
;
2133 Entry
= (ULONG
)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address
), &Entry
);
2134 Pfn
= PAE_PTE_TO_PFN(Entry
);
2139 Entry
= InterlockedExchange((PLONG
)ADDR_TO_PTE(Address
), PFN_TO_PTE(NewPage
) | PA_PRESENT
| PA_READWRITE
);
2140 Pfn
= PTE_TO_PFN(Entry
);
2148 MmDeleteHyperspaceMapping(PVOID Address
)
2151 ASSERT (IS_HYPERSPACE(Address
));
2154 ULONGLONG Entry
= 0LL;
2155 Entry
= (ULONG
)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address
), &Entry
);
2156 Pfn
= PAE_PTE_TO_PFN(Entry
);
2161 Entry
= InterlockedExchange((PLONG
)ADDR_TO_PTE(Address
), 0);
2162 Pfn
= PTE_TO_PFN(Entry
);
2170 MmUpdatePageDir(PEPROCESS Process
, PVOID Address
, ULONG Size
)
2172 ULONG StartOffset
, EndOffset
, Offset
;
2174 if (Address
< MmSystemRangeStart
)
2180 PULONGLONG PageDirTable
;
2182 ULONGLONG ZeroPde
= 0LL;
2185 for (i
= PAE_ADDR_TO_PDTE_OFFSET(Address
); i
<= PAE_ADDR_TO_PDTE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
)); i
++)
2187 if (i
== PAE_ADDR_TO_PDTE_OFFSET(Address
))
2189 StartOffset
= PAE_ADDR_TO_PDE_PAGE_OFFSET(Address
);
2195 if (i
== PAE_ADDR_TO_PDTE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
)))
2197 EndOffset
= PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
));
2204 if (Process
!= NULL
&& Process
!= PsGetCurrentProcess())
2206 PageDirTable
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
2207 Pde
= (PULONGLONG
)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable
[i
]));
2208 MmDeleteHyperspaceMapping(PageDirTable
);
2212 Pde
= (PULONGLONG
)PAE_PAGEDIRECTORY_MAP
+ i
*512;
2215 for (Offset
= StartOffset
; Offset
<= EndOffset
; Offset
++)
2217 if (i
* 512 + Offset
< PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) || i
* 512 + Offset
>= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
)+4)
2219 (void)ExfInterlockedCompareExchange64UL(&Pde
[Offset
], &MmGlobalKernelPageDirectoryForPAE
[i
*512 + Offset
], &ZeroPde
);
2222 MmUnmapPageTable((PULONG
)Pde
);
2228 StartOffset
= ADDR_TO_PDE_OFFSET(Address
);
2229 EndOffset
= ADDR_TO_PDE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
));
2231 if (Process
!= NULL
&& Process
!= PsGetCurrentProcess())
2233 Pde
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.u
.LowPart
));
2237 Pde
= (PULONG
)PAGEDIRECTORY_MAP
;
2239 for (Offset
= StartOffset
; Offset
<= EndOffset
; Offset
++)
2241 if (Offset
!= ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
))
2243 (void)InterlockedCompareExchangeUL(&Pde
[Offset
], MmGlobalKernelPageDirectory
[Offset
], 0);
2246 if (Pde
!= (PULONG
)PAGEDIRECTORY_MAP
)
2248 MmDeleteHyperspaceMapping(Pde
);
2256 MmInitGlobalKernelPageDirectory(VOID
)
2260 DPRINT("MmInitGlobalKernelPageDirectory()\n");
2264 PULONGLONG CurrentPageDirectory
= (PULONGLONG
)PAE_PAGEDIRECTORY_MAP
;
2265 for (i
= PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart
); i
< 4 * 512; i
++)
2267 if (!(i
>= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) && i
< PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) + 4) &&
2268 !(i
>= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE
) && i
< PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE
) + 2) &&
2269 0LL == MmGlobalKernelPageDirectoryForPAE
[i
] && 0LL != CurrentPageDirectory
[i
])
2271 (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE
[i
], &CurrentPageDirectory
[i
]);
2272 if (Ke386GlobalPagesEnabled
)
2274 MmGlobalKernelPageDirectoryForPAE
[i
] |= PA_GLOBAL
;
2275 CurrentPageDirectory
[i
] |= PA_GLOBAL
;
2282 PULONG CurrentPageDirectory
= (PULONG
)PAGEDIRECTORY_MAP
;
2283 for (i
= ADDR_TO_PDE_OFFSET(MmSystemRangeStart
); i
< 1024; i
++)
2285 if (i
!= ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) &&
2286 i
!= ADDR_TO_PDE_OFFSET(HYPERSPACE
) &&
2287 0 == MmGlobalKernelPageDirectory
[i
] && 0 != CurrentPageDirectory
[i
])
2289 MmGlobalKernelPageDirectory
[i
] = CurrentPageDirectory
[i
];
2290 if (Ke386GlobalPagesEnabled
)
2292 MmGlobalKernelPageDirectory
[i
] |= PA_GLOBAL
;
2293 CurrentPageDirectory
[i
] |= PA_GLOBAL
;
2302 MiGetUserPageDirectoryCount(VOID
)
2304 return Ke386Pae
? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart
) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart
);
2310 MiInitPageDirectoryMap(VOID
)
2312 MEMORY_AREA
* kernel_map_desc
= NULL
;
2313 MEMORY_AREA
* hyperspace_desc
= NULL
;
2314 PHYSICAL_ADDRESS BoundaryAddressMultiple
;
2318 DPRINT("MiInitPageDirectoryMap()\n");
2320 BoundaryAddressMultiple
.QuadPart
= 0;
2321 BaseAddress
= (PVOID
)PAGETABLE_MAP
;
2322 Status
= MmCreateMemoryArea(MmGetKernelAddressSpace(),
2325 Ke386Pae
? 0x800000 : 0x400000,
2330 BoundaryAddressMultiple
);
2331 if (!NT_SUCCESS(Status
))
2335 BaseAddress
= (PVOID
)HYPERSPACE
;
2336 Status
= MmCreateMemoryArea(MmGetKernelAddressSpace(),
2344 BoundaryAddressMultiple
);
2345 if (!NT_SUCCESS(Status
))