2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/i386/page.c
5 * PURPOSE: Low level memory managment manipulation
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
10 /* INCLUDES ***************************************************************/
16 #if defined (ALLOC_PRAGMA)
17 #pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
18 #pragma alloc_text(INIT, MiInitPageDirectoryMap)
22 /* GLOBALS *****************************************************************/
24 #define PA_BIT_PRESENT (0)
25 #define PA_BIT_READWRITE (1)
26 #define PA_BIT_USER (2)
29 #define PA_BIT_ACCESSED (5)
30 #define PA_BIT_DIRTY (6)
31 #define PA_BIT_GLOBAL (8)
33 #define PA_PRESENT (1 << PA_BIT_PRESENT)
34 #define PA_READWRITE (1 << PA_BIT_READWRITE)
35 #define PA_USER (1 << PA_BIT_USER)
36 #define PA_DIRTY (1 << PA_BIT_DIRTY)
37 #define PA_WT (1 << PA_BIT_WT)
38 #define PA_CD (1 << PA_BIT_CD)
39 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
40 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
42 #define PAGETABLE_MAP (0xc0000000)
43 #define PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (1024)))
45 #define PAE_PAGEDIRECTORY_MAP (0xc0000000 + (PAGETABLE_MAP / (512)))
47 #define HYPERSPACE (Ke386Pae ? 0xc0800000 : 0xc0400000)
48 #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
50 ULONG MmGlobalKernelPageDirectory
[1024];
51 ULONGLONG MmGlobalKernelPageDirectoryForPAE
[2048];
53 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
54 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
56 #define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
57 #define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
60 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
62 __inline LARGE_INTEGER
PTE_TO_PAGE(ULONG npage
)
65 dummy
.QuadPart
= (LONGLONG
)(PAGE_MASK(npage
));
70 extern BOOLEAN Ke386Pae
;
71 extern BOOLEAN Ke386NoExecute
;
73 /* FUNCTIONS ***************************************************************/
75 BOOLEAN
MmUnmapPageTable(PULONG Pt
);
79 MiFlushTlbIpiRoutine(ULONG_PTR Address
)
81 if (Address
== (ULONG_PTR
)-1)
85 else if (Address
== (ULONG_PTR
)-2)
91 __invlpg((PVOID
)Address
);
97 MiFlushTlb(PULONG Pt
, PVOID Address
)
102 MmUnmapPageTable(Pt
);
104 if (KeNumberProcessors
> 1)
106 KeIpiGenericCall(MiFlushTlbIpiRoutine
, (ULONG_PTR
)Address
);
110 MiFlushTlbIpiRoutine((ULONG_PTR
)Address
);
113 if ((Pt
&& MmUnmapPageTable(Pt
)) || Address
>= MmSystemRangeStart
)
123 MmGetPageDirectory(VOID
)
125 return (PULONG
)__readcr3();
129 ProtectToPTE(ULONG flProtect
)
131 ULONG Attributes
= 0;
133 if (flProtect
& (PAGE_NOACCESS
|PAGE_GUARD
))
137 else if (flProtect
& PAGE_IS_WRITABLE
)
139 Attributes
= PA_PRESENT
| PA_READWRITE
;
141 else if (flProtect
& (PAGE_IS_READABLE
| PAGE_IS_EXECUTABLE
))
143 Attributes
= PA_PRESENT
;
147 DPRINT1("Unknown main protection type.\n");
150 if (Ke386NoExecute
&&
151 !(flProtect
& PAGE_IS_EXECUTABLE
))
153 Attributes
= Attributes
| 0x80000000;
156 if (flProtect
& PAGE_SYSTEM
)
161 Attributes
= Attributes
| PA_USER
;
163 if (flProtect
& PAGE_NOCACHE
)
165 Attributes
= Attributes
| PA_CD
;
167 if (flProtect
& PAGE_WRITETHROUGH
)
169 Attributes
= Attributes
| PA_WT
;
174 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
176 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
177 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
178 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
180 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
182 #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
185 #define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE))
187 #define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
188 ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
189 #define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
192 #define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
194 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
196 #define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
198 #define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
203 Mmi386ReleaseMmInfo(PEPROCESS Process
)
205 PUSHORT LdtDescriptor
;
209 DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process
);
211 LdtDescriptor
= (PUSHORT
) &Process
->Pcb
.LdtDescriptor
;
212 LdtBase
= LdtDescriptor
[1] |
213 ((LdtDescriptor
[2] & 0xff) << 16) |
214 ((LdtDescriptor
[3] & ~0xff) << 16);
216 DPRINT("LdtBase: %x\n", LdtBase
);
220 ExFreePool((PVOID
) LdtBase
);
225 PULONGLONG PageDirTable
;
230 PageDirTable
= (PULONGLONG
)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
231 for (i
= 0; i
< 4; i
++)
233 PageDir
= (PULONGLONG
)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable
[i
]));
234 if (i
< PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart
))
236 for (j
= 0; j
< 512; j
++)
238 if (PageDir
[j
] != 0LL)
240 DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
241 Process
->UniqueProcessId
,
242 (i
* 512 + j
) * 512 * PAGE_SIZE
, (i
* 512 + j
+ 1) * 512 * PAGE_SIZE
- 1,
243 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
[i
*512 + j
]);
244 Pde
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir
[j
]));
245 for (k
= 0; k
< 512; k
++)
249 if (Pde
[k
] & PA_PRESENT
)
251 DPRINT1("Page at %08x is not freed\n",
252 (i
* 512 + j
) * 512 * PAGE_SIZE
+ k
* PAGE_SIZE
);
256 DPRINT1("Swapentry %x at %x is not freed\n",
257 (i
* 512 + j
) * 512 * PAGE_SIZE
+ k
* PAGE_SIZE
);
261 MmDeleteHyperspaceMapping(Pde
);
262 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDir
[j
]));
266 if (i
== PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE
))
268 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)]));
269 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)+1]));
271 MmDeleteHyperspaceMapping(PageDir
);
272 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(PageDirTable
[i
]));
274 MmDeleteHyperspaceMapping((PVOID
)PageDirTable
);
275 MmReleasePageMemoryConsumer(MC_NPPOOL
, PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
281 PageDir
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.u
.LowPart
));
282 for (i
= 0; i
< ADDR_TO_PDE_OFFSET(MmSystemRangeStart
); i
++)
286 DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
287 i
* 4 * 1024 * 1024, (i
+ 1) * 4 * 1024 * 1024 - 1,
288 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
[i
]);
289 Pde
= MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir
[i
]));
290 for (j
= 0; j
< 1024; j
++)
294 if (Pde
[j
] & PA_PRESENT
)
296 DPRINT1("Page at %08x is not freed\n",
297 i
* 4 * 1024 * 1024 + j
* PAGE_SIZE
);
301 DPRINT1("Swapentry %x at %x is not freed\n",
302 Pde
[j
], i
* 4 * 1024 * 1024 + j
* PAGE_SIZE
);
306 MmDeleteHyperspaceMapping(Pde
);
307 MmReleasePageMemoryConsumer(MC_NPPOOL
, PTE_TO_PFN(PageDir
[i
]));
310 MmReleasePageMemoryConsumer(MC_NPPOOL
, PTE_TO_PFN(PageDir
[ADDR_TO_PDE_OFFSET(HYPERSPACE
)]));
311 MmDeleteHyperspaceMapping(PageDir
);
312 MmReleasePageMemoryConsumer(MC_NPPOOL
, PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.u
.LowPart
));
315 #if defined(__GNUC__)
317 Process
->Pcb
.DirectoryTableBase
.QuadPart
= 0LL;
320 Process
->Pcb
.DirectoryTableBase
.QuadPart
= 0;
323 DPRINT("Finished Mmi386ReleaseMmInfo()\n");
324 return(STATUS_SUCCESS
);
329 MmInitializeHandBuiltProcess(IN PEPROCESS Process
,
330 IN PLARGE_INTEGER DirectoryTableBase
)
332 /* Share the directory base with the idle process */
333 *DirectoryTableBase
= PsGetCurrentProcess()->Pcb
.DirectoryTableBase
;
335 /* Initialize the Addresss Space */
336 MmInitializeAddressSpace(Process
, (PMADDRESS_SPACE
)&Process
->VadRoot
);
338 /* The process now has an address space */
339 Process
->HasAddressSpace
= TRUE
;
340 return STATUS_SUCCESS
;
345 MmCreateProcessAddressSpace(IN ULONG MinWs
,
346 IN PEPROCESS Process
,
347 IN PLARGE_INTEGER DirectoryTableBase
)
354 DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", MinWs
, Process
);
356 Count
= Ke386Pae
? 7 : 2;
358 for (i
= 0; i
< Count
; i
++)
360 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
[i
]);
361 if (!NT_SUCCESS(Status
))
363 for (j
= 0; j
< i
; j
++)
365 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
[j
]);
374 PULONGLONG PageDirTable
;
377 PageDirTable
= MmCreateHyperspaceMapping(Pfn
[0]);
378 for (i
= 0; i
< 4; i
++)
380 PageDirTable
[i
] = PAE_PFN_TO_PTE(Pfn
[1+i
]) | PA_PRESENT
;
382 MmDeleteHyperspaceMapping(PageDirTable
);
383 for (i
= PAE_ADDR_TO_PDTE_OFFSET(MmSystemRangeStart
); i
< 4; i
++)
385 PageDir
= (PULONGLONG
)MmCreateHyperspaceMapping(Pfn
[i
+1]);
386 memcpy(PageDir
, &MmGlobalKernelPageDirectoryForPAE
[i
* 512], 512 * sizeof(ULONGLONG
));
387 if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP
) == i
)
389 for (j
= 0; j
< 4; j
++)
391 PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP
) + j
] = PAE_PFN_TO_PTE(Pfn
[1+j
]) | PA_PRESENT
| PA_READWRITE
;
394 if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE
) == i
)
396 PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)] = PAE_PFN_TO_PTE(Pfn
[5]) | PA_PRESENT
| PA_READWRITE
;
397 PageDir
[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE
)+1] = PAE_PFN_TO_PTE(Pfn
[6]) | PA_PRESENT
| PA_READWRITE
;
399 MmDeleteHyperspaceMapping(PageDir
);
404 PULONG PageDirectory
;
405 PageDirectory
= MmCreateHyperspaceMapping(Pfn
[0]);
407 memcpy(PageDirectory
+ ADDR_TO_PDE_OFFSET(MmSystemRangeStart
),
408 MmGlobalKernelPageDirectory
+ ADDR_TO_PDE_OFFSET(MmSystemRangeStart
),
409 (1024 - ADDR_TO_PDE_OFFSET(MmSystemRangeStart
)) * sizeof(ULONG
));
411 DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
));
412 PageDirectory
[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
)] = PFN_TO_PTE(Pfn
[0]) | PA_PRESENT
| PA_READWRITE
;
413 PageDirectory
[ADDR_TO_PDE_OFFSET(HYPERSPACE
)] = PFN_TO_PTE(Pfn
[1]) | PA_PRESENT
| PA_READWRITE
;
415 MmDeleteHyperspaceMapping(PageDirectory
);
418 DirectoryTableBase
->QuadPart
= PFN_TO_PTE(Pfn
[0]);
419 DPRINT("Finished MmCopyMmInfo(): %I64x\n", DirectoryTableBase
->QuadPart
);
425 MmDeletePageTable(PEPROCESS Process
, PVOID Address
)
427 PEPROCESS CurrentProcess
= PsGetCurrentProcess();
429 if (Process
!= NULL
&& Process
!= CurrentProcess
)
431 KeAttachProcess(&Process
->Pcb
);
436 ULONGLONG ZeroPde
= 0LL;
437 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address
), &ZeroPde
);
438 MiFlushTlb((PULONG
)PAE_ADDR_TO_PDE(Address
), PAE_ADDR_TO_PTE(Address
));
442 *(ADDR_TO_PDE(Address
)) = 0;
443 MiFlushTlb(ADDR_TO_PDE(Address
), ADDR_TO_PTE(Address
));
445 if (Address
>= MmSystemRangeStart
)
448 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
450 if (Process
!= NULL
&& Process
!= CurrentProcess
)
458 MmFreePageTable(PEPROCESS Process
, PVOID Address
)
460 PEPROCESS CurrentProcess
= PsGetCurrentProcess();
464 DPRINT("ProcessId %d, Address %x\n", Process
->UniqueProcessId
, Address
);
465 if (Process
!= NULL
&& Process
!= CurrentProcess
)
467 KeAttachProcess(&Process
->Pcb
);
471 PULONGLONG PageTable
;
472 ULONGLONG ZeroPte
= 0LL;
473 PageTable
= (PULONGLONG
)PAGE_ROUND_DOWN((PVOID
)PAE_ADDR_TO_PTE(Address
));
474 for (i
= 0; i
< 512; i
++)
476 if (PageTable
[i
] != 0LL)
478 DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
479 ((ULONG
)Address
/ (4*1024*1024)), i
, PageTable
[i
]);
483 Pfn
= PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address
)));
484 (void)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address
), &ZeroPte
);
485 MiFlushTlb((PULONG
)PAE_ADDR_TO_PDE(Address
), PAE_ADDR_TO_PTE(Address
));
490 PageTable
= (PULONG
)PAGE_ROUND_DOWN((PVOID
)ADDR_TO_PTE(Address
));
491 for (i
= 0; i
< 1024; i
++)
493 if (PageTable
[i
] != 0)
495 DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
496 ((ULONG
)Address
/ (4*1024*1024)), i
, PageTable
[i
]);
500 Pfn
= PTE_TO_PFN(*(ADDR_TO_PDE(Address
)));
501 *(ADDR_TO_PDE(Address
)) = 0;
502 MiFlushTlb(ADDR_TO_PDE(Address
), ADDR_TO_PTE(Address
));
505 if (Address
>= MmSystemRangeStart
)
507 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
512 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
514 if (Process
!= NULL
&& Process
!= CurrentProcess
)
521 MmGetPageTableForProcessForPAE(PEPROCESS Process
, PVOID Address
, BOOLEAN Create
)
526 ULONGLONG ZeroEntry
= 0LL;
529 PULONGLONG PageDirTable
;
531 DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
532 Process
, Address
, Create
);
533 if (Address
>= (PVOID
)PAGETABLE_MAP
&& Address
< (PVOID
)((ULONG_PTR
)PAGETABLE_MAP
+ 0x800000))
537 if (Address
< MmSystemRangeStart
&& Process
&& Process
!= PsGetCurrentProcess())
539 PageDirTable
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
540 if (PageDirTable
== NULL
)
544 PageDir
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable
[PAE_ADDR_TO_PDTE_OFFSET(Address
)]));
545 MmDeleteHyperspaceMapping(PageDirTable
);
550 PageDir
+= PAE_ADDR_TO_PDE_PAGE_OFFSET(Address
);
551 Entry
= ExfInterlockedCompareExchange64UL(PageDir
, &ZeroEntry
, &ZeroEntry
);
556 MmDeleteHyperspaceMapping(PageDir
);
559 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
560 if (!NT_SUCCESS(Status
))
564 Entry
= PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
;
565 Entry
= ExfInterlockedCompareExchange64UL(PageDir
, &Entry
, &ZeroEntry
);
568 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
569 Pfn
= PAE_PTE_TO_PFN(Entry
);
574 Pfn
= PAE_PTE_TO_PFN(Entry
);
576 MmDeleteHyperspaceMapping(PageDir
);
577 Pt
= MmCreateHyperspaceMapping(Pfn
);
582 return Pt
+ PAE_ADDR_TO_PTE_OFFSET(Address
);
584 PageDir
= PAE_ADDR_TO_PDE(Address
);
585 if (0LL == ExfInterlockedCompareExchange64UL(PageDir
, &ZeroEntry
, &ZeroEntry
))
587 if (Address
>= MmSystemRangeStart
)
589 if (MmGlobalKernelPageDirectoryForPAE
[PAE_ADDR_TO_PDE_OFFSET(Address
)] == 0LL)
595 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
596 if (!NT_SUCCESS(Status
))
600 Entry
= PAE_PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
;
601 if (Ke386GlobalPagesEnabled
)
605 if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE
[PAE_ADDR_TO_PDE_OFFSET(Address
)], &Entry
, &ZeroEntry
))
607 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
610 (void)ExfInterlockedCompareExchange64UL(PageDir
, &MmGlobalKernelPageDirectoryForPAE
[PAE_ADDR_TO_PDE_OFFSET(Address
)], &ZeroEntry
);
618 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
619 if (!NT_SUCCESS(Status
))
623 Entry
= PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
;
624 Entry
= ExfInterlockedCompareExchange64UL(PageDir
, &Entry
, &ZeroEntry
);
627 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
631 return (PULONGLONG
)PAE_ADDR_TO_PTE(Address
);
635 MmGetPageTableForProcess(PEPROCESS Process
, PVOID Address
, BOOLEAN Create
)
637 ULONG PdeOffset
= ADDR_TO_PDE_OFFSET(Address
);
643 if (Address
< MmSystemRangeStart
&& Process
&& Process
!= PsGetCurrentProcess())
645 PageDir
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.LowPart
));
650 if (0 == InterlockedCompareExchangeUL(&PageDir
[PdeOffset
], 0, 0))
654 MmDeleteHyperspaceMapping(PageDir
);
657 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
658 if (!NT_SUCCESS(Status
) || Pfn
== 0)
662 Entry
= InterlockedCompareExchangeUL(&PageDir
[PdeOffset
], PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
, 0);
665 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
666 Pfn
= PTE_TO_PFN(Entry
);
671 Pfn
= PTE_TO_PFN(PageDir
[PdeOffset
]);
673 MmDeleteHyperspaceMapping(PageDir
);
674 Pt
= MmCreateHyperspaceMapping(Pfn
);
679 return Pt
+ ADDR_TO_PTE_OFFSET(Address
);
681 PageDir
= ADDR_TO_PDE(Address
);
682 if (0 == InterlockedCompareExchangeUL(PageDir
, 0, 0))
684 if (Address
>= MmSystemRangeStart
)
686 if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory
[PdeOffset
], 0, 0))
692 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
693 if (!NT_SUCCESS(Status
) || Pfn
== 0)
697 Entry
= PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
;
698 if (Ke386GlobalPagesEnabled
)
702 if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory
[PdeOffset
], Entry
, 0))
704 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
707 (void)InterlockedExchangeUL(PageDir
, MmGlobalKernelPageDirectory
[PdeOffset
]);
715 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
716 if (!NT_SUCCESS(Status
) || Pfn
== 0)
720 Entry
= InterlockedCompareExchangeUL(PageDir
, PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
, 0);
723 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
727 return (PULONG
)ADDR_TO_PTE(Address
);
730 BOOLEAN
MmUnmapPageTable(PULONG Pt
)
734 if ((PULONGLONG
)Pt
>= (PULONGLONG
)PAGETABLE_MAP
&& (PULONGLONG
)Pt
< (PULONGLONG
)PAGETABLE_MAP
+ 4*512*512)
741 if (Pt
>= (PULONG
)PAGETABLE_MAP
&& Pt
< (PULONG
)PAGETABLE_MAP
+ 1024*1024)
748 MmDeleteHyperspaceMapping((PVOID
)PAGE_ROUND_DOWN(Pt
));
753 static ULONGLONG
MmGetPageEntryForProcessForPAE(PEPROCESS Process
, PVOID Address
)
758 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
762 MmUnmapPageTable((PULONG
)Pt
);
768 static ULONG
MmGetPageEntryForProcess(PEPROCESS Process
, PVOID Address
)
773 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
777 MmUnmapPageTable(Pt
);
785 MmGetPfnForProcess(PEPROCESS Process
,
792 Entry
= MmGetPageEntryForProcessForPAE(Process
, Address
);
793 if (!(Entry
& PA_PRESENT
))
797 return(PAE_PTE_TO_PFN(Entry
));
802 Entry
= MmGetPageEntryForProcess(Process
, Address
);
803 if (!(Entry
& PA_PRESENT
))
807 return(PTE_TO_PFN(Entry
));
813 MmDisableVirtualMapping(PEPROCESS Process
, PVOID Address
, BOOLEAN
* WasDirty
, PPFN_TYPE Page
)
815 * FUNCTION: Delete a virtual mapping
825 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
831 * Atomically disable the present bit and get the old value.
836 tmpPte
= Pte
& ~PA_PRESENT
;
837 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
839 MiFlushTlb((PULONG
)Pt
, Address
);
840 WasValid
= PAE_PAGE_MASK(Pte
) != 0LL ? TRUE
: FALSE
;
847 * Return some information to the caller
849 if (WasDirty
!= NULL
)
851 *WasDirty
= Pte
& PA_DIRTY
? TRUE
: FALSE
;
855 *Page
= PAE_PTE_TO_PFN(Pte
);
863 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
869 * Atomically disable the present bit and get the old value.
874 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
& ~PA_PRESENT
, Pte
));
876 MiFlushTlb(Pt
, Address
);
877 WasValid
= (PAGE_MASK(Pte
) != 0);
884 * Return some information to the caller
886 if (WasDirty
!= NULL
)
888 *WasDirty
= Pte
& PA_DIRTY
;
892 *Page
= PTE_TO_PFN(Pte
);
899 MmRawDeleteVirtualMapping(PVOID Address
)
904 ULONGLONG ZeroPte
= 0LL;
905 Pt
= MmGetPageTableForProcessForPAE(NULL
, Address
, FALSE
);
909 * Set the entry to zero
911 (void)ExfpInterlockedExchange64UL(Pt
, &ZeroPte
);
912 MiFlushTlb((PULONG
)Pt
, Address
);
919 Pt
= MmGetPageTableForProcess(NULL
, Address
, FALSE
);
923 * Set the entry to zero
925 (void)InterlockedExchangeUL(Pt
, 0);
926 MiFlushTlb(Pt
, Address
);
933 MmDeleteVirtualMapping(PEPROCESS Process
, PVOID Address
, BOOLEAN FreePage
,
934 BOOLEAN
* WasDirty
, PPFN_TYPE Page
)
936 * FUNCTION: Delete a virtual mapping
939 BOOLEAN WasValid
= FALSE
;
942 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
943 Process
, Address
, FreePage
, WasDirty
, Page
);
949 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
952 if (WasDirty
!= NULL
)
964 * Atomically set the entry to zero and get the old value.
967 Pte
= ExfpInterlockedExchange64UL(Pt
, &Pte
);
969 MiFlushTlb((PULONG
)Pt
, Address
);
971 WasValid
= PAE_PAGE_MASK(Pte
) != 0 ? TRUE
: FALSE
;
974 Pfn
= PAE_PTE_TO_PFN(Pte
);
975 MmMarkPageUnmapped(Pfn
);
982 if (FreePage
&& WasValid
)
984 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
988 * Return some information to the caller
990 if (WasDirty
!= NULL
)
992 *WasDirty
= Pte
& PA_DIRTY
? TRUE
: FALSE
;
1004 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1008 if (WasDirty
!= NULL
)
1020 * Atomically set the entry to zero and get the old value.
1022 Pte
= InterlockedExchangeUL(Pt
, 0);
1024 MiFlushTlb(Pt
, Address
);
1026 WasValid
= (PAGE_MASK(Pte
) != 0);
1029 Pfn
= PTE_TO_PFN(Pte
);
1030 MmMarkPageUnmapped(Pfn
);
1037 if (FreePage
&& WasValid
)
1039 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
1043 * Return some information to the caller
1045 if (WasDirty
!= NULL
)
1047 *WasDirty
= Pte
& PA_DIRTY
? TRUE
: FALSE
;
1055 * Decrement the reference count for this page table.
1057 if (Process
!= NULL
&& WasValid
&&
1058 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1059 Address
< MmSystemRangeStart
)
1064 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1065 Idx
= Ke386Pae
? PAE_ADDR_TO_PAGE_TABLE(Address
) : ADDR_TO_PAGE_TABLE(Address
);
1070 MmFreePageTable(Process
, Address
);
1077 MmDeletePageFileMapping(PEPROCESS Process
, PVOID Address
,
1078 SWAPENTRY
* SwapEntry
)
1080 * FUNCTION: Delete a virtual mapping
1088 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1096 * Atomically set the entry to zero and get the old value.
1099 Pte
= ExfpInterlockedExchange64UL(Pt
, &Pte
);
1101 MiFlushTlb((PULONG
)Pt
, Address
);
1104 * Decrement the reference count for this page table.
1106 if (Process
!= NULL
&& Pte
&&
1107 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1108 Address
< MmSystemRangeStart
)
1112 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1114 Ptrc
[PAE_ADDR_TO_PAGE_TABLE(Address
)]--;
1115 if (Ptrc
[PAE_ADDR_TO_PAGE_TABLE(Address
)] == 0)
1117 MmFreePageTable(Process
, Address
);
1123 * Return some information to the caller
1125 *SwapEntry
= Pte
>> 1;
1132 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1141 * Atomically set the entry to zero and get the old value.
1143 Pte
= InterlockedExchangeUL(Pt
, 0);
1145 MiFlushTlb(Pt
, Address
);
1148 * Decrement the reference count for this page table.
1150 if (Process
!= NULL
&& Pte
&&
1151 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1152 Address
< MmSystemRangeStart
)
1156 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1158 Ptrc
[ADDR_TO_PAGE_TABLE(Address
)]--;
1159 if (Ptrc
[ADDR_TO_PAGE_TABLE(Address
)] == 0)
1161 MmFreePageTable(Process
, Address
);
1167 * Return some information to the caller
1169 *SwapEntry
= Pte
>> 1;
1174 Mmi386MakeKernelPageTableGlobal(PVOID PAddress
)
1180 Pde
= PAE_ADDR_TO_PDE(PAddress
);
1183 Pt
= MmGetPageTableForProcessForPAE(NULL
, PAddress
, FALSE
);
1185 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1186 FLASH_TLB_ONE(PAddress
);
1197 Pde
= ADDR_TO_PDE(PAddress
);
1200 Pt
= MmGetPageTableForProcess(NULL
, PAddress
, FALSE
);
1202 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1203 FLASH_TLB_ONE(PAddress
);
1216 MmIsDirtyPage(PEPROCESS Process
, PVOID Address
)
1220 return MmGetPageEntryForProcessForPAE(Process
, Address
) & PA_DIRTY
? TRUE
: FALSE
;
1224 return MmGetPageEntryForProcess(Process
, Address
) & PA_DIRTY
? TRUE
: FALSE
;
1230 MmIsAccessedAndResetAccessPage(PEPROCESS Process
, PVOID Address
)
1232 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
1234 DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1243 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1252 tmpPte
= Pte
& ~PA_ACCESSED
;
1253 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1255 if (Pte
& PA_ACCESSED
)
1257 MiFlushTlb((PULONG
)Pt
, Address
);
1262 MmUnmapPageTable((PULONG
)Pt
);
1271 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1280 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
& ~PA_ACCESSED
, Pte
));
1282 if (Pte
& PA_ACCESSED
)
1284 MiFlushTlb(Pt
, Address
);
1289 MmUnmapPageTable(Pt
);
1297 MmSetCleanPage(PEPROCESS Process
, PVOID Address
)
1299 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
1301 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1310 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1320 tmpPte
= Pte
& ~PA_DIRTY
;
1321 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1325 MiFlushTlb((PULONG
)Pt
, Address
);
1329 MmUnmapPageTable((PULONG
)Pt
);
1337 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1347 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
& ~PA_DIRTY
, Pte
));
1351 MiFlushTlb(Pt
, Address
);
1355 MmUnmapPageTable(Pt
);
1362 MmSetDirtyPage(PEPROCESS Process
, PVOID Address
)
1364 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
1366 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1375 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1384 tmpPte
= Pte
| PA_DIRTY
;
1385 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1386 if (!(Pte
& PA_DIRTY
))
1388 MiFlushTlb((PULONG
)Pt
, Address
);
1392 MmUnmapPageTable((PULONG
)Pt
);
1400 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1409 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
| PA_DIRTY
, Pte
));
1410 if (!(Pte
& PA_DIRTY
))
1412 MiFlushTlb(Pt
, Address
);
1416 MmUnmapPageTable(Pt
);
1423 MmEnableVirtualMapping(PEPROCESS Process
, PVOID Address
)
1431 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
1440 tmpPte
= Pte
| PA_PRESENT
;
1441 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
1442 if (!(Pte
& PA_PRESENT
))
1444 MiFlushTlb((PULONG
)Pt
, Address
);
1448 MmUnmapPageTable((PULONG
)Pt
);
1456 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
1465 } while (Pte
!= InterlockedCompareExchangeUL(Pt
, Pte
| PA_PRESENT
, Pte
));
1466 if (!(Pte
& PA_PRESENT
))
1468 MiFlushTlb(Pt
, Address
);
1472 MmUnmapPageTable(Pt
);
1479 MmIsPagePresent(PEPROCESS Process
, PVOID Address
)
1483 return MmGetPageEntryForProcessForPAE(Process
, Address
) & PA_PRESENT
? TRUE
: FALSE
;
1487 return MmGetPageEntryForProcess(Process
, Address
) & PA_PRESENT
? TRUE
: FALSE
;
1493 MmIsPageSwapEntry(PEPROCESS Process
, PVOID Address
)
1498 Entry
= MmGetPageEntryForProcessForPAE(Process
, Address
);
1499 return !(Entry
& PA_PRESENT
) && Entry
!= 0 ? TRUE
: FALSE
;
1504 Entry
= MmGetPageEntryForProcess(Process
, Address
);
1505 return !(Entry
& PA_PRESENT
) && Entry
!= 0 ? TRUE
: FALSE
;
1511 MmCreateVirtualMappingForKernel(PVOID Address
,
1519 ULONG PdeOffset
, oldPdeOffset
;
1520 BOOLEAN NoExecute
= FALSE
;
1522 DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1523 Address
, flProtect
, Pages
, PageCount
);
1525 if (Address
< MmSystemRangeStart
)
1527 DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1531 Attributes
= ProtectToPTE(flProtect
);
1532 if (Attributes
& 0x80000000)
1536 Attributes
&= 0xfff;
1537 if (Ke386GlobalPagesEnabled
)
1539 Attributes
|= PA_GLOBAL
;
1546 PULONGLONG Pt
= NULL
;
1549 oldPdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
) + 1;
1550 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1552 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1554 DPRINT1("Setting physical address but not allowing access at address "
1555 "0x%.8X with attributes %x/%x.\n",
1556 Addr
, Attributes
, flProtect
);
1560 PdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
);
1561 if (oldPdeOffset
!= PdeOffset
)
1563 Pt
= MmGetPageTableForProcessForPAE(NULL
, Addr
, TRUE
);
1573 oldPdeOffset
= PdeOffset
;
1575 Pte
= PFN_TO_PTE(Pages
[i
]) | Attributes
;
1578 Pte
|= 0x8000000000000000LL
;
1580 Pte
= ExfpInterlockedExchange64UL(Pt
, &Pte
);
1592 oldPdeOffset
= ADDR_TO_PDE_OFFSET(Addr
);
1593 Pt
= MmGetPageTableForProcess(NULL
, Addr
, TRUE
);
1600 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1602 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1604 DPRINT1("Setting physical address but not allowing access at address "
1605 "0x%.8X with attributes %x/%x.\n",
1606 Addr
, Attributes
, flProtect
);
1610 PdeOffset
= ADDR_TO_PDE_OFFSET(Addr
);
1611 if (oldPdeOffset
!= PdeOffset
)
1613 Pt
= MmGetPageTableForProcess(NULL
, Addr
, TRUE
);
1623 oldPdeOffset
= PdeOffset
;
1630 (void)InterlockedExchangeUL(Pt
, PFN_TO_PTE(Pages
[i
]) | Attributes
);
1634 return(STATUS_SUCCESS
);
1639 MmCreatePageFileMapping(PEPROCESS Process
,
1641 SWAPENTRY SwapEntry
)
1643 if (Process
== NULL
&& Address
< MmSystemRangeStart
)
1645 DPRINT1("No process\n");
1648 if (Process
!= NULL
&& Address
>= MmSystemRangeStart
)
1650 DPRINT1("Setting kernel address with process context\n");
1653 if (SwapEntry
& (1 << 31))
1664 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, TRUE
);
1669 tmpPte
= SwapEntry
<< 1;
1670 Pte
= ExfpInterlockedExchange64UL(Pt
, &tmpPte
);
1671 if (PAE_PAGE_MASK((Pte
)) != 0)
1673 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte
)));
1678 MiFlushTlb((PULONG
)Pt
, Address
);
1682 MmUnmapPageTable((PULONG
)Pt
);
1690 Pt
= MmGetPageTableForProcess(Process
, Address
, TRUE
);
1696 if (PAGE_MASK((Pte
)) != 0)
1698 MmMarkPageUnmapped(PTE_TO_PFN((Pte
)));
1700 (void)InterlockedExchangeUL(Pt
, SwapEntry
<< 1);
1703 MiFlushTlb(Pt
, Address
);
1707 MmUnmapPageTable(Pt
);
1710 if (Process
!= NULL
&&
1711 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1712 Address
< MmSystemRangeStart
)
1717 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1718 Idx
= Ke386Pae
? PAE_ADDR_TO_PAGE_TABLE(Address
) : ADDR_TO_PAGE_TABLE(Address
);
1721 return(STATUS_SUCCESS
);
1727 MmCreateVirtualMappingUnsafe(PEPROCESS Process
,
1736 ULONG oldPdeOffset
, PdeOffset
;
1737 BOOLEAN NoExecute
= FALSE
;
1739 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1740 Process
, Address
, flProtect
, Pages
, *Pages
, PageCount
);
1742 if (Process
== NULL
)
1744 if (Address
< MmSystemRangeStart
)
1746 DPRINT1("No process\n");
1749 if (PageCount
> 0x10000 ||
1750 (ULONG_PTR
) Address
/ PAGE_SIZE
+ PageCount
> 0x100000)
1752 DPRINT1("Page count to large\n");
1758 if (Address
>= MmSystemRangeStart
)
1760 DPRINT1("Setting kernel address with process context\n");
1763 if (PageCount
> (ULONG_PTR
)MmSystemRangeStart
/ PAGE_SIZE
||
1764 (ULONG_PTR
) Address
/ PAGE_SIZE
+ PageCount
>
1765 (ULONG_PTR
)MmSystemRangeStart
/ PAGE_SIZE
)
1767 DPRINT1("Page Count to large\n");
1772 Attributes
= ProtectToPTE(flProtect
);
1773 if (Attributes
& 0x80000000)
1777 Attributes
&= 0xfff;
1778 if (Address
>= MmSystemRangeStart
)
1780 Attributes
&= ~PA_USER
;
1781 if (Ke386GlobalPagesEnabled
)
1783 Attributes
|= PA_GLOBAL
;
1788 Attributes
|= PA_USER
;
1795 ULONGLONG Pte
, tmpPte
;
1796 PULONGLONG Pt
= NULL
;
1798 oldPdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
) + 1;
1799 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1801 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1803 DPRINT1("Setting physical address but not allowing access at address "
1804 "0x%.8X with attributes %x/%x.\n",
1805 Addr
, Attributes
, flProtect
);
1808 PdeOffset
= PAE_ADDR_TO_PDE_OFFSET(Addr
);
1809 if (oldPdeOffset
!= PdeOffset
)
1811 MmUnmapPageTable((PULONG
)Pt
);
1812 Pt
= MmGetPageTableForProcessForPAE(Process
, Addr
, TRUE
);
1822 oldPdeOffset
= PdeOffset
;
1824 MmMarkPageMapped(Pages
[i
]);
1825 tmpPte
= PAE_PFN_TO_PTE(Pages
[i
]) | Attributes
;
1828 tmpPte
|= 0x8000000000000000LL
;
1830 Pte
= ExfpInterlockedExchange64UL(Pt
, &tmpPte
);
1831 if (PAE_PAGE_MASK((Pte
)) != 0LL && !((Pte
) & PA_PRESENT
))
1835 if (PAE_PAGE_MASK((Pte
)) != 0LL)
1837 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte
)));
1839 if (Address
< MmSystemRangeStart
&&
1840 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1841 Attributes
& PA_PRESENT
)
1845 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1847 Ptrc
[PAE_ADDR_TO_PAGE_TABLE(Addr
)]++;
1851 if (Address
> MmSystemRangeStart
||
1852 (Pt
>= (PULONGLONG
)PAGETABLE_MAP
&& Pt
< (PULONGLONG
)PAGETABLE_MAP
+ 4*512*512))
1854 MiFlushTlb((PULONG
)Pt
, Address
);
1860 MmUnmapPageTable((PULONG
)Pt
);
1867 oldPdeOffset
= ADDR_TO_PDE_OFFSET(Addr
) + 1;
1868 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
1870 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
1872 DPRINT1("Setting physical address but not allowing access at address "
1873 "0x%.8X with attributes %x/%x.\n",
1874 Addr
, Attributes
, flProtect
);
1877 PdeOffset
= ADDR_TO_PDE_OFFSET(Addr
);
1878 if (oldPdeOffset
!= PdeOffset
)
1880 MmUnmapPageTable(Pt
);
1881 Pt
= MmGetPageTableForProcess(Process
, Addr
, TRUE
);
1891 oldPdeOffset
= PdeOffset
;
1894 MmMarkPageMapped(Pages
[i
]);
1895 if (PAGE_MASK((Pte
)) != 0 && !((Pte
) & PA_PRESENT
))
1899 if (PAGE_MASK((Pte
)) != 0)
1901 MmMarkPageUnmapped(PTE_TO_PFN((Pte
)));
1903 (void)InterlockedExchangeUL(Pt
, PFN_TO_PTE(Pages
[i
]) | Attributes
);
1904 if (Address
< MmSystemRangeStart
&&
1905 ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
!= NULL
&&
1906 Attributes
& PA_PRESENT
)
1910 Ptrc
= ((PMADDRESS_SPACE
)&Process
->VadRoot
)->PageTableRefCountTable
;
1912 Ptrc
[ADDR_TO_PAGE_TABLE(Addr
)]++;
1916 if (Address
> MmSystemRangeStart
||
1917 (Pt
>= (PULONG
)PAGETABLE_MAP
&& Pt
< (PULONG
)PAGETABLE_MAP
+ 1024*1024))
1919 MiFlushTlb(Pt
, Address
);
1925 MmUnmapPageTable(Pt
);
1928 return(STATUS_SUCCESS
);
1933 MmCreateVirtualMapping(PEPROCESS Process
,
1941 for (i
= 0; i
< PageCount
; i
++)
1943 if (!MmIsPageInUse(Pages
[i
]))
1945 DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages
[i
]));
1950 return(MmCreateVirtualMappingUnsafe(Process
,
1959 MmGetPageProtect(PEPROCESS Process
, PVOID Address
)
1965 Entry
= MmGetPageEntryForProcessForPAE(Process
, Address
);
1969 Entry
= MmGetPageEntryForProcess(Process
, Address
);
1972 if (!(Entry
& PA_PRESENT
))
1974 Protect
= PAGE_NOACCESS
;
1978 if (Entry
& PA_READWRITE
)
1980 Protect
= PAGE_READWRITE
;
1984 Protect
= PAGE_EXECUTE_READ
;
1988 Protect
|= PAGE_NOCACHE
;
1992 Protect
|= PAGE_WRITETHROUGH
;
1994 if (!(Entry
& PA_USER
))
1996 Protect
|= PAGE_SYSTEM
;
2005 MmSetPageProtect(PEPROCESS Process
, PVOID Address
, ULONG flProtect
)
2007 ULONG Attributes
= 0;
2008 BOOLEAN NoExecute
= FALSE
;
2010 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
2011 Process
, Address
, flProtect
);
2013 Attributes
= ProtectToPTE(flProtect
);
2014 if (Attributes
& 0x80000000)
2018 Attributes
&= 0xfff;
2019 if (Address
>= MmSystemRangeStart
)
2021 Attributes
&= ~PA_USER
;
2022 if (Ke386GlobalPagesEnabled
)
2024 Attributes
|= PA_GLOBAL
;
2029 Attributes
|= PA_USER
;
2034 ULONGLONG tmpPte
, Pte
;
2036 Pt
= MmGetPageTableForProcessForPAE(Process
, Address
, FALSE
);
2039 DPRINT1("Address %x\n", Address
);
2045 tmpPte
= PAE_PAGE_MASK(Pte
) | Attributes
| (Pte
& (PA_ACCESSED
|PA_DIRTY
));
2048 tmpPte
|= 0x8000000000000000LL
;
2052 tmpPte
&= ~0x8000000000000000LL
;
2054 } while (Pte
!= ExfInterlockedCompareExchange64UL(Pt
, &tmpPte
, &Pte
));
2056 MiFlushTlb((PULONG
)Pt
, Address
);
2062 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
2067 InterlockedExchange((PLONG
)Pt
, PAGE_MASK(*Pt
) | Attributes
| (*Pt
& (PA_ACCESSED
|PA_DIRTY
)));
2068 MiFlushTlb(Pt
, Address
);
2075 PHYSICAL_ADDRESS NTAPI
2076 MmGetPhysicalAddress(PVOID vaddr
)
2078 * FUNCTION: Returns the physical address corresponding to a virtual address
2083 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr
);
2087 Pte
= MmGetPageEntryForProcessForPAE(NULL
, vaddr
);
2088 if (Pte
!= 0 && Pte
& PA_PRESENT
)
2090 p
.QuadPart
= PAE_PAGE_MASK(Pte
);
2091 p
.u
.LowPart
|= (ULONG_PTR
)vaddr
& (PAGE_SIZE
- 1);
2101 Pte
= MmGetPageEntryForProcess(NULL
, vaddr
);
2102 if (Pte
!= 0 && Pte
& PA_PRESENT
)
2104 p
.QuadPart
= PAGE_MASK(Pte
);
2105 p
.u
.LowPart
|= (ULONG_PTR
)vaddr
& (PAGE_SIZE
- 1);
2117 MmCreateHyperspaceMapping(PFN_TYPE Page
)
2125 ULONGLONG ZeroEntry
= 0LL;
2128 Entry
= PFN_TO_PTE(Page
) | PA_PRESENT
| PA_READWRITE
;
2129 Pte
= PAE_ADDR_TO_PTE(HYPERSPACE
) + Page
% 1024;
2133 for (i
= Page
%1024; i
< 1024; i
++, Pte
++)
2135 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2142 Pte
= PAE_ADDR_TO_PTE(HYPERSPACE
);
2143 for (i
= 0; i
< Page
% 1024; i
++, Pte
++)
2145 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2150 if (i
>= Page
% 1024)
2158 for (i
= Page
%1024; (LONG
)i
>= 0; i
--, Pte
--)
2160 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2167 Pte
= PAE_ADDR_TO_PTE(HYPERSPACE
) + 1023;
2168 for (i
= 1023; i
> Page
% 1024; i
--, Pte
--)
2170 if (0LL == ExfInterlockedCompareExchange64UL(Pte
, &Entry
, &ZeroEntry
))
2175 if (i
<= Page
% 1024)
2186 Entry
= PFN_TO_PTE(Page
) | PA_PRESENT
| PA_READWRITE
;
2187 Pte
= ADDR_TO_PTE(HYPERSPACE
) + Page
% 1024;
2190 for (i
= Page
% 1024; i
< 1024; i
++, Pte
++)
2192 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2199 Pte
= ADDR_TO_PTE(HYPERSPACE
);
2200 for (i
= 0; i
< Page
% 1024; i
++, Pte
++)
2202 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2207 if (i
>= Page
% 1024)
2215 for (i
= Page
% 1024; (LONG
)i
>= 0; i
--, Pte
--)
2217 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2224 Pte
= ADDR_TO_PTE(HYPERSPACE
) + 1023;
2225 for (i
= 1023; i
> Page
% 1024; i
--, Pte
--)
2227 if (0 == InterlockedCompareExchange((PLONG
)Pte
, (LONG
)Entry
, 0))
2232 if (i
<= Page
% 1024)
2239 Address
= (PVOID
)((ULONG_PTR
)HYPERSPACE
+ i
* PAGE_SIZE
);
2246 MmChangeHyperspaceMapping(PVOID Address
, PFN_TYPE NewPage
)
2249 ASSERT (IS_HYPERSPACE(Address
));
2252 ULONGLONG Entry
= PAE_PFN_TO_PTE(NewPage
) | PA_PRESENT
| PA_READWRITE
;
2253 Entry
= (ULONG
)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address
), &Entry
);
2254 Pfn
= PAE_PTE_TO_PFN(Entry
);
2259 Entry
= InterlockedExchange((PLONG
)ADDR_TO_PTE(Address
), PFN_TO_PTE(NewPage
) | PA_PRESENT
| PA_READWRITE
);
2260 Pfn
= PTE_TO_PFN(Entry
);
2268 MmDeleteHyperspaceMapping(PVOID Address
)
2271 ASSERT (IS_HYPERSPACE(Address
));
2274 ULONGLONG Entry
= 0LL;
2275 Entry
= (ULONG
)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address
), &Entry
);
2276 Pfn
= PAE_PTE_TO_PFN(Entry
);
2281 Entry
= InterlockedExchange((PLONG
)ADDR_TO_PTE(Address
), 0);
2282 Pfn
= PTE_TO_PFN(Entry
);
2290 MmUpdatePageDir(PEPROCESS Process
, PVOID Address
, ULONG Size
)
2292 ULONG StartOffset
, EndOffset
, Offset
;
2294 if (Address
< MmSystemRangeStart
)
2300 PULONGLONG PageDirTable
;
2302 ULONGLONG ZeroPde
= 0LL;
2305 for (i
= PAE_ADDR_TO_PDTE_OFFSET(Address
); i
<= PAE_ADDR_TO_PDTE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
)); i
++)
2307 if (i
== PAE_ADDR_TO_PDTE_OFFSET(Address
))
2309 StartOffset
= PAE_ADDR_TO_PDE_PAGE_OFFSET(Address
);
2315 if (i
== PAE_ADDR_TO_PDTE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
)))
2317 EndOffset
= PAE_ADDR_TO_PDE_PAGE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
));
2324 if (Process
!= NULL
&& Process
!= PsGetCurrentProcess())
2326 PageDirTable
= MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.QuadPart
));
2327 Pde
= (PULONGLONG
)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable
[i
]));
2328 MmDeleteHyperspaceMapping(PageDirTable
);
2332 Pde
= (PULONGLONG
)PAE_PAGEDIRECTORY_MAP
+ i
*512;
2335 for (Offset
= StartOffset
; Offset
<= EndOffset
; Offset
++)
2337 if (i
* 512 + Offset
< PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) || i
* 512 + Offset
>= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
)+4)
2339 (void)ExfInterlockedCompareExchange64UL(&Pde
[Offset
], &MmGlobalKernelPageDirectoryForPAE
[i
*512 + Offset
], &ZeroPde
);
2342 MmUnmapPageTable((PULONG
)Pde
);
2348 StartOffset
= ADDR_TO_PDE_OFFSET(Address
);
2349 EndOffset
= ADDR_TO_PDE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
));
2351 if (Process
!= NULL
&& Process
!= PsGetCurrentProcess())
2353 Pde
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
.u
.LowPart
));
2357 Pde
= (PULONG
)PAGEDIRECTORY_MAP
;
2359 for (Offset
= StartOffset
; Offset
<= EndOffset
; Offset
++)
2361 if (Offset
!= ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
))
2363 (void)InterlockedCompareExchangeUL(&Pde
[Offset
], MmGlobalKernelPageDirectory
[Offset
], 0);
2366 if (Pde
!= (PULONG
)PAGEDIRECTORY_MAP
)
2368 MmDeleteHyperspaceMapping(Pde
);
2376 MmInitGlobalKernelPageDirectory(VOID
)
2380 DPRINT("MmInitGlobalKernelPageDirectory()\n");
2384 PULONGLONG CurrentPageDirectory
= (PULONGLONG
)PAE_PAGEDIRECTORY_MAP
;
2385 for (i
= PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart
); i
< 4 * 512; i
++)
2387 if (!(i
>= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) && i
< PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) + 4) &&
2388 !(i
>= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE
) && i
< PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE
) + 2) &&
2389 0LL == MmGlobalKernelPageDirectoryForPAE
[i
] && 0LL != CurrentPageDirectory
[i
])
2391 (void)ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE
[i
], &CurrentPageDirectory
[i
]);
2392 if (Ke386GlobalPagesEnabled
)
2394 MmGlobalKernelPageDirectoryForPAE
[i
] |= PA_GLOBAL
;
2395 CurrentPageDirectory
[i
] |= PA_GLOBAL
;
2402 PULONG CurrentPageDirectory
= (PULONG
)PAGEDIRECTORY_MAP
;
2403 for (i
= ADDR_TO_PDE_OFFSET(MmSystemRangeStart
); i
< 1024; i
++)
2405 if (i
!= ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) &&
2406 i
!= ADDR_TO_PDE_OFFSET(HYPERSPACE
) &&
2407 0 == MmGlobalKernelPageDirectory
[i
] && 0 != CurrentPageDirectory
[i
])
2409 MmGlobalKernelPageDirectory
[i
] = CurrentPageDirectory
[i
];
2410 if (Ke386GlobalPagesEnabled
)
2412 MmGlobalKernelPageDirectory
[i
] |= PA_GLOBAL
;
2413 CurrentPageDirectory
[i
] |= PA_GLOBAL
;
2422 MiGetUserPageDirectoryCount(VOID
)
2424 return Ke386Pae
? PAE_ADDR_TO_PDE_OFFSET(MmSystemRangeStart
) : ADDR_TO_PDE_OFFSET(MmSystemRangeStart
);
2430 MiInitPageDirectoryMap(VOID
)
2432 MEMORY_AREA
* kernel_map_desc
= NULL
;
2433 MEMORY_AREA
* hyperspace_desc
= NULL
;
2434 PHYSICAL_ADDRESS BoundaryAddressMultiple
;
2438 DPRINT("MiInitPageDirectoryMap()\n");
2440 BoundaryAddressMultiple
.QuadPart
= 0;
2441 BaseAddress
= (PVOID
)PAGETABLE_MAP
;
2442 Status
= MmCreateMemoryArea(MmGetKernelAddressSpace(),
2445 Ke386Pae
? 0x800000 : 0x400000,
2450 BoundaryAddressMultiple
);
2451 if (!NT_SUCCESS(Status
))
2455 BaseAddress
= (PVOID
)HYPERSPACE
;
2456 Status
= MmCreateMemoryArea(MmGetKernelAddressSpace(),
2464 BoundaryAddressMultiple
);
2465 if (!NT_SUCCESS(Status
))