2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/i386/page.c
5 * PURPOSE: Low level memory managment manipulation
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
10 /* INCLUDES ***************************************************************/
16 #if defined (ALLOC_PRAGMA)
17 #pragma alloc_text(INIT, MmInitGlobalKernelPageDirectory)
18 #pragma alloc_text(INIT, MiInitPageDirectoryMap)
22 /* GLOBALS *****************************************************************/
24 #define PA_BIT_PRESENT (0)
25 #define PA_BIT_READWRITE (1)
26 #define PA_BIT_USER (2)
29 #define PA_BIT_ACCESSED (5)
30 #define PA_BIT_DIRTY (6)
31 #define PA_BIT_GLOBAL (8)
33 #define PA_PRESENT (1 << PA_BIT_PRESENT)
34 #define PA_READWRITE (1 << PA_BIT_READWRITE)
35 #define PA_USER (1 << PA_BIT_USER)
36 #define PA_DIRTY (1 << PA_BIT_DIRTY)
37 #define PA_WT (1 << PA_BIT_WT)
38 #define PA_CD (1 << PA_BIT_CD)
39 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
40 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
42 #define HYPERSPACE (0xc0400000)
43 #define IS_HYPERSPACE(v) (((ULONG)(v) >= HYPERSPACE && (ULONG)(v) < HYPERSPACE + 0x400000))
45 ULONG MmGlobalKernelPageDirectory
[1024];
47 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
48 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
51 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
53 __inline LARGE_INTEGER
PTE_TO_PAGE(ULONG npage
)
56 dummy
.QuadPart
= (LONGLONG
)(PAGE_MASK(npage
));
61 /* FUNCTIONS ***************************************************************/
63 BOOLEAN
MmUnmapPageTable(PULONG Pt
);
66 MiFlushTlb(PULONG Pt
, PVOID Address
)
68 if ((Pt
&& MmUnmapPageTable(Pt
)) || Address
>= MmSystemRangeStart
)
70 KeInvalidateTlbEntry(Address
);
75 ProtectToPTE(ULONG flProtect
)
79 if (flProtect
& (PAGE_NOACCESS
|PAGE_GUARD
))
83 else if (flProtect
& PAGE_IS_WRITABLE
)
85 Attributes
= PA_PRESENT
| PA_READWRITE
;
87 else if (flProtect
& (PAGE_IS_READABLE
| PAGE_IS_EXECUTABLE
))
89 Attributes
= PA_PRESENT
;
93 DPRINT1("Unknown main protection type.\n");
94 KeBugCheck(MEMORY_MANAGEMENT
);
97 if (flProtect
& PAGE_SYSTEM
)
102 Attributes
= Attributes
| PA_USER
;
104 if (flProtect
& PAGE_NOCACHE
)
106 Attributes
= Attributes
| PA_CD
;
108 if (flProtect
& PAGE_WRITETHROUGH
)
110 Attributes
= Attributes
| PA_WT
;
116 MmGetPageTableForProcess(PEPROCESS Process
, PVOID Address
, BOOLEAN Create
)
118 ULONG PdeOffset
= ADDR_TO_PDE_OFFSET(Address
);
124 if (Address
< MmSystemRangeStart
&& Process
&& Process
!= PsGetCurrentProcess())
126 PageDir
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
[0]));
129 KeBugCheck(MEMORY_MANAGEMENT
);
131 if (0 == InterlockedCompareExchangePte(&PageDir
[PdeOffset
], 0, 0))
135 MmDeleteHyperspaceMapping(PageDir
);
138 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
139 if (!NT_SUCCESS(Status
) || Pfn
== 0)
141 KeBugCheck(MEMORY_MANAGEMENT
);
143 Entry
= InterlockedCompareExchangePte(&PageDir
[PdeOffset
], PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
, 0);
146 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
147 Pfn
= PTE_TO_PFN(Entry
);
152 Pfn
= PTE_TO_PFN(PageDir
[PdeOffset
]);
154 MmDeleteHyperspaceMapping(PageDir
);
155 Pt
= MmCreateHyperspaceMapping(Pfn
);
158 KeBugCheck(MEMORY_MANAGEMENT
);
160 return Pt
+ ADDR_TO_PTE_OFFSET(Address
);
162 PageDir
= (PULONG
)MiAddressToPde(Address
);
163 if (0 == InterlockedCompareExchangePte(PageDir
, 0, 0))
165 if (Address
>= MmSystemRangeStart
)
167 if (0 == InterlockedCompareExchangePte(&MmGlobalKernelPageDirectory
[PdeOffset
], 0, 0))
173 Status
= MmRequestPageMemoryConsumer(MC_SYSTEM
, FALSE
, &Pfn
);
174 if (!NT_SUCCESS(Status
) || Pfn
== 0)
176 KeBugCheck(MEMORY_MANAGEMENT
);
178 Entry
= PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
;
179 if(0 != InterlockedCompareExchangePte(&MmGlobalKernelPageDirectory
[PdeOffset
], Entry
, 0))
181 MmReleasePageMemoryConsumer(MC_SYSTEM
, Pfn
);
183 InterlockedExchangePte(PageDir
, MmGlobalKernelPageDirectory
[PdeOffset
]);
184 RtlZeroMemory(MiPteToAddress(PageDir
), PAGE_SIZE
);
185 return (PULONG
)MiAddressToPte(Address
);
187 InterlockedExchangePte(PageDir
, MmGlobalKernelPageDirectory
[PdeOffset
]);
195 Status
= MmRequestPageMemoryConsumer(MC_NPPOOL
, FALSE
, &Pfn
);
196 if (!NT_SUCCESS(Status
) || Pfn
== 0)
198 KeBugCheck(MEMORY_MANAGEMENT
);
200 Entry
= InterlockedCompareExchangePte(PageDir
, PFN_TO_PTE(Pfn
) | PA_PRESENT
| PA_READWRITE
| PA_USER
, 0);
203 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
207 return (PULONG
)MiAddressToPte(Address
);
210 BOOLEAN
MmUnmapPageTable(PULONG Pt
)
212 if (Pt
>= (PULONG
)PAGETABLE_MAP
&& Pt
< (PULONG
)PAGETABLE_MAP
+ 1024*1024)
219 MmDeleteHyperspaceMapping((PVOID
)PAGE_ROUND_DOWN(Pt
));
224 static ULONG
MmGetPageEntryForProcess(PEPROCESS Process
, PVOID Address
)
229 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
233 MmUnmapPageTable(Pt
);
241 MmGetPfnForProcess(PEPROCESS Process
,
245 Entry
= MmGetPageEntryForProcess(Process
, Address
);
246 if (!(Entry
& PA_PRESENT
))
250 return(PTE_TO_PFN(Entry
));
255 MmDisableVirtualMapping(PEPROCESS Process
, PVOID Address
, BOOLEAN
* WasDirty
, PPFN_NUMBER Page
)
257 * FUNCTION: Delete a virtual mapping
264 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
267 KeBugCheck(MEMORY_MANAGEMENT
);
270 * Atomically disable the present bit and get the old value.
275 } while (Pte
!= InterlockedCompareExchangePte(Pt
, Pte
& ~PA_PRESENT
, Pte
));
277 MiFlushTlb(Pt
, Address
);
278 WasValid
= (PAGE_MASK(Pte
) != 0);
281 KeBugCheck(MEMORY_MANAGEMENT
);
285 * Return some information to the caller
287 if (WasDirty
!= NULL
)
289 *WasDirty
= Pte
& PA_DIRTY
;
293 *Page
= PTE_TO_PFN(Pte
);
299 MmRawDeleteVirtualMapping(PVOID Address
)
303 Pt
= MmGetPageTableForProcess(NULL
, Address
, FALSE
);
307 * Set the entry to zero
309 InterlockedExchangePte(Pt
, 0);
310 MiFlushTlb(Pt
, Address
);
316 MmDeleteVirtualMapping(PEPROCESS Process
, PVOID Address
, BOOLEAN FreePage
,
317 BOOLEAN
* WasDirty
, PPFN_NUMBER Page
)
319 * FUNCTION: Delete a virtual mapping
322 BOOLEAN WasValid
= FALSE
;
327 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
328 Process
, Address
, FreePage
, WasDirty
, Page
);
330 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
334 if (WasDirty
!= NULL
)
346 * Atomically set the entry to zero and get the old value.
348 Pte
= InterlockedExchangePte(Pt
, 0);
350 MiFlushTlb(Pt
, Address
);
352 WasValid
= (PAGE_MASK(Pte
) != 0);
355 Pfn
= PTE_TO_PFN(Pte
);
362 if (FreePage
&& WasValid
)
364 MmReleasePageMemoryConsumer(MC_NPPOOL
, Pfn
);
368 * Return some information to the caller
370 if (WasDirty
!= NULL
)
372 *WasDirty
= Pte
& PA_DIRTY
? TRUE
: FALSE
;
382 MmDeletePageFileMapping(PEPROCESS Process
, PVOID Address
,
383 SWAPENTRY
* SwapEntry
)
385 * FUNCTION: Delete a virtual mapping
391 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
400 * Atomically set the entry to zero and get the old value.
402 Pte
= InterlockedExchangePte(Pt
, 0);
404 MiFlushTlb(Pt
, Address
);
407 * Return some information to the caller
409 *SwapEntry
= Pte
>> 1;
413 Mmi386MakeKernelPageTableGlobal(PVOID PAddress
)
416 Pde
= (PULONG
)MiAddressToPde(PAddress
);
419 Pt
= MmGetPageTableForProcess(NULL
, PAddress
, FALSE
);
430 MmIsDirtyPage(PEPROCESS Process
, PVOID Address
)
432 return MmGetPageEntryForProcess(Process
, Address
) & PA_DIRTY
? TRUE
: FALSE
;
437 MmSetCleanPage(PEPROCESS Process
, PVOID Address
)
442 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
444 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
445 KeBugCheck(MEMORY_MANAGEMENT
);
448 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
452 KeBugCheck(MEMORY_MANAGEMENT
);
458 } while (Pte
!= InterlockedCompareExchangePte(Pt
, Pte
& ~PA_DIRTY
, Pte
));
462 MiFlushTlb(Pt
, Address
);
466 MmUnmapPageTable(Pt
);
472 MmSetDirtyPage(PEPROCESS Process
, PVOID Address
)
477 if (Address
< MmSystemRangeStart
&& Process
== NULL
)
479 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
480 KeBugCheck(MEMORY_MANAGEMENT
);
483 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
486 KeBugCheck(MEMORY_MANAGEMENT
);
492 } while (Pte
!= InterlockedCompareExchangePte(Pt
, Pte
| PA_DIRTY
, Pte
));
493 if (!(Pte
& PA_DIRTY
))
495 MiFlushTlb(Pt
, Address
);
499 MmUnmapPageTable(Pt
);
505 MmEnableVirtualMapping(PEPROCESS Process
, PVOID Address
)
510 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
513 KeBugCheck(MEMORY_MANAGEMENT
);
519 } while (Pte
!= InterlockedCompareExchangePte(Pt
, Pte
| PA_PRESENT
, Pte
));
520 if (!(Pte
& PA_PRESENT
))
522 MiFlushTlb(Pt
, Address
);
526 MmUnmapPageTable(Pt
);
532 MmIsPagePresent(PEPROCESS Process
, PVOID Address
)
534 return MmGetPageEntryForProcess(Process
, Address
) & PA_PRESENT
;
539 MmIsPageSwapEntry(PEPROCESS Process
, PVOID Address
)
542 Entry
= MmGetPageEntryForProcess(Process
, Address
);
543 return !(Entry
& PA_PRESENT
) && (Entry
& 0x800) && Entry
!= 0;
548 MmCreatePageFileMapping(PEPROCESS Process
,
555 if (Process
== NULL
&& Address
< MmSystemRangeStart
)
557 DPRINT1("No process\n");
558 KeBugCheck(MEMORY_MANAGEMENT
);
560 if (Process
!= NULL
&& Address
>= MmSystemRangeStart
)
562 DPRINT1("Setting kernel address with process context\n");
563 KeBugCheck(MEMORY_MANAGEMENT
);
566 if (SwapEntry
& (1 << 31))
568 KeBugCheck(MEMORY_MANAGEMENT
);
571 Pt
= MmGetPageTableForProcess(Process
, Address
, TRUE
);
574 KeBugCheck(MEMORY_MANAGEMENT
);
577 InterlockedExchangePte(Pt
, SwapEntry
<< 1);
580 MiFlushTlb(Pt
, Address
);
584 MmUnmapPageTable(Pt
);
587 return(STATUS_SUCCESS
);
593 MmCreateVirtualMappingUnsafe(PEPROCESS Process
,
602 ULONG oldPdeOffset
, PdeOffset
;
605 BOOLEAN NoExecute
= FALSE
;
607 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
608 Process
, Address
, flProtect
, Pages
, *Pages
, PageCount
);
612 if (Address
< MmSystemRangeStart
)
614 DPRINT1("No process\n");
615 KeBugCheck(MEMORY_MANAGEMENT
);
617 if (PageCount
> 0x10000 ||
618 (ULONG_PTR
) Address
/ PAGE_SIZE
+ PageCount
> 0x100000)
620 DPRINT1("Page count too large\n");
621 KeBugCheck(MEMORY_MANAGEMENT
);
626 if (Address
>= MmSystemRangeStart
)
628 DPRINT1("Setting kernel address with process context\n");
629 KeBugCheck(MEMORY_MANAGEMENT
);
631 if (PageCount
> (ULONG_PTR
)MmSystemRangeStart
/ PAGE_SIZE
||
632 (ULONG_PTR
) Address
/ PAGE_SIZE
+ PageCount
>
633 (ULONG_PTR
)MmSystemRangeStart
/ PAGE_SIZE
)
635 DPRINT1("Page Count too large\n");
636 KeBugCheck(MEMORY_MANAGEMENT
);
640 Attributes
= ProtectToPTE(flProtect
);
641 if (Attributes
& 0x80000000)
646 if (Address
>= MmSystemRangeStart
)
648 Attributes
&= ~PA_USER
;
652 Attributes
|= PA_USER
;
656 oldPdeOffset
= ADDR_TO_PDE_OFFSET(Addr
) + 1;
657 for (i
= 0; i
< PageCount
; i
++, Addr
= (PVOID
)((ULONG_PTR
)Addr
+ PAGE_SIZE
))
659 if (!(Attributes
& PA_PRESENT
) && Pages
[i
] != 0)
661 DPRINT1("Setting physical address but not allowing access at address "
662 "0x%.8X with attributes %x/%x.\n",
663 Addr
, Attributes
, flProtect
);
664 KeBugCheck(MEMORY_MANAGEMENT
);
666 PdeOffset
= ADDR_TO_PDE_OFFSET(Addr
);
667 if (oldPdeOffset
!= PdeOffset
)
669 MmUnmapPageTable(Pt
);
670 Pt
= MmGetPageTableForProcess(Process
, Addr
, TRUE
);
673 KeBugCheck(MEMORY_MANAGEMENT
);
680 oldPdeOffset
= PdeOffset
;
683 if (PAGE_MASK(Pte
) != 0 && !(Pte
& PA_PRESENT
) && (Pte
& 0x800))
685 DPRINT1("Bad PTE %lx\n", Pte
);
686 KeBugCheck(MEMORY_MANAGEMENT
);
688 InterlockedExchangePte(Pt
, PFN_TO_PTE(Pages
[i
]) | Attributes
);
691 if (Address
> MmSystemRangeStart
||
692 (Pt
>= (PULONG
)PAGETABLE_MAP
&& Pt
< (PULONG
)PAGETABLE_MAP
+ 1024*1024))
694 MiFlushTlb(Pt
, Address
);
700 MmUnmapPageTable(Pt
);
703 return(STATUS_SUCCESS
);
708 MmCreateVirtualMapping(PEPROCESS Process
,
716 for (i
= 0; i
< PageCount
; i
++)
718 if (!MmIsPageInUse(Pages
[i
]))
720 DPRINT1("Page at address %x not in use\n", PFN_TO_PTE(Pages
[i
]));
721 KeBugCheck(MEMORY_MANAGEMENT
);
725 return(MmCreateVirtualMappingUnsafe(Process
,
734 MmGetPageProtect(PEPROCESS Process
, PVOID Address
)
739 Entry
= MmGetPageEntryForProcess(Process
, Address
);
742 if (!(Entry
& PA_PRESENT
))
744 Protect
= PAGE_NOACCESS
;
748 if (Entry
& PA_READWRITE
)
750 Protect
= PAGE_READWRITE
;
754 Protect
= PAGE_EXECUTE_READ
;
758 Protect
|= PAGE_NOCACHE
;
762 Protect
|= PAGE_WRITETHROUGH
;
764 if (!(Entry
& PA_USER
))
766 Protect
|= PAGE_SYSTEM
;
775 MmSetPageProtect(PEPROCESS Process
, PVOID Address
, ULONG flProtect
)
777 ULONG Attributes
= 0;
778 BOOLEAN NoExecute
= FALSE
;
781 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
782 Process
, Address
, flProtect
);
784 Attributes
= ProtectToPTE(flProtect
);
786 if (Attributes
& 0x80000000)
791 if (Address
>= MmSystemRangeStart
)
793 Attributes
&= ~PA_USER
;
797 Attributes
|= PA_USER
;
800 Pt
= MmGetPageTableForProcess(Process
, Address
, FALSE
);
803 KeBugCheck(MEMORY_MANAGEMENT
);
805 InterlockedExchangePte(Pt
, PAGE_MASK(*Pt
) | Attributes
| (*Pt
& (PA_ACCESSED
|PA_DIRTY
)));
806 MiFlushTlb(Pt
, Address
);
812 PHYSICAL_ADDRESS NTAPI
813 MmGetPhysicalAddress(PVOID vaddr
)
815 * FUNCTION: Returns the physical address corresponding to a virtual address
821 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr
);
822 Pte
= MmGetPageEntryForProcess(NULL
, vaddr
);
823 if (Pte
!= 0 && Pte
& PA_PRESENT
)
825 p
.QuadPart
= PAGE_MASK(Pte
);
826 p
.u
.LowPart
|= (ULONG_PTR
)vaddr
& (PAGE_SIZE
- 1);
837 MmUpdatePageDir(PEPROCESS Process
, PVOID Address
, ULONG Size
)
839 ULONG StartOffset
, EndOffset
, Offset
;
843 // Check if the process isn't there anymore
844 // This is probably a bad sign, since it means the caller is setting cr3 to
847 if ((PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
[0]) == 0) && (Process
!= PsGetCurrentProcess()))
849 DPRINT1("Process: %16s is dead: %p\n", Process
->ImageFileName
, Process
->Pcb
.DirectoryTableBase
[0]);
854 if (Address
< MmSystemRangeStart
)
856 KeBugCheck(MEMORY_MANAGEMENT
);
859 StartOffset
= ADDR_TO_PDE_OFFSET(Address
);
860 EndOffset
= ADDR_TO_PDE_OFFSET((PVOID
)((ULONG_PTR
)Address
+ Size
));
862 if (Process
!= NULL
&& Process
!= PsGetCurrentProcess())
864 Pde
= MmCreateHyperspaceMapping(PTE_TO_PFN(Process
->Pcb
.DirectoryTableBase
[0]));
868 Pde
= (PULONG
)PAGEDIRECTORY_MAP
;
870 for (Offset
= StartOffset
; Offset
<= EndOffset
; Offset
++)
872 if (Offset
!= ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
))
874 InterlockedCompareExchangePte(&Pde
[Offset
], MmGlobalKernelPageDirectory
[Offset
], 0);
877 if (Pde
!= (PULONG
)PAGEDIRECTORY_MAP
)
879 MmDeleteHyperspaceMapping(Pde
);
886 MmInitGlobalKernelPageDirectory(VOID
)
889 PULONG CurrentPageDirectory
= (PULONG
)PAGEDIRECTORY_MAP
;
891 DPRINT("MmInitGlobalKernelPageDirectory()\n");
893 for (i
= ADDR_TO_PDE_OFFSET(MmSystemRangeStart
); i
< 1024; i
++)
895 if (i
!= ADDR_TO_PDE_OFFSET(PAGETABLE_MAP
) &&
896 i
!= ADDR_TO_PDE_OFFSET(HYPERSPACE
) &&
897 0 == MmGlobalKernelPageDirectory
[i
] && 0 != CurrentPageDirectory
[i
])
899 MmGlobalKernelPageDirectory
[i
] = CurrentPageDirectory
[i
];