projects / reactos.git / blob
? search:


e309e8a0ab6a57317ee31fa6c2337b4e9e20c563
[reactos.git] / reactos / ntoskrnl / mm / i386 / page.c
1 /*
2 * ReactOS kernel
3 * Copyright (C) 1998, 1999, 2000, 2001 ReactOS Team
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19 /* $Id$
20 *
21 * PROJECT: ReactOS kernel
22 * FILE: ntoskrnl/mm/i386/page.c
23 * PURPOSE: low level memory managment manipulation
24 * PROGRAMER: David Welch (welch@cwcom.net)
25 * UPDATE HISTORY:
26 * 9/3/98: Created
27 */
28
29 /* INCLUDES ***************************************************************/
30
31 #include <ntoskrnl.h>
32 #define NDEBUG
33 #include <internal/debug.h>
34
35 /* GLOBALS *****************************************************************/
36
37 #define PA_BIT_PRESENT (0)
38 #define PA_BIT_READWRITE (1)
39 #define PA_BIT_USER (2)
40 #define PA_BIT_WT (3)
41 #define PA_BIT_CD (4)
42 #define PA_BIT_ACCESSED (5)
43 #define PA_BIT_DIRTY (6)
44 #define PA_BIT_GLOBAL (8)
45
46 #define PA_PRESENT (1 << PA_BIT_PRESENT)
47 #define PA_READWRITE (1 << PA_BIT_READWRITE)
48 #define PA_USER (1 << PA_BIT_USER)
49 #define PA_DIRTY (1 << PA_BIT_DIRTY)
50 #define PA_WT (1 << PA_BIT_WT)
51 #define PA_CD (1 << PA_BIT_CD)
52 #define PA_ACCESSED (1 << PA_BIT_ACCESSED)
53 #define PA_GLOBAL (1 << PA_BIT_GLOBAL)
54
55 #define PAGETABLE_MAP (0xf0000000)
56 #define PAGEDIRECTORY_MAP (0xf0000000 + (PAGETABLE_MAP / (1024)))
57
58 #define PAE_PAGEDIRECTORY_MAP (0xf0000000 + (PAGETABLE_MAP / (512)))
59
60 #define HYPERSPACE (0xf0800000)
61 #define IS_HYPERSPACE(v) (((ULONG)(v) >= 0xF0800000 && (ULONG)(v) < 0xF0C00000))
62
63 ULONG MmGlobalKernelPageDirectory[1024];
64 ULONGLONG MmGlobalKernelPageDirectoryForPAE[2048];
65
66 #define PTE_TO_PFN(X) ((X) >> PAGE_SHIFT)
67 #define PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
68
69 #define PAE_PTE_TO_PFN(X) (PAE_PAGE_MASK(X) >> PAGE_SHIFT)
70 #define PAE_PFN_TO_PTE(X) ((X) << PAGE_SHIFT)
71
72 #if defined(__GNUC__)
73 #define PTE_TO_PAGE(X) ((LARGE_INTEGER)(LONGLONG)(PAGE_MASK(X)))
74 #else
75 __inline LARGE_INTEGER PTE_TO_PAGE(ULONG npage)
76 {
77 LARGE_INTEGER dummy;
78 dummy.QuadPart = (LONGLONG)(PAGE_MASK(npage));
79 return dummy;
80 }
81 #endif
82
83 extern BOOLEAN Ke386Pae;
84 extern BOOLEAN Ke386NoExecute;
85 extern BOOLEAN Ke386GlobalPagesEnabled;
86
87 /* FUNCTIONS ***************************************************************/
88
89 BOOLEAN MmUnmapPageTable(PULONG Pt);
90
91 VOID
92 STDCALL
93 MiFlushTlbIpiRoutine(PVOID Address)
94 {
95 if (Address == (PVOID)0xffffffff)
96 {
97 KeFlushCurrentTb();
98 }
99 else if (Address == (PVOID)0xfffffffe)
100 {
101 FLUSH_TLB;
102 }
103 else
104 {
105 FLUSH_TLB_ONE(Address);
106 }
107 }
108
109 VOID
110 MiFlushTlb(PULONG Pt, PVOID Address)
111 {
112 #ifdef CONFIG_SMP
113 if (Pt)
114 {
115 MmUnmapPageTable(Pt);
116 }
117 if (KeNumberProcessors>1)
118 {
119 KeIpiGenericCall(MiFlushTlbIpiRoutine, Address);
120 }
121 else
122 {
123 MiFlushTlbIpiRoutine(Address);
124 }
125 #else
126 if ((Pt && MmUnmapPageTable(Pt)) || Address >= (PVOID)KERNEL_BASE)
127 {
128 FLUSH_TLB_ONE(Address);
129 }
130 #endif
131 }
132
133
134
135 PULONG
136 MmGetPageDirectory(VOID)
137 {
138 unsigned int page_dir=0;
139 Ke386GetPageTableDirectory(page_dir);
140 return((PULONG)page_dir);
141 }
142
143 static ULONG
144 ProtectToPTE(ULONG flProtect)
145 {
146 ULONG Attributes = 0;
147
148 if (flProtect & (PAGE_NOACCESS|PAGE_GUARD))
149 {
150 Attributes = 0;
151 }
152 else if (flProtect & PAGE_IS_WRITABLE)
153 {
154 Attributes = PA_PRESENT | PA_READWRITE;
155 }
156 else if (flProtect & (PAGE_IS_READABLE | PAGE_IS_EXECUTABLE))
157 {
158 Attributes = PA_PRESENT;
159 }
160 else
161 {
162 DPRINT1("Unknown main protection type.\n");
163 KEBUGCHECK(0);
164 }
165 if (Ke386NoExecute &&
166 !(flProtect & PAGE_IS_EXECUTABLE))
167 {
168 Attributes = Attributes | 0x80000000;
169 }
170
171 if (flProtect & PAGE_SYSTEM)
172 {
173 }
174 else
175 {
176 Attributes = Attributes | PA_USER;
177 }
178 if (flProtect & PAGE_NOCACHE)
179 {
180 Attributes = Attributes | PA_CD;
181 }
182 if (flProtect & PAGE_WRITETHROUGH)
183 {
184 Attributes = Attributes | PA_WT;
185 }
186 return(Attributes);
187 }
188
189 #define ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (1024 * PAGE_SIZE))
190
191 #define ADDR_TO_PDE(v) (PULONG)(PAGEDIRECTORY_MAP + \
192 ((((ULONG)(v)) / (1024 * 1024))&(~0x3)))
193 #define ADDR_TO_PTE(v) (PULONG)(PAGETABLE_MAP + ((((ULONG)(v) / 1024))&(~0x3)))
194
195 #define ADDR_TO_PDE_OFFSET(v) ((((ULONG)(v)) / (1024 * PAGE_SIZE)))
196
197 #define ADDR_TO_PTE_OFFSET(v) ((((ULONG)(v)) % (1024 * PAGE_SIZE)) / PAGE_SIZE)
198
199
200 #define PAE_ADDR_TO_PAGE_TABLE(v) (((ULONG)(v)) / (512 * PAGE_SIZE))
201
202 #define PAE_ADDR_TO_PDE(v) (PULONGLONG) (PAE_PAGEDIRECTORY_MAP + \
203 ((((ULONG_PTR)(v)) / (512 * 512))&(~0x7)))
204 #define PAE_ADDR_TO_PTE(v) (PULONGLONG) (PAGETABLE_MAP + ((((ULONG_PTR)(v) / 512))&(~0x7)))
205
206
207 #define PAE_ADDR_TO_PDTE_OFFSET(v) (((ULONG_PTR)(v)) / (512 * 512 * PAGE_SIZE))
208
209 #define PAE_ADDR_TO_PDE_PAGE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * 512 * PAGE_SIZE)) / (512 * PAGE_SIZE))
210
211 #define PAE_ADDR_TO_PDE_OFFSET(v) (((ULONG_PTR)(v))/ (512 * PAGE_SIZE))
212
213 #define PAE_ADDR_TO_PTE_OFFSET(v) ((((ULONG_PTR)(v)) % (512 * PAGE_SIZE)) / PAGE_SIZE)
214
215
216 NTSTATUS Mmi386ReleaseMmInfo(PEPROCESS Process)
217 {
218 PUSHORT LdtDescriptor;
219 ULONG LdtBase;
220 ULONG i, j;
221
222 DPRINT("Mmi386ReleaseMmInfo(Process %x)\n",Process);
223
224 LdtDescriptor = (PUSHORT) &Process->Pcb.LdtDescriptor[0];
225 LdtBase = LdtDescriptor[1] |
226 ((LdtDescriptor[2] & 0xff) << 16) |
227 ((LdtDescriptor[3] & ~0xff) << 16);
228
229 DPRINT("LdtBase: %x\n", LdtBase);
230
231 if (LdtBase)
232 {
233 ExFreePool((PVOID) LdtBase);
234 }
235
236 if (Ke386Pae)
237 {
238 PULONGLONG PageDirTable;
239 PULONGLONG PageDir;
240 PULONGLONG Pde;
241 ULONG k;
242
243 PageDirTable = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
244 for (i = 0; i < 4; i++)
245 {
246 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[i]));
247 if (i < PAE_ADDR_TO_PDTE_OFFSET(KERNEL_BASE))
248 {
249 for (j = 0; j < 512; j++)
250 {
251 if (PageDir[j] != 0LL)
252 {
253 DPRINT1("ProcessId %d, Pde for %08x - %08x is not freed, RefCount %d\n",
254 Process->UniqueProcessId,
255 (i * 512 + j) * 512 * PAGE_SIZE, (i * 512 + j + 1) * 512 * PAGE_SIZE - 1,
256 Process->AddressSpace.PageTableRefCountTable[i*512 + j]);
257 Pde = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDir[j]));
258 for (k = 0; k < 512; k++)
259 {
260 if(Pde[k] != 0)
261 {
262 if (Pde[k] & PA_PRESENT)
263 {
264 DPRINT1("Page at %08x is not freed\n",
265 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
266 }
267 else
268 {
269 DPRINT1("Swapentry %x at %x is not freed\n",
270 (i * 512 + j) * 512 * PAGE_SIZE + k * PAGE_SIZE);
271 }
272 }
273 }
274 MmDeleteHyperspaceMapping(Pde);
275 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[j]));
276 }
277 }
278 }
279 else
280 {
281 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)]));
282 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1]));
283 }
284 MmDeleteHyperspaceMapping(PageDir);
285 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(PageDirTable[i]));
286 }
287 MmDeleteHyperspaceMapping((PVOID)PageDirTable);
288 MmReleasePageMemoryConsumer(MC_NPPOOL, PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
289 }
290 else
291 {
292 PULONG Pde;
293 PULONG PageDir;
294 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
295 for (i = 0; i < ADDR_TO_PDE_OFFSET(KERNEL_BASE); i++)
296 {
297 if (PageDir[i] != 0)
298 {
299 DPRINT1("Pde for %08x - %08x is not freed, RefCount %d\n",
300 i * 4 * 1024 * 1024, (i + 1) * 4 * 1024 * 1024 - 1,
301 Process->AddressSpace.PageTableRefCountTable[i]);
302 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(PageDir[i]));
303 for (j = 0; j < 1024; j++)
304 {
305 if(Pde[j] != 0)
306 {
307 if (Pde[j] & PA_PRESENT)
308 {
309 DPRINT1("Page at %08x is not freed\n",
310 i * 4 * 1024 * 1024 + j * PAGE_SIZE);
311 }
312 else
313 {
314 DPRINT1("Swapentry %x at %x is not freed\n",
315 Pde[j], i * 4 * 1024 * 1024 + j * PAGE_SIZE);
316 }
317 }
318 }
319 MmDeleteHyperspaceMapping(Pde);
320 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[i]));
321 }
322 }
323 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(PageDir[ADDR_TO_PDE_OFFSET(HYPERSPACE)]));
324 MmDeleteHyperspaceMapping(PageDir);
325 MmReleasePageMemoryConsumer(MC_NPPOOL, PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
326 }
327
328 #if defined(__GNUC__)
329
330 Process->Pcb.DirectoryTableBase.QuadPart = 0LL;
331 #else
332
333 Process->Pcb.DirectoryTableBase.QuadPart = 0;
334 #endif
335
336 DPRINT("Finished Mmi386ReleaseMmInfo()\n");
337 return(STATUS_SUCCESS);
338 }
339
340 NTSTATUS MmCopyMmInfo(PEPROCESS Src, PEPROCESS Dest)
341 {
342 PKPROCESS KProcess = &Dest->Pcb;
343 NTSTATUS Status;
344 ULONG i, j;
345 PFN_TYPE Pfn[7];
346 ULONG Count;
347
348 DPRINT("MmCopyMmInfo(Src %x, Dest %x)\n", Src, Dest);
349
350 Count = Ke386Pae ? 7 : 2;
351
352 for (i = 0; i < Count; i++)
353 {
354 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn[i]);
355 if (!NT_SUCCESS(Status))
356 {
357 for (j = 0; j < i; j++)
358 {
359 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn[j]);
360 }
361 return Status;
362 }
363 }
364
365 if (Ke386Pae)
366 {
367 PULONGLONG PageDirTable;
368 PULONGLONG PageDir;
369
370 PageDirTable = MmCreateHyperspaceMapping(Pfn[0]);
371 for (i = 0; i < 4; i++)
372 {
373 PageDirTable[i] = PAE_PFN_TO_PTE(Pfn[1+i]) | PA_PRESENT;
374 }
375 MmDeleteHyperspaceMapping(PageDirTable);
376 for (i = PAE_ADDR_TO_PDTE_OFFSET(KERNEL_BASE); i < 4; i++)
377 {
378 PageDir = (PULONGLONG)MmCreateHyperspaceMapping(Pfn[i+1]);
379 memcpy(PageDir, &MmGlobalKernelPageDirectoryForPAE[i * 512], 512 * sizeof(ULONGLONG));
380 if (PAE_ADDR_TO_PDTE_OFFSET(PAGETABLE_MAP) == i)
381 {
382 for (j = 0; j < 4; j++)
383 {
384 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(PAGETABLE_MAP) + j] = PAE_PFN_TO_PTE(Pfn[1+j]) | PA_PRESENT | PA_READWRITE;
385 }
386 }
387 if (PAE_ADDR_TO_PDTE_OFFSET(HYPERSPACE) == i)
388 {
389 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)] = PAE_PFN_TO_PTE(Pfn[5]) | PA_PRESENT | PA_READWRITE;
390 PageDir[PAE_ADDR_TO_PDE_PAGE_OFFSET(HYPERSPACE)+1] = PAE_PFN_TO_PTE(Pfn[6]) | PA_PRESENT | PA_READWRITE;
391 }
392 MmDeleteHyperspaceMapping(PageDir);
393 }
394 }
395 else
396 {
397 PULONG PageDirectory;
398 PageDirectory = MmCreateHyperspaceMapping(Pfn[0]);
399
400 memcpy(PageDirectory + ADDR_TO_PDE_OFFSET(KERNEL_BASE),
401 MmGlobalKernelPageDirectory + ADDR_TO_PDE_OFFSET(KERNEL_BASE),
402 (1024 - ADDR_TO_PDE_OFFSET(KERNEL_BASE)) * sizeof(ULONG));
403
404 DPRINT("Addr %x\n",ADDR_TO_PDE_OFFSET(PAGETABLE_MAP));
405 PageDirectory[ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)] = PFN_TO_PTE(Pfn[0]) | PA_PRESENT | PA_READWRITE;
406 PageDirectory[ADDR_TO_PDE_OFFSET(HYPERSPACE)] = PFN_TO_PTE(Pfn[1]) | PA_PRESENT | PA_READWRITE;
407
408 MmDeleteHyperspaceMapping(PageDirectory);
409 }
410 KProcess->DirectoryTableBase.QuadPart = PFN_TO_PTE(Pfn[0]);
411 DPRINT("Finished MmCopyMmInfo()\n");
412 return(STATUS_SUCCESS);
413 }
414
415 VOID MmDeletePageTable(PEPROCESS Process, PVOID Address)
416 {
417 PEPROCESS CurrentProcess = PsGetCurrentProcess();
418
419 if (Process != NULL && Process != CurrentProcess)
420 {
421 KeAttachProcess(&Process->Pcb);
422 }
423
424 if (Ke386Pae)
425 {
426 ULONGLONG ZeroPde = 0LL;
427 ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPde);
428 }
429 else
430 {
431 *(ADDR_TO_PDE(Address)) = 0;
432 }
433 if (Address >= (PVOID)KERNEL_BASE)
434 {
435 KEBUGCHECK(0);
436 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
437 }
438 MiFlushTlb(NULL, Address);
439 if (Process != NULL && Process != CurrentProcess)
440 {
441 KeDetachProcess();
442 }
443 }
444
445 VOID MmFreePageTable(PEPROCESS Process, PVOID Address)
446 {
447 PEPROCESS CurrentProcess = PsGetCurrentProcess();
448 ULONG i;
449 PFN_TYPE Pfn;
450
451 DPRINT("ProcessId %d, Address %x\n", Process->UniqueProcessId, Address);
452 if (Process != NULL && Process != CurrentProcess)
453 {
454 KeAttachProcess(&Process->Pcb);
455 }
456 if (Ke386Pae)
457 {
458 PULONGLONG PageTable;
459 ULONGLONG ZeroPte = 0LL;
460 PageTable = (PULONGLONG)PAGE_ROUND_DOWN((PVOID)PAE_ADDR_TO_PTE(Address));
461 for (i = 0; i < 512; i++)
462 {
463 if (PageTable[i] != 0LL)
464 {
465 DbgPrint("Page table entry not clear at %x/%x (is %I64x)\n",
466 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
467 KEBUGCHECK(0);
468 }
469 }
470 Pfn = PAE_PTE_TO_PFN(*(PAE_ADDR_TO_PDE(Address)));
471 ExfpInterlockedExchange64UL(PAE_ADDR_TO_PDE(Address), &ZeroPte);
472 }
473 else
474 {
475 PULONG PageTable;
476 PageTable = (PULONG)PAGE_ROUND_DOWN((PVOID)ADDR_TO_PTE(Address));
477 for (i = 0; i < 1024; i++)
478 {
479 if (PageTable[i] != 0)
480 {
481 DbgPrint("Page table entry not clear at %x/%x (is %x)\n",
482 ((ULONG)Address / (4*1024*1024)), i, PageTable[i]);
483 KEBUGCHECK(0);
484 }
485 }
486 Pfn = PTE_TO_PFN(*(ADDR_TO_PDE(Address)));
487 *(ADDR_TO_PDE(Address)) = 0;
488 }
489 MiFlushTlb(NULL, Address);
490
491 if (Address >= (PVOID)KERNEL_BASE)
492 {
493 // MmGlobalKernelPageDirectory[ADDR_TO_PDE_OFFSET(Address)] = 0;
494 KEBUGCHECK(0);
495 }
496 else
497 {
498 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
499 }
500 if (Process != NULL && Process != CurrentProcess)
501 {
502 KeDetachProcess();
503 }
504 }
505
506 static PULONGLONG
507 MmGetPageTableForProcessForPAE(PEPROCESS Process, PVOID Address, BOOLEAN Create)
508 {
509 NTSTATUS Status;
510 PFN_TYPE Pfn;
511 ULONGLONG Entry;
512 ULONGLONG ZeroEntry = 0LL;
513 PULONGLONG Pt;
514 PULONGLONG PageDir;
515 PULONGLONG PageDirTable;
516
517 DPRINT("MmGetPageTableForProcessForPAE(%x %x %d)\n",
518 Process, Address, Create);
519 if (Address >= (PVOID)PAGETABLE_MAP && Address < (PVOID)PAGETABLE_MAP + 0x800000)
520 {
521 KEBUGCHECK(0);
522 }
523 if (Address < (PVOID)KERNEL_BASE && Process && Process != PsGetCurrentProcess())
524 {
525 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
526 if (PageDirTable == NULL)
527 {
528 KEBUGCHECK(0);
529 }
530 PageDir = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(PageDirTable[PAE_ADDR_TO_PDTE_OFFSET(Address)]));
531 MmDeleteHyperspaceMapping(PageDirTable);
532 if (PageDir == NULL)
533 {
534 KEBUGCHECK(0);
535 }
536 PageDir += PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
537 Entry = ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry);
538 if (Entry == 0LL)
539 {
540 if (Create == FALSE)
541 {
542 MmDeleteHyperspaceMapping(PageDir);
543 return NULL;
544 }
545 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
546 if (!NT_SUCCESS(Status))
547 {
548 KEBUGCHECK(0);
549 }
550 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
551 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
552 if (Entry != 0LL)
553 {
554 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
555 Pfn = PAE_PTE_TO_PFN(Entry);
556 }
557 }
558 else
559 {
560 Pfn = PAE_PTE_TO_PFN(Entry);
561 }
562 MmDeleteHyperspaceMapping(PageDir);
563 Pt = MmCreateHyperspaceMapping(Pfn);
564 if (Pt == NULL)
565 {
566 KEBUGCHECK(0);
567 }
568 return Pt + PAE_ADDR_TO_PTE_OFFSET(Address);
569 }
570 PageDir = PAE_ADDR_TO_PDE(Address);
571 if (0LL == ExfInterlockedCompareExchange64UL(PageDir, &ZeroEntry, &ZeroEntry))
572 {
573 if (Address >= (PVOID)KERNEL_BASE)
574 {
575 if (MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)] == 0LL)
576 {
577 if (Create == FALSE)
578 {
579 return NULL;
580 }
581 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
582 if (!NT_SUCCESS(Status))
583 {
584 KEBUGCHECK(0);
585 }
586 Entry = PAE_PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
587 if (Ke386GlobalPagesEnabled)
588 {
589 Entry |= PA_GLOBAL;
590 }
591 if (0LL != ExfInterlockedCompareExchange64UL(&MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &Entry, &ZeroEntry))
592 {
593 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
594 }
595 }
596 ExfInterlockedCompareExchange64UL(PageDir, &MmGlobalKernelPageDirectoryForPAE[PAE_ADDR_TO_PDE_OFFSET(Address)], &ZeroEntry);
597 }
598 else
599 {
600 if (Create == FALSE)
601 {
602 return NULL;
603 }
604 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
605 if (!NT_SUCCESS(Status))
606 {
607 KEBUGCHECK(0);
608 }
609 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER;
610 Entry = ExfInterlockedCompareExchange64UL(PageDir, &Entry, &ZeroEntry);
611 if (Entry != 0LL)
612 {
613 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
614 }
615 }
616 }
617 return (PULONGLONG)PAE_ADDR_TO_PTE(Address);
618 }
619
620 static PULONG
621 MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create)
622 {
623 ULONG PdeOffset = ADDR_TO_PDE_OFFSET(Address);
624 NTSTATUS Status;
625 PFN_TYPE Pfn;
626 ULONG Entry;
627 PULONG Pt, PageDir;
628
629 if (Address < (PVOID)KERNEL_BASE && Process && Process != PsGetCurrentProcess())
630 {
631 PageDir = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
632 if (PageDir == NULL)
633 {
634 KEBUGCHECK(0);
635 }
636 if (0 == InterlockedCompareExchangeUL(&PageDir[PdeOffset], 0, 0))
637 {
638 if (Create == FALSE)
639 {
640 MmDeleteHyperspaceMapping(PageDir);
641 return NULL;
642 }
643 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
644 if (!NT_SUCCESS(Status) || Pfn == 0)
645 {
646 KEBUGCHECK(0);
647 }
648 Entry = InterlockedCompareExchangeUL(&PageDir[PdeOffset], PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
649 if (Entry != 0)
650 {
651 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
652 Pfn = PTE_TO_PFN(Entry);
653 }
654 }
655 else
656 {
657 Pfn = PTE_TO_PFN(PageDir[PdeOffset]);
658 }
659 MmDeleteHyperspaceMapping(PageDir);
660 Pt = MmCreateHyperspaceMapping(Pfn);
661 if (Pt == NULL)
662 {
663 KEBUGCHECK(0);
664 }
665 return Pt + ADDR_TO_PTE_OFFSET(Address);
666 }
667 PageDir = ADDR_TO_PDE(Address);
668 if (0 == InterlockedCompareExchangeUL(PageDir, 0, 0))
669 {
670 if (Address >= (PVOID)KERNEL_BASE)
671 {
672 if (0 == InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], 0, 0))
673 {
674 if (Create == FALSE)
675 {
676 return NULL;
677 }
678 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
679 if (!NT_SUCCESS(Status) || Pfn == 0)
680 {
681 KEBUGCHECK(0);
682 }
683 Entry = PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE;
684 if (Ke386GlobalPagesEnabled)
685 {
686 Entry |= PA_GLOBAL;
687 }
688 if(0 != InterlockedCompareExchangeUL(&MmGlobalKernelPageDirectory[PdeOffset], Entry, 0))
689 {
690 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
691 }
692 }
693 InterlockedExchangeUL(PageDir, MmGlobalKernelPageDirectory[PdeOffset]);
694 }
695 else
696 {
697 if (Create == FALSE)
698 {
699 return NULL;
700 }
701 Status = MmRequestPageMemoryConsumer(MC_NPPOOL, FALSE, &Pfn);
702 if (!NT_SUCCESS(Status) || Pfn == 0)
703 {
704 KEBUGCHECK(0);
705 }
706 Entry = InterlockedCompareExchangeUL(PageDir, PFN_TO_PTE(Pfn) | PA_PRESENT | PA_READWRITE | PA_USER, 0);
707 if (Entry != 0)
708 {
709 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
710 }
711 }
712 }
713 return (PULONG)ADDR_TO_PTE(Address);
714 }
715
716 BOOLEAN MmUnmapPageTable(PULONG Pt)
717 {
718 if (Ke386Pae)
719 {
720 if ((PULONGLONG)Pt >= (PULONGLONG)PAGETABLE_MAP && (PULONGLONG)Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512)
721 {
722 return TRUE;
723 }
724 }
725 else
726 {
727 if (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024)
728 {
729 return TRUE;
730 }
731 }
732 if (Pt)
733 {
734 MmDeleteHyperspaceMapping((PVOID)PAGE_ROUND_DOWN(Pt));
735 }
736 return FALSE;
737 }
738
739 static ULONGLONG MmGetPageEntryForProcessForPAE(PEPROCESS Process, PVOID Address)
740 {
741 ULONGLONG Pte;
742 PULONGLONG Pt;
743
744 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
745 if (Pt)
746 {
747 Pte = *Pt;
748 MmUnmapPageTable((PULONG)Pt);
749 return Pte;
750 }
751 return 0;
752 }
753
754 static ULONG MmGetPageEntryForProcess(PEPROCESS Process, PVOID Address)
755 {
756 ULONG Pte;
757 PULONG Pt;
758
759 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
760 if (Pt)
761 {
762 Pte = *Pt;
763 MmUnmapPageTable(Pt);
764 return Pte;
765 }
766 return 0;
767 }
768
769 PFN_TYPE
770 MmGetPfnForProcess(PEPROCESS Process,
771 PVOID Address)
772 {
773
774 if (Ke386Pae)
775 {
776 ULONGLONG Entry;
777 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
778 if (!(Entry & PA_PRESENT))
779 {
780 return 0;
781 }
782 return(PAE_PTE_TO_PFN(Entry));
783 }
784 else
785 {
786 ULONG Entry;
787 Entry = MmGetPageEntryForProcess(Process, Address);
788 if (!(Entry & PA_PRESENT))
789 {
790 return 0;
791 }
792 return(PTE_TO_PFN(Entry));
793 }
794 }
795
796 VOID
797 MmDisableVirtualMapping(PEPROCESS Process, PVOID Address, BOOL* WasDirty, PPFN_TYPE Page)
798 /*
799 * FUNCTION: Delete a virtual mapping
800 */
801 {
802 BOOLEAN WasValid;
803 if (Ke386Pae)
804 {
805 ULONGLONG Pte;
806 ULONGLONG tmpPte;
807 PULONGLONG Pt;
808
809 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
810 if (Pt == NULL)
811 {
812 KEBUGCHECK(0);
813 }
814 /*
815 * Atomically disable the present bit and get the old value.
816 */
817 do
818 {
819 Pte = *Pt;
820 tmpPte = Pte & ~PA_PRESENT;
821 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
822
823 MiFlushTlb((PULONG)Pt, Address);
824 WasValid = PAE_PAGE_MASK(Pte) != 0LL ? TRUE : FALSE;
825 if (!WasValid)
826 {
827 KEBUGCHECK(0);
828 }
829
830 /*
831 * Return some information to the caller
832 */
833 if (WasDirty != NULL)
834 {
835 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
836 }
837 if (Page != NULL)
838 {
839 *Page = PAE_PTE_TO_PFN(Pte);
840 }
841 }
842 else
843 {
844 ULONG Pte;
845 PULONG Pt;
846
847 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
848 if (Pt == NULL)
849 {
850 KEBUGCHECK(0);
851 }
852 /*
853 * Atomically disable the present bit and get the old value.
854 */
855 do
856 {
857 Pte = *Pt;
858 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_PRESENT, Pte));
859
860 MiFlushTlb(Pt, Address);
861 WasValid = (PAGE_MASK(Pte) != 0);
862 if (!WasValid)
863 {
864 KEBUGCHECK(0);
865 }
866
867 /*
868 * Return some information to the caller
869 */
870 if (WasDirty != NULL)
871 {
872 *WasDirty = Pte & PA_DIRTY;
873 }
874 if (Page != NULL)
875 {
876 *Page = PTE_TO_PFN(Pte);
877 }
878 }
879 }
880
881 VOID
882 MmRawDeleteVirtualMapping(PVOID Address)
883 {
884 if (Ke386Pae)
885 {
886 PULONGLONG Pt;
887 ULONGLONG ZeroPte = 0LL;
888 Pt = MmGetPageTableForProcessForPAE(NULL, Address, FALSE);
889 if (Pt)
890 {
891 /*
892 * Set the entry to zero
893 */
894 ExfpInterlockedExchange64UL(Pt, &ZeroPte);
895 MiFlushTlb((PULONG)Pt, Address);
896 }
897 }
898 else
899 {
900 PULONG Pt;
901
902 Pt = MmGetPageTableForProcess(NULL, Address, FALSE);
903 if (Pt && *Pt)
904 {
905 /*
906 * Set the entry to zero
907 */
908 InterlockedExchangeUL(Pt, 0);
909 MiFlushTlb(Pt, Address);
910 }
911 }
912 }
913
914 VOID
915 MmDeleteVirtualMapping(PEPROCESS Process, PVOID Address, BOOL FreePage,
916 BOOL* WasDirty, PPFN_TYPE Page)
917 /*
918 * FUNCTION: Delete a virtual mapping
919 */
920 {
921 BOOLEAN WasValid = FALSE;
922 PFN_TYPE Pfn;
923
924 DPRINT("MmDeleteVirtualMapping(%x, %x, %d, %x, %x)\n",
925 Process, Address, FreePage, WasDirty, Page);
926 if (Ke386Pae)
927 {
928 ULONGLONG Pte;
929 PULONGLONG Pt;
930
931 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
932 if (Pt == NULL)
933 {
934 if (WasDirty != NULL)
935 {
936 *WasDirty = FALSE;
937 }
938 if (Page != NULL)
939 {
940 *Page = 0;
941 }
942 return;
943 }
944
945 /*
946 * Atomically set the entry to zero and get the old value.
947 */
948 Pte = 0LL;
949 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
950
951 MiFlushTlb((PULONG)Pt, Address);
952
953 WasValid = PAE_PAGE_MASK(Pte) != 0 ? TRUE : FALSE;
954 if (WasValid)
955 {
956 Pfn = PAE_PTE_TO_PFN(Pte);
957 MmMarkPageUnmapped(Pfn);
958 }
959 else
960 {
961 Pfn = 0;
962 }
963
964 if (FreePage && WasValid)
965 {
966 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
967 }
968
969 /*
970 * Return some information to the caller
971 */
972 if (WasDirty != NULL)
973 {
974 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
975 }
976 if (Page != NULL)
977 {
978 *Page = Pfn;
979 }
980 }
981 else
982 {
983 ULONG Pte;
984 PULONG Pt;
985
986 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
987
988 if (Pt == NULL)
989 {
990 if (WasDirty != NULL)
991 {
992 *WasDirty = FALSE;
993 }
994 if (Page != NULL)
995 {
996 *Page = 0;
997 }
998 return;
999 }
1000
1001 /*
1002 * Atomically set the entry to zero and get the old value.
1003 */
1004 Pte = InterlockedExchangeUL(Pt, 0);
1005
1006 MiFlushTlb(Pt, Address);
1007
1008 WasValid = (PAGE_MASK(Pte) != 0);
1009 if (WasValid)
1010 {
1011 Pfn = PTE_TO_PFN(Pte);
1012 MmMarkPageUnmapped(Pfn);
1013 }
1014 else
1015 {
1016 Pfn = 0;
1017 }
1018
1019 if (FreePage && WasValid)
1020 {
1021 MmReleasePageMemoryConsumer(MC_NPPOOL, Pfn);
1022 }
1023
1024 /*
1025 * Return some information to the caller
1026 */
1027 if (WasDirty != NULL)
1028 {
1029 *WasDirty = Pte & PA_DIRTY ? TRUE : FALSE;
1030 }
1031 if (Page != NULL)
1032 {
1033 *Page = Pfn;
1034 }
1035 }
1036 /*
1037 * Decrement the reference count for this page table.
1038 */
1039 if (Process != NULL && WasValid &&
1040 Process->AddressSpace.PageTableRefCountTable != NULL &&
1041 Address < (PVOID)KERNEL_BASE)
1042 {
1043 PUSHORT Ptrc;
1044 ULONG Idx;
1045
1046 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1047 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1048
1049 Ptrc[Idx]--;
1050 if (Ptrc[Idx] == 0)
1051 {
1052 MmFreePageTable(Process, Address);
1053 }
1054 }
1055 }
1056
1057 VOID
1058 MmDeletePageFileMapping(PEPROCESS Process, PVOID Address,
1059 SWAPENTRY* SwapEntry)
1060 /*
1061 * FUNCTION: Delete a virtual mapping
1062 */
1063 {
1064 if (Ke386Pae)
1065 {
1066 ULONGLONG Pte;
1067 PULONGLONG Pt;
1068
1069 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1070 if (Pt == NULL)
1071 {
1072 *SwapEntry = 0;
1073 return;
1074 }
1075
1076 /*
1077 * Atomically set the entry to zero and get the old value.
1078 */
1079 Pte = 0LL;
1080 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1081
1082 MiFlushTlb((PULONG)Pt, Address);
1083
1084 /*
1085 * Decrement the reference count for this page table.
1086 */
1087 if (Process != NULL && Pte &&
1088 Process->AddressSpace.PageTableRefCountTable != NULL &&
1089 Address < (PVOID)KERNEL_BASE)
1090 {
1091 PUSHORT Ptrc;
1092
1093 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1094
1095 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)]--;
1096 if (Ptrc[PAE_ADDR_TO_PAGE_TABLE(Address)] == 0)
1097 {
1098 MmFreePageTable(Process, Address);
1099 }
1100 }
1101
1102
1103 /*
1104 * Return some information to the caller
1105 */
1106 *SwapEntry = Pte >> 1;
1107 }
1108 else
1109 {
1110 ULONG Pte;
1111 PULONG Pt;
1112
1113 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1114
1115 if (Pt == NULL)
1116 {
1117 *SwapEntry = 0;
1118 return;
1119 }
1120
1121 /*
1122 * Atomically set the entry to zero and get the old value.
1123 */
1124 Pte = InterlockedExchangeUL(Pt, 0);
1125
1126 MiFlushTlb(Pt, Address);
1127
1128 /*
1129 * Decrement the reference count for this page table.
1130 */
1131 if (Process != NULL && Pte &&
1132 Process->AddressSpace.PageTableRefCountTable != NULL &&
1133 Address < (PVOID)KERNEL_BASE)
1134 {
1135 PUSHORT Ptrc;
1136
1137 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1138
1139 Ptrc[ADDR_TO_PAGE_TABLE(Address)]--;
1140 if (Ptrc[ADDR_TO_PAGE_TABLE(Address)] == 0)
1141 {
1142 MmFreePageTable(Process, Address);
1143 }
1144 }
1145
1146
1147 /*
1148 * Return some information to the caller
1149 */
1150 *SwapEntry = Pte >> 1;
1151 }
1152 }
1153
1154 BOOLEAN
1155 Mmi386MakeKernelPageTableGlobal(PVOID PAddress)
1156 {
1157 if (Ke386Pae)
1158 {
1159 PULONGLONG Pt;
1160 PULONGLONG Pde;
1161 Pde = PAE_ADDR_TO_PDE(PAddress);
1162 if (*Pde == 0LL)
1163 {
1164 Pt = MmGetPageTableForProcessForPAE(NULL, PAddress, FALSE);
1165 #if 0
1166 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1167 FLASH_TLB_ONE(PAddress);
1168 #endif
1169 if (Pt != NULL)
1170 {
1171 return TRUE;
1172 }
1173 }
1174 }
1175 else
1176 {
1177 PULONG Pt, Pde;
1178 Pde = ADDR_TO_PDE(PAddress);
1179 if (*Pde == 0)
1180 {
1181 Pt = MmGetPageTableForProcess(NULL, PAddress, FALSE);
1182 #if 0
1183 /* Non existing mappings are not cached within the tlb. We must not invalidate this entry */
1184 FLASH_TLB_ONE(PAddress);
1185 #endif
1186 if (Pt != NULL)
1187 {
1188 return TRUE;
1189 }
1190 }
1191 }
1192 return(FALSE);
1193 }
1194
1195 BOOLEAN MmIsDirtyPage(PEPROCESS Process, PVOID Address)
1196 {
1197 if (Ke386Pae)
1198 {
1199 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1200 }
1201 else
1202 {
1203 return MmGetPageEntryForProcess(Process, Address) & PA_DIRTY ? TRUE : FALSE;
1204 }
1205 }
1206
1207 BOOLEAN
1208 MmIsAccessedAndResetAccessPage(PEPROCESS Process, PVOID Address)
1209 {
1210 if (Address < (PVOID)KERNEL_BASE && Process == NULL)
1211 {
1212 DPRINT1("MmIsAccessedAndResetAccessPage is called for user space without a process.\n");
1213 KEBUGCHECK(0);
1214 }
1215 if (Ke386Pae)
1216 {
1217 PULONGLONG Pt;
1218 ULONGLONG Pte;
1219 ULONGLONG tmpPte;
1220
1221 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1222 if (Pt == NULL)
1223 {
1224 KEBUGCHECK(0);
1225 }
1226
1227 do
1228 {
1229 Pte = *Pt;
1230 tmpPte = Pte & ~PA_ACCESSED;
1231 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1232
1233 if (Pte & PA_ACCESSED)
1234 {
1235 MiFlushTlb((PULONG)Pt, Address);
1236 return TRUE;
1237 }
1238 else
1239 {
1240 MmUnmapPageTable((PULONG)Pt);
1241 return FALSE;
1242 }
1243 }
1244 else
1245 {
1246 PULONG Pt;
1247 ULONG Pte;
1248
1249 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1250 if (Pt == NULL)
1251 {
1252 KEBUGCHECK(0);
1253 }
1254
1255 do
1256 {
1257 Pte = *Pt;
1258 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_ACCESSED, Pte));
1259
1260 if (Pte & PA_ACCESSED)
1261 {
1262 MiFlushTlb(Pt, Address);
1263 return TRUE;
1264 }
1265 else
1266 {
1267 MmUnmapPageTable(Pt);
1268 return FALSE;
1269 }
1270 }
1271 }
1272
1273 VOID MmSetCleanPage(PEPROCESS Process, PVOID Address)
1274 {
1275 if (Address < (PVOID)KERNEL_BASE && Process == NULL)
1276 {
1277 DPRINT1("MmSetCleanPage is called for user space without a process.\n");
1278 KEBUGCHECK(0);
1279 }
1280 if (Ke386Pae)
1281 {
1282 PULONGLONG Pt;
1283 ULONGLONG Pte;
1284 ULONGLONG tmpPte;
1285
1286 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1287
1288 if (Pt == NULL)
1289 {
1290 KEBUGCHECK(0);
1291 }
1292
1293 do
1294 {
1295 Pte = *Pt;
1296 tmpPte = Pte & ~PA_DIRTY;
1297 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1298
1299 if (Pte & PA_DIRTY)
1300 {
1301 MiFlushTlb((PULONG)Pt, Address);
1302 }
1303 else
1304 {
1305 MmUnmapPageTable((PULONG)Pt);
1306 }
1307 }
1308 else
1309 {
1310 PULONG Pt;
1311 ULONG Pte;
1312
1313 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1314
1315 if (Pt == NULL)
1316 {
1317 KEBUGCHECK(0);
1318 }
1319
1320 do
1321 {
1322 Pte = *Pt;
1323 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte & ~PA_DIRTY, Pte));
1324
1325 if (Pte & PA_DIRTY)
1326 {
1327 MiFlushTlb(Pt, Address);
1328 }
1329 else
1330 {
1331 MmUnmapPageTable(Pt);
1332 }
1333 }
1334 }
1335
1336 VOID MmSetDirtyPage(PEPROCESS Process, PVOID Address)
1337 {
1338 if (Address < (PVOID)KERNEL_BASE && Process == NULL)
1339 {
1340 DPRINT1("MmSetDirtyPage is called for user space without a process.\n");
1341 KEBUGCHECK(0);
1342 }
1343 if (Ke386Pae)
1344 {
1345 PULONGLONG Pt;
1346 ULONGLONG Pte;
1347 ULONGLONG tmpPte;
1348
1349 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1350 if (Pt == NULL)
1351 {
1352 KEBUGCHECK(0);
1353 }
1354
1355 do
1356 {
1357 Pte = *Pt;
1358 tmpPte = Pte | PA_DIRTY;
1359 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1360 if (!(Pte & PA_DIRTY))
1361 {
1362 MiFlushTlb((PULONG)Pt, Address);
1363 }
1364 else
1365 {
1366 MmUnmapPageTable((PULONG)Pt);
1367 }
1368 }
1369 else
1370 {
1371 PULONG Pt;
1372 ULONG Pte;
1373
1374 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1375 if (Pt == NULL)
1376 {
1377 KEBUGCHECK(0);
1378 }
1379
1380 do
1381 {
1382 Pte = *Pt;
1383 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_DIRTY, Pte));
1384 if (!(Pte & PA_DIRTY))
1385 {
1386 MiFlushTlb(Pt, Address);
1387 }
1388 else
1389 {
1390 MmUnmapPageTable(Pt);
1391 }
1392 }
1393 }
1394
1395 VOID MmEnableVirtualMapping(PEPROCESS Process, PVOID Address)
1396 {
1397 if (Ke386Pae)
1398 {
1399 PULONGLONG Pt;
1400 ULONGLONG Pte;
1401 ULONGLONG tmpPte;
1402
1403 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1404 if (Pt == NULL)
1405 {
1406 KEBUGCHECK(0);
1407 }
1408
1409 do
1410 {
1411 Pte = *Pt;
1412 tmpPte = Pte | PA_PRESENT;
1413 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
1414 if (!(Pte & PA_PRESENT))
1415 {
1416 MiFlushTlb((PULONG)Pt, Address);
1417 }
1418 else
1419 {
1420 MmUnmapPageTable((PULONG)Pt);
1421 }
1422 }
1423 else
1424 {
1425 PULONG Pt;
1426 ULONG Pte;
1427
1428 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
1429 if (Pt == NULL)
1430 {
1431 KEBUGCHECK(0);
1432 }
1433
1434 do
1435 {
1436 Pte = *Pt;
1437 } while (Pte != InterlockedCompareExchangeUL(Pt, Pte | PA_PRESENT, Pte));
1438 if (!(Pte & PA_PRESENT))
1439 {
1440 MiFlushTlb(Pt, Address);
1441 }
1442 else
1443 {
1444 MmUnmapPageTable(Pt);
1445 }
1446 }
1447 }
1448
1449 BOOLEAN MmIsPagePresent(PEPROCESS Process, PVOID Address)
1450 {
1451 if (Ke386Pae)
1452 {
1453 return MmGetPageEntryForProcessForPAE(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1454 }
1455 else
1456 {
1457 return MmGetPageEntryForProcess(Process, Address) & PA_PRESENT ? TRUE : FALSE;
1458 }
1459 }
1460
1461 BOOLEAN MmIsPageSwapEntry(PEPROCESS Process, PVOID Address)
1462 {
1463 if (Ke386Pae)
1464 {
1465 ULONGLONG Entry;
1466 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1467 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1468 }
1469 else
1470 {
1471 ULONG Entry;
1472 Entry = MmGetPageEntryForProcess(Process, Address);
1473 return !(Entry & PA_PRESENT) && Entry != 0 ? TRUE : FALSE;
1474 }
1475 }
1476
1477 NTSTATUS
1478 MmCreateVirtualMappingForKernel(PVOID Address,
1479 ULONG flProtect,
1480 PPFN_TYPE Pages,
1481 ULONG PageCount)
1482 {
1483 ULONG Attributes;
1484 ULONG i;
1485 PVOID Addr;
1486 ULONG PdeOffset, oldPdeOffset;
1487 BOOLEAN NoExecute = FALSE;
1488
1489 DPRINT("MmCreateVirtualMappingForKernel(%x, %x, %x, %d)\n",
1490 Address, flProtect, Pages, PageCount);
1491
1492 if (Address < (PVOID)KERNEL_BASE)
1493 {
1494 DPRINT1("MmCreateVirtualMappingForKernel is called for user space\n");
1495 KEBUGCHECK(0);
1496 }
1497
1498 Attributes = ProtectToPTE(flProtect);
1499 if (Attributes & 0x80000000)
1500 {
1501 NoExecute = TRUE;
1502 }
1503 Attributes &= 0xfff;
1504 if (Ke386GlobalPagesEnabled)
1505 {
1506 Attributes |= PA_GLOBAL;
1507 }
1508
1509 Addr = Address;
1510
1511 if (Ke386Pae)
1512 {
1513 PULONGLONG Pt = NULL;
1514 ULONGLONG Pte;
1515
1516 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1517 for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1518 {
1519 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1520 {
1521 DPRINT1("Setting physical address but not allowing access at address "
1522 "0x%.8X with attributes %x/%x.\n",
1523 Addr, Attributes, flProtect);
1524 KEBUGCHECK(0);
1525 }
1526
1527 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1528 if (oldPdeOffset != PdeOffset)
1529 {
1530 Pt = MmGetPageTableForProcessForPAE(NULL, Addr, TRUE);
1531 if (Pt == NULL)
1532 {
1533 KEBUGCHECK(0);
1534 }
1535 }
1536 else
1537 {
1538 Pt++;
1539 }
1540 oldPdeOffset = PdeOffset;
1541
1542 Pte = PFN_TO_PTE(Pages[i]) | Attributes;
1543 if (NoExecute)
1544 {
1545 Pte |= 0x8000000000000000LL;
1546 }
1547 Pte = ExfpInterlockedExchange64UL(Pt, &Pte);
1548 if (Pte != 0LL)
1549 {
1550 KEBUGCHECK(0);
1551 }
1552 }
1553 }
1554 else
1555 {
1556 PULONG Pt;
1557 ULONG Pte;
1558
1559 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1560 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1561 if (Pt == NULL)
1562 {
1563 KEBUGCHECK(0);
1564 }
1565 Pt--;
1566
1567 for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1568 {
1569 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1570 {
1571 DPRINT1("Setting physical address but not allowing access at address "
1572 "0x%.8X with attributes %x/%x.\n",
1573 Addr, Attributes, flProtect);
1574 KEBUGCHECK(0);
1575 }
1576
1577 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1578 if (oldPdeOffset != PdeOffset)
1579 {
1580 Pt = MmGetPageTableForProcess(NULL, Addr, TRUE);
1581 if (Pt == NULL)
1582 {
1583 KEBUGCHECK(0);
1584 }
1585 }
1586 else
1587 {
1588 Pt++;
1589 }
1590 oldPdeOffset = PdeOffset;
1591
1592 Pte = *Pt;
1593 if (Pte != 0)
1594 {
1595 KEBUGCHECK(0);
1596 }
1597 InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1598 }
1599 }
1600
1601 return(STATUS_SUCCESS);
1602 }
1603
1604 NTSTATUS
1605 MmCreatePageFileMapping(PEPROCESS Process,
1606 PVOID Address,
1607 SWAPENTRY SwapEntry)
1608 {
1609 if (Process == NULL && Address < (PVOID)KERNEL_BASE)
1610 {
1611 DPRINT1("No process\n");
1612 KEBUGCHECK(0);
1613 }
1614 if (Process != NULL && Address >= (PVOID)KERNEL_BASE)
1615 {
1616 DPRINT1("Setting kernel address with process context\n");
1617 KEBUGCHECK(0);
1618 }
1619 if (SwapEntry & (1 << 31))
1620 {
1621 KEBUGCHECK(0);
1622 }
1623
1624 if (Ke386Pae)
1625 {
1626 PULONGLONG Pt;
1627 ULONGLONG Pte;
1628 ULONGLONG tmpPte;
1629
1630 Pt = MmGetPageTableForProcessForPAE(Process, Address, TRUE);
1631 if (Pt == NULL)
1632 {
1633 KEBUGCHECK(0);
1634 }
1635 tmpPte = SwapEntry << 1;
1636 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1637 if (PAE_PAGE_MASK((Pte)) != 0)
1638 {
1639 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1640 }
1641
1642 if (Pte != 0)
1643 {
1644 MiFlushTlb((PULONG)Pt, Address);
1645 }
1646 else
1647 {
1648 MmUnmapPageTable((PULONG)Pt);
1649 }
1650 }
1651 else
1652 {
1653 PULONG Pt;
1654 ULONG Pte;
1655
1656 Pt = MmGetPageTableForProcess(Process, Address, TRUE);
1657 if (Pt == NULL)
1658 {
1659 KEBUGCHECK(0);
1660 }
1661 Pte = *Pt;
1662 if (PAGE_MASK((Pte)) != 0)
1663 {
1664 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1665 }
1666 InterlockedExchangeUL(Pt, SwapEntry << 1);
1667 if (Pte != 0)
1668 {
1669 MiFlushTlb(Pt, Address);
1670 }
1671 else
1672 {
1673 MmUnmapPageTable(Pt);
1674 }
1675 }
1676 if (Process != NULL &&
1677 Process->AddressSpace.PageTableRefCountTable != NULL &&
1678 Address < (PVOID)KERNEL_BASE)
1679 {
1680 PUSHORT Ptrc;
1681 ULONG Idx;
1682
1683 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1684 Idx = Ke386Pae ? PAE_ADDR_TO_PAGE_TABLE(Address) : ADDR_TO_PAGE_TABLE(Address);
1685 Ptrc[Idx]++;
1686 }
1687 return(STATUS_SUCCESS);
1688 }
1689
1690
1691 NTSTATUS
1692 MmCreateVirtualMappingUnsafe(PEPROCESS Process,
1693 PVOID Address,
1694 ULONG flProtect,
1695 PPFN_TYPE Pages,
1696 ULONG PageCount)
1697 {
1698 ULONG Attributes;
1699 PVOID Addr;
1700 ULONG i;
1701 ULONG oldPdeOffset, PdeOffset;
1702 BOOLEAN NoExecute = FALSE;
1703
1704 DPRINT("MmCreateVirtualMappingUnsafe(%x, %x, %x, %x (%x), %d)\n",
1705 Process, Address, flProtect, Pages, *Pages, PageCount);
1706
1707 if (Process == NULL)
1708 {
1709 if (Address < (PVOID)KERNEL_BASE)
1710 {
1711 DPRINT1("No process\n");
1712 KEBUGCHECK(0);
1713 }
1714 if (PageCount > 0x10000 ||
1715 (ULONG_PTR) Address / PAGE_SIZE + PageCount > 0x100000)
1716 {
1717 DPRINT1("Page count to large\n");
1718 KEBUGCHECK(0);
1719 }
1720 }
1721 else
1722 {
1723 if (Address >= (PVOID)KERNEL_BASE)
1724 {
1725 DPRINT1("Setting kernel address with process context\n");
1726 KEBUGCHECK(0);
1727 }
1728 if (PageCount > KERNEL_BASE / PAGE_SIZE ||
1729 (ULONG_PTR) Address / PAGE_SIZE + PageCount > KERNEL_BASE / PAGE_SIZE)
1730 {
1731 DPRINT1("Page Count to large\n");
1732 KEBUGCHECK(0);
1733 }
1734 }
1735
1736 Attributes = ProtectToPTE(flProtect);
1737 if (Attributes & 0x80000000)
1738 {
1739 NoExecute = TRUE;
1740 }
1741 Attributes &= 0xfff;
1742 if (Address >= (PVOID)KERNEL_BASE)
1743 {
1744 Attributes &= ~PA_USER;
1745 if (Ke386GlobalPagesEnabled)
1746 {
1747 Attributes |= PA_GLOBAL;
1748 }
1749 }
1750 else
1751 {
1752 Attributes |= PA_USER;
1753 }
1754
1755 Addr = Address;
1756
1757 if (Ke386Pae)
1758 {
1759 ULONGLONG Pte, tmpPte;
1760 PULONGLONG Pt = NULL;
1761
1762 oldPdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr) + 1;
1763 for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1764 {
1765 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1766 {
1767 DPRINT1("Setting physical address but not allowing access at address "
1768 "0x%.8X with attributes %x/%x.\n",
1769 Addr, Attributes, flProtect);
1770 KEBUGCHECK(0);
1771 }
1772 PdeOffset = PAE_ADDR_TO_PDE_OFFSET(Addr);
1773 if (oldPdeOffset != PdeOffset)
1774 {
1775 MmUnmapPageTable((PULONG)Pt);
1776 Pt = MmGetPageTableForProcessForPAE(Process, Addr, TRUE);
1777 if (Pt == NULL)
1778 {
1779 KEBUGCHECK(0);
1780 }
1781 }
1782 else
1783 {
1784 Pt++;
1785 }
1786 oldPdeOffset = PdeOffset;
1787
1788 MmMarkPageMapped(Pages[i]);
1789 tmpPte = PAE_PFN_TO_PTE(Pages[i]) | Attributes;
1790 if (NoExecute)
1791 {
1792 tmpPte |= 0x8000000000000000LL;
1793 }
1794 Pte = ExfpInterlockedExchange64UL(Pt, &tmpPte);
1795 if (PAE_PAGE_MASK((Pte)) != 0LL && !((Pte) & PA_PRESENT))
1796 {
1797 KEBUGCHECK(0);
1798 }
1799 if (PAE_PAGE_MASK((Pte)) != 0LL)
1800 {
1801 MmMarkPageUnmapped(PAE_PTE_TO_PFN((Pte)));
1802 }
1803 if (Address < (PVOID)KERNEL_BASE &&
1804 Process->AddressSpace.PageTableRefCountTable != NULL &&
1805 Attributes & PA_PRESENT)
1806 {
1807 PUSHORT Ptrc;
1808
1809 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1810
1811 Ptrc[PAE_ADDR_TO_PAGE_TABLE(Addr)]++;
1812 }
1813 if (Pte != 0LL)
1814 {
1815 if (Address > (PVOID)KERNEL_BASE ||
1816 (Pt >= (PULONGLONG)PAGETABLE_MAP && Pt < (PULONGLONG)PAGETABLE_MAP + 4*512*512))
1817 {
1818 MiFlushTlb((PULONG)Pt, Address);
1819 }
1820 }
1821 }
1822 if (Addr > Address)
1823 {
1824 MmUnmapPageTable((PULONG)Pt);
1825 }
1826 }
1827 else
1828 {
1829 PULONG Pt = NULL;
1830 ULONG Pte;
1831 oldPdeOffset = ADDR_TO_PDE_OFFSET(Addr) + 1;
1832 for (i = 0; i < PageCount; i++, Addr += PAGE_SIZE)
1833 {
1834 if (!(Attributes & PA_PRESENT) && Pages[i] != 0)
1835 {
1836 DPRINT1("Setting physical address but not allowing access at address "
1837 "0x%.8X with attributes %x/%x.\n",
1838 Addr, Attributes, flProtect);
1839 KEBUGCHECK(0);
1840 }
1841 PdeOffset = ADDR_TO_PDE_OFFSET(Addr);
1842 if (oldPdeOffset != PdeOffset)
1843 {
1844 MmUnmapPageTable(Pt);
1845 Pt = MmGetPageTableForProcess(Process, Addr, TRUE);
1846 if (Pt == NULL)
1847 {
1848 KEBUGCHECK(0);
1849 }
1850 }
1851 else
1852 {
1853 Pt++;
1854 }
1855 oldPdeOffset = PdeOffset;
1856
1857 Pte = *Pt;
1858 MmMarkPageMapped(Pages[i]);
1859 if (PAGE_MASK((Pte)) != 0 && !((Pte) & PA_PRESENT))
1860 {
1861 KEBUGCHECK(0);
1862 }
1863 if (PAGE_MASK((Pte)) != 0)
1864 {
1865 MmMarkPageUnmapped(PTE_TO_PFN((Pte)));
1866 }
1867 InterlockedExchangeUL(Pt, PFN_TO_PTE(Pages[i]) | Attributes);
1868 if (Address < (PVOID)KERNEL_BASE &&
1869 Process->AddressSpace.PageTableRefCountTable != NULL &&
1870 Attributes & PA_PRESENT)
1871 {
1872 PUSHORT Ptrc;
1873
1874 Ptrc = Process->AddressSpace.PageTableRefCountTable;
1875
1876 Ptrc[ADDR_TO_PAGE_TABLE(Addr)]++;
1877 }
1878 if (Pte != 0)
1879 {
1880 if (Address > (PVOID)KERNEL_BASE ||
1881 (Pt >= (PULONG)PAGETABLE_MAP && Pt < (PULONG)PAGETABLE_MAP + 1024*1024))
1882 {
1883 MiFlushTlb(Pt, Address);
1884 }
1885 }
1886 }
1887 if (Addr > Address)
1888 {
1889 MmUnmapPageTable(Pt);
1890 }
1891 }
1892 return(STATUS_SUCCESS);
1893 }
1894
1895 NTSTATUS
1896 MmCreateVirtualMapping(PEPROCESS Process,
1897 PVOID Address,
1898 ULONG flProtect,
1899 PPFN_TYPE Pages,
1900 ULONG PageCount)
1901 {
1902 ULONG i;
1903
1904 for (i = 0; i < PageCount; i++)
1905 {
1906 if (!MmIsUsablePage(Pages[i]))
1907 {
1908 DPRINT1("Page at address %x not usable\n", PFN_TO_PTE(Pages[i]));
1909 KEBUGCHECK(0);
1910 }
1911 }
1912
1913 return(MmCreateVirtualMappingUnsafe(Process,
1914 Address,
1915 flProtect,
1916 Pages,
1917 PageCount));
1918 }
1919
1920 ULONG
1921 MmGetPageProtect(PEPROCESS Process, PVOID Address)
1922 {
1923 ULONG Entry;
1924 ULONG Protect;
1925 if (Ke386Pae)
1926 {
1927 Entry = MmGetPageEntryForProcessForPAE(Process, Address);
1928 }
1929 else
1930 {
1931 Entry = MmGetPageEntryForProcess(Process, Address);
1932 }
1933
1934 if (!(Entry & PA_PRESENT))
1935 {
1936 Protect = PAGE_NOACCESS;
1937 }
1938 else
1939 {
1940 if (Entry & PA_READWRITE)
1941 {
1942 Protect = PAGE_READWRITE;
1943 }
1944 else
1945 {
1946 Protect = PAGE_EXECUTE_READ;
1947 }
1948 if (Entry & PA_CD)
1949 {
1950 Protect |= PAGE_NOCACHE;
1951 }
1952 if (Entry & PA_WT)
1953 {
1954 Protect |= PAGE_WRITETHROUGH;
1955 }
1956 if (!(Entry & PA_USER))
1957 {
1958 Protect |= PAGE_SYSTEM;
1959 }
1960
1961 }
1962 return(Protect);
1963 }
1964
1965 VOID
1966 MmSetPageProtect(PEPROCESS Process, PVOID Address, ULONG flProtect)
1967 {
1968 ULONG Attributes = 0;
1969 BOOLEAN NoExecute = FALSE;
1970
1971 DPRINT("MmSetPageProtect(Process %x Address %x flProtect %x)\n",
1972 Process, Address, flProtect);
1973
1974 Attributes = ProtectToPTE(flProtect);
1975 if (Attributes & 0x80000000)
1976 {
1977 NoExecute = TRUE;
1978 }
1979 Attributes &= 0xfff;
1980 if (Address >= (PVOID)KERNEL_BASE)
1981 {
1982 Attributes &= ~PA_USER;
1983 if (Ke386GlobalPagesEnabled)
1984 {
1985 Attributes |= PA_GLOBAL;
1986 }
1987 }
1988 else
1989 {
1990 Attributes |= PA_USER;
1991 }
1992 if (Ke386Pae)
1993 {
1994 PULONGLONG Pt;
1995 ULONGLONG tmpPte, Pte;
1996
1997 Pt = MmGetPageTableForProcessForPAE(Process, Address, FALSE);
1998 if (Pt == NULL)
1999 {
2000 DPRINT1("Address %x\n", Address);
2001 KEBUGCHECK(0);
2002 }
2003 do
2004 {
2005 Pte = *Pt;
2006 tmpPte = PAE_PAGE_MASK(Pte) | Attributes | (Pte & (PA_ACCESSED|PA_DIRTY));
2007 if (NoExecute)
2008 {
2009 tmpPte |= 0x8000000000000000LL;
2010 }
2011 else
2012 {
2013 tmpPte &= ~0x8000000000000000LL;
2014 }
2015 } while (Pte != ExfInterlockedCompareExchange64UL(Pt, &tmpPte, &Pte));
2016
2017 MiFlushTlb((PULONG)Pt, Address);
2018 }
2019 else
2020 {
2021 PULONG Pt;
2022
2023 Pt = MmGetPageTableForProcess(Process, Address, FALSE);
2024 if (Pt == NULL)
2025 {
2026 KEBUGCHECK(0);
2027 }
2028 InterlockedExchange(Pt, PAGE_MASK(*Pt) | Attributes | (*Pt & (PA_ACCESSED|PA_DIRTY)));
2029 MiFlushTlb(Pt, Address);
2030 }
2031 }
2032
2033 /*
2034 * @implemented
2035 */
2036 PHYSICAL_ADDRESS STDCALL
2037 MmGetPhysicalAddress(PVOID vaddr)
2038 /*
2039 * FUNCTION: Returns the physical address corresponding to a virtual address
2040 */
2041 {
2042 PHYSICAL_ADDRESS p;
2043
2044 DPRINT("MmGetPhysicalAddress(vaddr %x)\n", vaddr);
2045 if (Ke386Pae)
2046 {
2047 ULONGLONG Pte;
2048 Pte = MmGetPageEntryForProcessForPAE(NULL, vaddr);
2049 if (Pte != 0 && Pte & PA_PRESENT)
2050 {
2051 p.QuadPart = PAE_PAGE_MASK(Pte);
2052 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2053 }
2054 else
2055 {
2056 p.QuadPart = 0;
2057 }
2058 }
2059 else
2060 {
2061 ULONG Pte;
2062 Pte = MmGetPageEntryForProcess(NULL, vaddr);
2063 if (Pte != 0 && Pte & PA_PRESENT)
2064 {
2065 p.QuadPart = PAGE_MASK(Pte);
2066 p.u.LowPart |= (ULONG_PTR)vaddr & (PAGE_SIZE - 1);
2067 }
2068 else
2069 {
2070 p.QuadPart = 0;
2071 }
2072 }
2073 return p;
2074 }
2075
2076 PVOID
2077 MmCreateHyperspaceMapping(PFN_TYPE Page)
2078 {
2079 PVOID Address;
2080 LONG i;
2081
2082 if (Ke386Pae)
2083 {
2084 ULONGLONG Entry;
2085 ULONGLONG ZeroEntry = 0LL;
2086 PULONGLONG Pte;
2087
2088 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2089 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2090
2091 if (Page & 1024)
2092 {
2093 for (i = Page %1024; i < 1024; i++, Pte++)
2094 {
2095 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2096 {
2097 break;
2098 }
2099 }
2100 if (i >= 1024)
2101 {
2102 Pte = PAE_ADDR_TO_PTE(HYPERSPACE);
2103 for (i = 0; i < Page % 1024; i++, Pte++)
2104 {
2105 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2106 {
2107 break;
2108 }
2109 }
2110 if (i >= Page % 1024)
2111 {
2112 KEBUGCHECK(0);
2113 }
2114 }
2115 }
2116 else
2117 {
2118 for (i = Page %1024; i >= 0; i--, Pte--)
2119 {
2120 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2121 {
2122 break;
2123 }
2124 }
2125 if (i < 0)
2126 {
2127 Pte = PAE_ADDR_TO_PTE(HYPERSPACE) + 1023;
2128 for (i = 1023; i > Page % 1024; i--, Pte--)
2129 {
2130 if (0LL == ExfInterlockedCompareExchange64UL(Pte, &Entry, &ZeroEntry))
2131 {
2132 break;
2133 }
2134 }
2135 if (i <= Page % 1024)
2136 {
2137 KEBUGCHECK(0);
2138 }
2139 }
2140 }
2141 }
2142 else
2143 {
2144 ULONG Entry;
2145 PULONG Pte;
2146 Entry = PFN_TO_PTE(Page) | PA_PRESENT | PA_READWRITE;
2147 Pte = ADDR_TO_PTE(HYPERSPACE) + Page % 1024;
2148 if (Page & 1024)
2149 {
2150 for (i = Page % 1024; i < 1024; i++, Pte++)
2151 {
2152 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2153 {
2154 break;
2155 }
2156 }
2157 if (i >= 1024)
2158 {
2159 Pte = ADDR_TO_PTE(HYPERSPACE);
2160 for (i = 0; i < Page % 1024; i++, Pte++)
2161 {
2162 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2163 {
2164 break;
2165 }
2166 }
2167 if (i >= Page % 1024)
2168 {
2169 KEBUGCHECK(0);
2170 }
2171 }
2172 }
2173 else
2174 {
2175 for (i = Page % 1024; i >= 0; i--, Pte--)
2176 {
2177 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2178 {
2179 break;
2180 }
2181 }
2182 if (i < 0)
2183 {
2184 Pte = ADDR_TO_PTE(HYPERSPACE) + 1023;
2185 for (i = 1023; i > Page % 1024; i--, Pte--)
2186 {
2187 if (0 == InterlockedCompareExchange((PLONG)Pte, (LONG)Entry, 0))
2188 {
2189 break;
2190 }
2191 }
2192 if (i <= Page % 1024)
2193 {
2194 KEBUGCHECK(0);
2195 }
2196 }
2197 }
2198 }
2199 Address = (PVOID)HYPERSPACE + i * PAGE_SIZE;
2200 FLUSH_TLB_ONE(Address);
2201 return Address;
2202 }
2203
2204 PFN_TYPE
2205 MmChangeHyperspaceMapping(PVOID Address, PFN_TYPE NewPage)
2206 {
2207 PFN_TYPE Pfn;
2208 ASSERT (IS_HYPERSPACE(Address));
2209 if (Ke386Pae)
2210 {
2211 ULONGLONG Entry = PAE_PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE;
2212 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2213 Pfn = PAE_PTE_TO_PFN(Entry);
2214 }
2215 else
2216 {
2217 ULONG Entry;
2218 Entry = InterlockedExchange(ADDR_TO_PTE(Address), PFN_TO_PTE(NewPage) | PA_PRESENT | PA_READWRITE);
2219 Pfn = PTE_TO_PFN(Entry);
2220 }
2221 FLUSH_TLB_ONE(Address);
2222 return Pfn;
2223 }
2224
2225 PFN_TYPE
2226 MmDeleteHyperspaceMapping(PVOID Address)
2227 {
2228 PFN_TYPE Pfn;
2229 ASSERT (IS_HYPERSPACE(Address));
2230 if (Ke386Pae)
2231 {
2232 ULONGLONG Entry = 0LL;
2233 Entry = (ULONG)ExfpInterlockedExchange64UL(PAE_ADDR_TO_PTE(Address), &Entry);
2234 Pfn = PAE_PTE_TO_PFN(Entry);
2235 }
2236 else
2237 {
2238 ULONG Entry;
2239 Entry = InterlockedExchange(ADDR_TO_PTE(Address), 0);
2240 Pfn = PTE_TO_PFN(Entry);
2241 }
2242 FLUSH_TLB_ONE(Address);
2243 return Pfn;
2244 }
2245
2246 VOID MmUpdatePageDir(PEPROCESS Process, PVOID Address, ULONG Size)
2247 {
2248 ULONG StartOffset, EndOffset, Offset;
2249
2250 if (Address < (PVOID)KERNEL_BASE)
2251 {
2252 KEBUGCHECK(0);
2253 }
2254 if (Ke386Pae)
2255 {
2256 PULONGLONG PageDirTable;
2257 PULONGLONG Pde;
2258 ULONGLONG ZeroPde = 0LL;
2259 ULONG i;
2260
2261 for (i = PAE_ADDR_TO_PDTE_OFFSET(Address); i <= PAE_ADDR_TO_PDTE_OFFSET(Address + Size); i++)
2262 {
2263 if (i == PAE_ADDR_TO_PDTE_OFFSET(Address))
2264 {
2265 StartOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address);
2266 }
2267 else
2268 {
2269 StartOffset = 0;
2270 }
2271 if (i == PAE_ADDR_TO_PDTE_OFFSET(Address + Size))
2272 {
2273 EndOffset = PAE_ADDR_TO_PDE_PAGE_OFFSET(Address + Size);
2274 }
2275 else
2276 {
2277 EndOffset = 511;
2278 }
2279
2280 if (Process != NULL && Process != PsGetCurrentProcess())
2281 {
2282 PageDirTable = MmCreateHyperspaceMapping(PAE_PTE_TO_PFN(Process->Pcb.DirectoryTableBase.QuadPart));
2283 Pde = (PULONGLONG)MmCreateHyperspaceMapping(PTE_TO_PFN(PageDirTable[i]));
2284 MmDeleteHyperspaceMapping(PageDirTable);
2285 }
2286 else
2287 {
2288 Pde = (PULONGLONG)PAE_PAGEDIRECTORY_MAP + i*512;
2289 }
2290
2291 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2292 {
2293 if (i * 512 + Offset < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) || i * 512 + Offset >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP)+4)
2294 {
2295 ExfInterlockedCompareExchange64UL(&Pde[Offset], &MmGlobalKernelPageDirectoryForPAE[i*512 + Offset], &ZeroPde);
2296 }
2297 }
2298 MmUnmapPageTable((PULONG)Pde);
2299 }
2300 }
2301 else
2302 {
2303 PULONG Pde;
2304 StartOffset = ADDR_TO_PDE_OFFSET(Address);
2305 EndOffset = ADDR_TO_PDE_OFFSET(Address + Size);
2306
2307 if (Process != NULL && Process != PsGetCurrentProcess())
2308 {
2309 Pde = MmCreateHyperspaceMapping(PTE_TO_PFN(Process->Pcb.DirectoryTableBase.u.LowPart));
2310 }
2311 else
2312 {
2313 Pde = (PULONG)PAGEDIRECTORY_MAP;
2314 }
2315 for (Offset = StartOffset; Offset <= EndOffset; Offset++)
2316 {
2317 if (Offset != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP))
2318 {
2319 InterlockedCompareExchangeUL(&Pde[Offset], MmGlobalKernelPageDirectory[Offset], 0);
2320 }
2321 }
2322 if (Pde != (PULONG)PAGEDIRECTORY_MAP)
2323 {
2324 MmDeleteHyperspaceMapping(Pde);
2325 }
2326 }
2327 }
2328
2329 VOID INIT_FUNCTION
2330 MmInitGlobalKernelPageDirectory(VOID)
2331 {
2332 ULONG i;
2333
2334 DPRINT("MmInitGlobalKernelPageDirectory()\n");
2335
2336 if (Ke386Pae)
2337 {
2338 PULONGLONG CurrentPageDirectory = (PULONGLONG)PAE_PAGEDIRECTORY_MAP;
2339 for (i = PAE_ADDR_TO_PDE_OFFSET(KERNEL_BASE); i < 4 * 512; i++)
2340 {
2341 if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
2342 !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) + 2) &&
2343 0LL == MmGlobalKernelPageDirectoryForPAE[i] && 0LL != CurrentPageDirectory[i])
2344 {
2345 ExfpInterlockedExchange64UL(&MmGlobalKernelPageDirectoryForPAE[i], &CurrentPageDirectory[i]);
2346 if (Ke386GlobalPagesEnabled)
2347 {
2348 MmGlobalKernelPageDirectoryForPAE[i] |= PA_GLOBAL;
2349 CurrentPageDirectory[i] |= PA_GLOBAL;
2350 }
2351 }
2352 }
2353 }
2354 else
2355 {
2356 PULONG CurrentPageDirectory = (PULONG)PAGEDIRECTORY_MAP;
2357 for (i = ADDR_TO_PDE_OFFSET(KERNEL_BASE); i < 1024; i++)
2358 {
2359 if (i != ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) &&
2360 i != ADDR_TO_PDE_OFFSET(HYPERSPACE) &&
2361 0 == MmGlobalKernelPageDirectory[i] && 0 != CurrentPageDirectory[i])
2362 {
2363 MmGlobalKernelPageDirectory[i] = CurrentPageDirectory[i];
2364 if (Ke386GlobalPagesEnabled)
2365 {
2366 MmGlobalKernelPageDirectory[i] |= PA_GLOBAL;
2367 CurrentPageDirectory[i] |= PA_GLOBAL;
2368 }
2369 }
2370 }
2371 }
2372 }
2373
2374 extern ULONGLONG pae_pagedirtable;
2375
2376 VOID INIT_FUNCTION
2377 MiEnablePAE(PVOID* LastKernelAddress)
2378 {
2379 PULONGLONG PageDirTable;
2380 PULONGLONG PageDir;
2381
2382 ULONG i, k;
2383
2384 ULONG Flags;
2385
2386 PageDirTable = &pae_pagedirtable;
2387
2388 if (LastKernelAddress)
2389 {
2390 /* This is the boot processor */
2391 memcpy(PageDirTable, (PVOID)PAGEDIRECTORY_MAP, PAGE_SIZE);
2392
2393 PageDir = (PULONGLONG)*LastKernelAddress;
2394 (*LastKernelAddress) += 6 * PAGE_SIZE;
2395 PageDirTable[0] = MmGetPhysicalAddress((PVOID)PageDir).QuadPart | PA_PRESENT;
2396 PageDirTable[1] = PageDirTable[0] + PAGE_SIZE;
2397 PageDirTable[2] = PageDirTable[1] + PAGE_SIZE;
2398 PageDirTable[3] = PageDirTable[2] + PAGE_SIZE;
2399
2400 memset(PageDir, 0, 6 * PAGE_SIZE);
2401
2402 for (i = 0; i < 4; i++)
2403 {
2404 PageDir[PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + i] = PageDirTable[i] | PA_READWRITE;
2405 }
2406 PageDir[PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE)] = (PageDirTable[0] + 4 * PAGE_SIZE) | PA_READWRITE;
2407 PageDir[PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE)+1] = (PageDirTable[0] + 5 * PAGE_SIZE) | PA_READWRITE;
2408
2409 for (i = 0; i < 2048; i++)
2410 {
2411 if (!(i >= PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) && i < PAE_ADDR_TO_PDE_OFFSET(PAGETABLE_MAP) + 4) &&
2412 !(i >= PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE) && i < PAE_ADDR_TO_PDE_OFFSET(HYPERSPACE)+2))
2413
2414 {
2415 PVOID Address = (PVOID)(i * 512 * PAGE_SIZE);
2416 PULONG Pde = ADDR_TO_PDE(Address);
2417 PULONGLONG PtePAE=NULL;
2418
2419 if (*Pde != 0)
2420 {
2421 if (PageDir[i] == 0LL)
2422 {
2423 PtePAE = (PULONGLONG)(ULONG_PTR)*LastKernelAddress;
2424 (*LastKernelAddress) += PAGE_SIZE;
2425 PageDir[i] = MmGetPhysicalAddress((PVOID)PtePAE).QuadPart | PA_PRESENT | PA_READWRITE;
2426 memset(PtePAE, 0, PAGE_SIZE);
2427 }
2428 for (k = 0; k < 512; k++)
2429 {
2430 PULONG Pte = ADDR_TO_PTE(Address + k * PAGE_SIZE);
2431 if (*Pte != 0)
2432 {
2433 PtePAE[k] = *Pte;
2434
2435 }
2436 }
2437 }
2438 }
2439 }
2440 }
2441 else
2442 {
2443 /* this is an application processor in a mp system */
2444 if (!Ke386Pae)
2445 {
2446 return;
2447 }
2448 }
2449
2450 Ke386SaveFlags(Flags);
2451 Ke386DisableInterrupts();
2452
2453 Ke386SetPageTableDirectory(MmGetPhysicalAddress(PageDirTable).u.LowPart)
2454
2455 Ke386SetCr4(Ke386GetCr4() | X86_CR4_PAE);
2456 if (LastKernelAddress)
2457 {
2458 Ke386Pae = TRUE;
2459 }
2460 Ke386RestoreFlags(Flags);
2461 }
2462
2463 ULONG
2464 MiGetUserPageDirectoryCount(VOID)
2465 {
2466 return Ke386Pae ? PAE_ADDR_TO_PDE_OFFSET(KERNEL_BASE) : ADDR_TO_PDE_OFFSET(KERNEL_BASE);
2467 }
2468
2469 VOID INIT_FUNCTION
2470 MiInitPageDirectoryMap(VOID)
2471 {
2472 MEMORY_AREA* kernel_map_desc = NULL;
2473 MEMORY_AREA* hyperspace_desc = NULL;
2474 PHYSICAL_ADDRESS BoundaryAddressMultiple;
2475 PVOID BaseAddress;
2476
2477 DPRINT("MiInitPageDirectoryMap()\n");
2478
2479 BoundaryAddressMultiple.QuadPart = 0;
2480 BaseAddress = (PVOID)PAGETABLE_MAP;
2481 MmCreateMemoryArea(NULL,
2482 MmGetKernelAddressSpace(),
2483 MEMORY_AREA_SYSTEM,
2484 &BaseAddress,
2485 Ke386Pae ? 0x800000 : 0x400000,
2486 0,
2487 &kernel_map_desc,
2488 TRUE,
2489 FALSE,
2490 BoundaryAddressMultiple);
2491 BaseAddress = (PVOID)HYPERSPACE;
2492 MmCreateMemoryArea(NULL,
2493 MmGetKernelAddressSpace(),
2494 MEMORY_AREA_SYSTEM,
2495 &BaseAddress,
2496 0x400000,
2497 0,
2498 &hyperspace_desc,
2499 TRUE,
2500 FALSE,
2501 BoundaryAddressMultiple);
2502 }
2503
2504 /* EOF */
A free Windows-compatible Operating System