projects / reactos.git / blob
? search:


f355de53097f42a2627bac76f339269516e2a2f6
[reactos.git] / reactos / ntoskrnl / mm / freelist.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/freelist.c
5 * PURPOSE: Handle the list of free physical pages
6 * PROGRAMMER: David Welch (welch@cwcom.net)
7 * UPDATE HISTORY:
8 * 27/05/98: Created
9 * 18/08/98: Added a fix from Robert Bergkvist
10 */
11
12 /* INCLUDES ****************************************************************/
13
14 #include <ddk/ntddk.h>
15 #include <internal/mm.h>
16 #include <internal/ntoskrnl.h>
17
18 #define NDEBUG
19 #include <internal/debug.h>
20
21 /* TYPES *******************************************************************/
22
23 #define MM_PHYSICAL_PAGE_FREE (0x1)
24 #define MM_PHYSICAL_PAGE_USED (0x2)
25 #define MM_PHYSICAL_PAGE_BIOS (0x3)
26
27 typedef struct _PHYSICAL_PAGE
28 {
29 union
30 {
31 struct
32 {
33 ULONG Type: 2;
34 ULONG Consumer: 3;
35 ULONG Zero: 1;
36 }
37 Flags;
38 ULONG AllFlags;
39 };
40
41 LIST_ENTRY ListEntry;
42 ULONG ReferenceCount;
43 SWAPENTRY SavedSwapEntry;
44 ULONG LockCount;
45 ULONG MapCount;
46 struct _MM_RMAP_ENTRY* RmapListHead;
47 }
48 PHYSICAL_PAGE, *PPHYSICAL_PAGE;
49
50
51 /* GLOBALS ****************************************************************/
52
53 static PPHYSICAL_PAGE MmPageArray;
54 ULONG MmPageArraySize;
55
56 static KSPIN_LOCK PageListLock;
57 static LIST_ENTRY UsedPageListHeads[MC_MAXIMUM];
58 static LIST_ENTRY FreeZeroedPageListHead;
59 static LIST_ENTRY FreeUnzeroedPageListHead;
60 static LIST_ENTRY BiosPageListHead;
61
62 static HANDLE ZeroPageThreadHandle;
63 static CLIENT_ID ZeroPageThreadId;
64 static KEVENT ZeroPageThreadEvent;
65
66 static ULONG UnzeroedPageCount = 0;
67
68 /* FUNCTIONS *************************************************************/
69
70 VOID
71 MmTransferOwnershipPage(PFN_TYPE Pfn, ULONG NewConsumer)
72 {
73 KIRQL oldIrql;
74
75 KeAcquireSpinLock(&PageListLock, &oldIrql);
76 if (MmPageArray[Pfn].MapCount != 0)
77 {
78 DbgPrint("Transfering mapped page.\n");
79 KEBUGCHECK(0);
80 }
81 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
82 {
83 DPRINT1("Type: %d\n", MmPageArray[Pfn].Flags.Type);
84 KEBUGCHECK(0);
85 }
86 if (MmPageArray[Pfn].ReferenceCount != 1)
87 {
88 DPRINT1("ReferenceCount: %d\n", MmPageArray[Pfn].ReferenceCount);
89 KEBUGCHECK(0);
90 }
91 RemoveEntryList(&MmPageArray[Pfn].ListEntry);
92 InsertTailList(&UsedPageListHeads[NewConsumer],
93 &MmPageArray[Pfn].ListEntry);
94 MmPageArray[Pfn].Flags.Consumer = NewConsumer;
95 KeReleaseSpinLock(&PageListLock, oldIrql);
96 MiZeroPage(Pfn);
97 }
98
99 PFN_TYPE
100 MmGetLRUFirstUserPage(VOID)
101 {
102 PLIST_ENTRY NextListEntry;
103 PHYSICAL_PAGE* PageDescriptor;
104 KIRQL oldIrql;
105
106 KeAcquireSpinLock(&PageListLock, &oldIrql);
107 NextListEntry = UsedPageListHeads[MC_USER].Flink;
108 if (NextListEntry == &UsedPageListHeads[MC_USER])
109 {
110 KeReleaseSpinLock(&PageListLock, oldIrql);
111 return 0;
112 }
113 PageDescriptor = CONTAINING_RECORD(NextListEntry, PHYSICAL_PAGE, ListEntry);
114 KeReleaseSpinLock(&PageListLock, oldIrql);
115 return PageDescriptor - MmPageArray;
116 }
117
118 VOID
119 MmSetLRULastPage(PFN_TYPE Pfn)
120 {
121 KIRQL oldIrql;
122
123 assert (Pfn < MmPageArraySize);
124 KeAcquireSpinLock(&PageListLock, &oldIrql);
125 if (MmPageArray[Pfn].Flags.Type == MM_PHYSICAL_PAGE_USED &&
126 MmPageArray[Pfn].Flags.Consumer == MC_USER)
127 {
128 RemoveEntryList(&MmPageArray[Pfn].ListEntry);
129 InsertTailList(&UsedPageListHeads[MC_USER],
130 &MmPageArray[Pfn].ListEntry);
131 }
132 KeReleaseSpinLock(&PageListLock, oldIrql);
133 }
134
135 PFN_TYPE
136 MmGetLRUNextUserPage(PFN_TYPE PreviousPfn)
137 {
138 PLIST_ENTRY NextListEntry;
139 PHYSICAL_PAGE* PageDescriptor;
140 KIRQL oldIrql;
141
142 KeAcquireSpinLock(&PageListLock, &oldIrql);
143 if (MmPageArray[PreviousPfn].Flags.Type != MM_PHYSICAL_PAGE_USED ||
144 MmPageArray[PreviousPfn].Flags.Consumer != MC_USER)
145 {
146 NextListEntry = UsedPageListHeads[MC_USER].Flink;
147 }
148 else
149 {
150 NextListEntry = MmPageArray[PreviousPfn].ListEntry.Flink;
151 }
152 if (NextListEntry == &UsedPageListHeads[MC_USER])
153 {
154 KeReleaseSpinLock(&PageListLock, oldIrql);
155 return 0;
156 }
157 PageDescriptor = CONTAINING_RECORD(NextListEntry, PHYSICAL_PAGE, ListEntry);
158 KeReleaseSpinLock(&PageListLock, oldIrql);
159 return PageDescriptor - MmPageArray;
160 }
161
162 PFN_TYPE
163 MmGetContinuousPages(ULONG NumberOfBytes,
164 PHYSICAL_ADDRESS LowestAcceptableAddress,
165 PHYSICAL_ADDRESS HighestAcceptableAddress,
166 ULONG Alignment)
167 {
168 ULONG NrPages;
169 ULONG i;
170 ULONG start;
171 ULONG length;
172 KIRQL oldIrql;
173
174 NrPages = PAGE_ROUND_UP(NumberOfBytes) / PAGE_SIZE;
175
176 KeAcquireSpinLock(&PageListLock, &oldIrql);
177
178 start = -1;
179 length = 0;
180 for (i = (LowestAcceptableAddress.QuadPart / PAGE_SIZE); i < (HighestAcceptableAddress.QuadPart / PAGE_SIZE); )
181 {
182 if (MmPageArray[i].Flags.Type == MM_PHYSICAL_PAGE_FREE)
183 {
184 if (start == -1)
185 {
186 start = i;
187 length = 1;
188 }
189 else
190 {
191 length++;
192 }
193 i++;
194 if (length == NrPages)
195 {
196 break;
197 }
198 }
199 else
200 {
201 start = -1;
202 /*
203 * Fast forward to the base of the next aligned region
204 */
205 i = ROUND_UP((i + 1), (Alignment / PAGE_SIZE));
206 }
207 }
208 if (start == -1 || length != NrPages)
209 {
210 KeReleaseSpinLock(&PageListLock, oldIrql);
211 return 0;
212 }
213 for (i = start; i < (start + length); i++)
214 {
215 RemoveEntryList(&MmPageArray[i].ListEntry);
216 if (MmPageArray[i].Flags.Zero == 0)
217 {
218 UnzeroedPageCount--;
219 }
220 MmStats.NrFreePages--;
221 MmStats.NrSystemPages++;
222 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
223 MmPageArray[i].Flags.Consumer = MC_NPPOOL;
224 MmPageArray[i].ReferenceCount = 1;
225 MmPageArray[i].LockCount = 0;
226 MmPageArray[i].MapCount = 0;
227 MmPageArray[i].SavedSwapEntry = 0;
228 InsertTailList(&UsedPageListHeads[MC_NPPOOL],
229 &MmPageArray[i].ListEntry);
230 }
231 KeReleaseSpinLock(&PageListLock, oldIrql);
232 for (i = start; i < (start + length); i++)
233 {
234 if (MmPageArray[i].Flags.Zero == 0)
235 {
236 MiZeroPage(i);
237 }
238 else
239 {
240 MmPageArray[i].Flags.Zero = 0;
241 }
242 }
243
244 return start;
245 }
246
247 VOID INIT_FUNCTION
248 MiParseRangeToFreeList(PADDRESS_RANGE Range)
249 {
250 ULONG i, first, last;
251
252 /* FIXME: Not 64-bit ready */
253
254 DPRINT("Range going to free list (Base 0x%X, Length 0x%X, Type 0x%X)\n",
255 Range->BaseAddrLow,
256 Range->LengthLow,
257 Range->Type);
258
259 first = (Range->BaseAddrLow + PAGE_SIZE - 1) / PAGE_SIZE;
260 last = first + ((Range->LengthLow + PAGE_SIZE - 1) / PAGE_SIZE);
261 for (i = first; i < last && i < MmPageArraySize; i++)
262 {
263 if (MmPageArray[i].Flags.Type == 0)
264 {
265 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
266 MmPageArray[i].Flags.Zero = 0;
267 MmPageArray[i].ReferenceCount = 0;
268 InsertTailList(&FreeUnzeroedPageListHead,
269 &MmPageArray[i].ListEntry);
270 MmStats.NrFreePages++;
271 UnzeroedPageCount++;
272 }
273 }
274 }
275
276 VOID INIT_FUNCTION
277 MiParseRangeToBiosList(PADDRESS_RANGE Range)
278 {
279 ULONG i, first, last;
280
281 /* FIXME: Not 64-bit ready */
282
283 DPRINT("Range going to bios list (Base 0x%X, Length 0x%X, Type 0x%X)\n",
284 Range->BaseAddrLow,
285 Range->LengthLow,
286 Range->Type);
287
288 first = (Range->BaseAddrLow + PAGE_SIZE - 1) / PAGE_SIZE;
289 last = first + ((Range->LengthLow + PAGE_SIZE - 1) / PAGE_SIZE);
290 for (i = first; i < last && i < MmPageArraySize; i++)
291 {
292 /* Remove the page from the free list if it is there */
293 if (MmPageArray[i].Flags.Type == MM_PHYSICAL_PAGE_FREE)
294 {
295 RemoveEntryList(&MmPageArray[i].ListEntry);
296 UnzeroedPageCount--;
297 MmStats.NrFreePages--;
298
299 }
300
301 if (MmPageArray[i].Flags.Type != MM_PHYSICAL_PAGE_BIOS)
302 {
303 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
304 MmPageArray[i].Flags.Consumer = MC_NPPOOL;
305 MmPageArray[i].ReferenceCount = 1;
306 InsertTailList(&BiosPageListHead,
307 &MmPageArray[i].ListEntry);
308 MmStats.NrSystemPages++;
309 }
310 }
311 }
312
313 VOID INIT_FUNCTION
314 MiParseBIOSMemoryMap(PADDRESS_RANGE BIOSMemoryMap,
315 ULONG AddressRangeCount)
316 {
317 PADDRESS_RANGE p;
318 ULONG i;
319
320 p = BIOSMemoryMap;
321 for (i = 0; i < AddressRangeCount; i++, p++)
322 {
323 if (p->Type == 1)
324 {
325 MiParseRangeToFreeList(p);
326 }
327 else
328 {
329 MiParseRangeToBiosList(p);
330 }
331 }
332 }
333
334 PVOID INIT_FUNCTION
335 MmInitializePageList(PVOID FirstPhysKernelAddress,
336 PVOID LastPhysKernelAddress,
337 ULONG MemorySizeInPages,
338 ULONG LastKernelAddress,
339 PADDRESS_RANGE BIOSMemoryMap,
340 ULONG AddressRangeCount)
341 /*
342 * FUNCTION: Initializes the page list with all pages free
343 * except those known to be reserved and those used by the kernel
344 * ARGUMENTS:
345 * PageBuffer = Page sized buffer
346 * FirstKernelAddress = First physical address used by the kernel
347 * LastKernelAddress = Last physical address used by the kernel
348 */
349 {
350 ULONG i;
351 ULONG Reserved;
352 NTSTATUS Status;
353
354 DPRINT("MmInitializePageList(FirstPhysKernelAddress %x, "
355 "LastPhysKernelAddress %x, "
356 "MemorySizeInPages %x, LastKernelAddress %x)\n",
357 FirstPhysKernelAddress,
358 LastPhysKernelAddress,
359 MemorySizeInPages,
360 LastKernelAddress);
361
362 for (i = 0; i < MC_MAXIMUM; i++)
363 {
364 InitializeListHead(&UsedPageListHeads[i]);
365 }
366 KeInitializeSpinLock(&PageListLock);
367 InitializeListHead(&FreeUnzeroedPageListHead);
368 InitializeListHead(&FreeZeroedPageListHead);
369 InitializeListHead(&BiosPageListHead);
370
371 LastKernelAddress = PAGE_ROUND_UP(LastKernelAddress);
372
373 MmPageArraySize = MemorySizeInPages;
374 Reserved =
375 PAGE_ROUND_UP((MmPageArraySize * sizeof(PHYSICAL_PAGE))) / PAGE_SIZE;
376 MmPageArray = (PHYSICAL_PAGE *)LastKernelAddress;
377
378 DPRINT("Reserved %d\n", Reserved);
379
380 LastKernelAddress = PAGE_ROUND_UP(LastKernelAddress);
381 LastKernelAddress = ((ULONG)LastKernelAddress + (Reserved * PAGE_SIZE));
382 LastPhysKernelAddress = (PVOID)PAGE_ROUND_UP(LastPhysKernelAddress);
383 LastPhysKernelAddress = (char*)LastPhysKernelAddress + (Reserved * PAGE_SIZE);
384
385 MmStats.NrTotalPages = 0;
386 MmStats.NrSystemPages = 0;
387 MmStats.NrUserPages = 0;
388 MmStats.NrReservedPages = 0;
389 MmStats.NrFreePages = 0;
390 MmStats.NrLockedPages = 0;
391
392 for (i = 0; i < Reserved; i++)
393 {
394 PVOID Address = (char*)(ULONG)MmPageArray + (i * PAGE_SIZE);
395 if (!MmIsPagePresent(NULL, Address))
396 {
397 ULONG Pfn = ((ULONG_PTR)LastPhysKernelAddress >> PAGE_SHIFT) - Reserved + i;
398 Status =
399 MmCreateVirtualMappingUnsafe(NULL,
400 Address,
401 PAGE_READWRITE,
402 &Pfn,
403 1);
404 if (!NT_SUCCESS(Status))
405 {
406 DbgPrint("Unable to create virtual mapping\n");
407 KEBUGCHECK(0);
408 }
409 }
410 memset((char*)MmPageArray + (i * PAGE_SIZE), 0, PAGE_SIZE);
411 }
412
413
414 /*
415 * Page zero is reserved
416 */
417 MmPageArray[0].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
418 MmPageArray[0].Flags.Consumer = MC_NPPOOL;
419 MmPageArray[0].Flags.Zero = 0;
420 MmPageArray[0].ReferenceCount = 0;
421 InsertTailList(&BiosPageListHead,
422 &MmPageArray[0].ListEntry);
423
424 /*
425 * Page one is reserved for the initial KPCR
426 */
427 MmPageArray[1].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
428 MmPageArray[1].Flags.Consumer = MC_NPPOOL;
429 MmPageArray[1].Flags.Zero = 0;
430 MmPageArray[1].ReferenceCount = 0;
431 InsertTailList(&BiosPageListHead,
432 &MmPageArray[1].ListEntry);
433
434 i = 2;
435 if ((ULONG)FirstPhysKernelAddress < 0xa0000)
436 {
437 MmStats.NrFreePages += (((ULONG)FirstPhysKernelAddress/PAGE_SIZE) - 2);
438 for (; i<((ULONG)FirstPhysKernelAddress/PAGE_SIZE); i++)
439 {
440 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
441 MmPageArray[i].Flags.Zero = 0;
442 MmPageArray[i].ReferenceCount = 0;
443 InsertTailList(&FreeUnzeroedPageListHead,
444 &MmPageArray[i].ListEntry);
445 UnzeroedPageCount++;
446 }
447 MmStats.NrSystemPages +=
448 ((((ULONG)LastPhysKernelAddress) / PAGE_SIZE) - i);
449 for (; i<((ULONG)LastPhysKernelAddress / PAGE_SIZE); i++)
450 {
451 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
452 MmPageArray[i].Flags.Zero = 0;
453 MmPageArray[i].Flags.Consumer = MC_NPPOOL;
454 MmPageArray[i].ReferenceCount = 1;
455 MmPageArray[i].MapCount = 1;
456 InsertTailList(&UsedPageListHeads[MC_NPPOOL],
457 &MmPageArray[i].ListEntry);
458 }
459 MmStats.NrFreePages += ((0xa0000/PAGE_SIZE) - i);
460 for (; i<(0xa0000/PAGE_SIZE); i++)
461 {
462 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
463 MmPageArray[i].Flags.Zero = 0;
464 MmPageArray[i].ReferenceCount = 0;
465 InsertTailList(&FreeUnzeroedPageListHead,
466 &MmPageArray[i].ListEntry);
467 UnzeroedPageCount++;
468 }
469 MmStats.NrReservedPages += ((0x100000/PAGE_SIZE) - i);
470 for (; i<(0x100000 / PAGE_SIZE); i++)
471 {
472 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
473 MmPageArray[i].Flags.Zero = 0;
474 MmPageArray[i].Flags.Consumer = MC_NPPOOL;
475 MmPageArray[i].ReferenceCount = 1;
476 InsertTailList(&BiosPageListHead,
477 &MmPageArray[i].ListEntry);
478 }
479 }
480 else
481 {
482 MmStats.NrFreePages += ((0xa0000 / PAGE_SIZE) - 2);
483 for (; i<(0xa0000 / PAGE_SIZE); i++)
484 {
485 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
486 MmPageArray[i].Flags.Zero = 0;
487 MmPageArray[i].ReferenceCount = 0;
488 InsertTailList(&FreeUnzeroedPageListHead,
489 &MmPageArray[i].ListEntry);
490 UnzeroedPageCount++;
491 }
492 MmStats.NrReservedPages += (0x60000 / PAGE_SIZE);
493 for (; i<(0x100000 / PAGE_SIZE); i++)
494 {
495 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_BIOS;
496 MmPageArray[i].Flags.Zero = 0;
497 MmPageArray[i].Flags.Consumer = MC_NPPOOL;
498 MmPageArray[i].ReferenceCount = 1;
499 InsertTailList(&BiosPageListHead,
500 &MmPageArray[i].ListEntry);
501 }
502 MmStats.NrFreePages += (((ULONG)FirstPhysKernelAddress/PAGE_SIZE) - i);
503 for (; i<((ULONG)FirstPhysKernelAddress/PAGE_SIZE); i++)
504 {
505 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
506 MmPageArray[i].Flags.Zero = 0;
507 MmPageArray[i].ReferenceCount = 0;
508 InsertTailList(&FreeUnzeroedPageListHead,
509 &MmPageArray[i].ListEntry);
510 UnzeroedPageCount++;
511 }
512 MmStats.NrSystemPages +=
513 (((ULONG)LastPhysKernelAddress/PAGE_SIZE) - i);
514 for (; i<((ULONG)LastPhysKernelAddress/PAGE_SIZE); i++)
515 {
516 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_USED;
517 MmPageArray[i].Flags.Zero = 0;
518 MmPageArray[i].Flags.Consumer = MC_NPPOOL;
519 MmPageArray[i].ReferenceCount = 1;
520 MmPageArray[i].MapCount = 1;
521 InsertTailList(&UsedPageListHeads[MC_NPPOOL],
522 &MmPageArray[i].ListEntry);
523 }
524 }
525
526 MmStats.NrFreePages += (MemorySizeInPages - i);
527 for (; i<MemorySizeInPages; i++)
528 {
529 MmPageArray[i].Flags.Type = MM_PHYSICAL_PAGE_FREE;
530 MmPageArray[i].Flags.Zero = 0;
531 MmPageArray[i].ReferenceCount = 0;
532 InsertTailList(&FreeUnzeroedPageListHead,
533 &MmPageArray[i].ListEntry);
534 UnzeroedPageCount++;
535 }
536
537 if ((BIOSMemoryMap != NULL) && (AddressRangeCount > 0))
538 {
539 MiParseBIOSMemoryMap(
540 BIOSMemoryMap,
541 AddressRangeCount);
542 }
543
544 KeInitializeEvent(&ZeroPageThreadEvent, NotificationEvent, TRUE);
545
546 MmStats.NrTotalPages = MmStats.NrFreePages + MmStats.NrSystemPages +
547 MmStats.NrReservedPages + MmStats.NrUserPages;
548 MmInitializeBalancer(MmStats.NrFreePages, MmStats.NrSystemPages + MmStats.NrReservedPages);
549 return((PVOID)LastKernelAddress);
550 }
551
552 VOID
553 MmSetFlagsPage(PFN_TYPE Pfn, ULONG Flags)
554 {
555 KIRQL oldIrql;
556
557 assert (Pfn < MmPageArraySize);
558 KeAcquireSpinLock(&PageListLock, &oldIrql);
559 MmPageArray[Pfn].AllFlags = Flags;
560 KeReleaseSpinLock(&PageListLock, oldIrql);
561 }
562
563 VOID
564 MmSetRmapListHeadPage(PFN_TYPE Pfn, struct _MM_RMAP_ENTRY* ListHead)
565 {
566 MmPageArray[Pfn].RmapListHead = ListHead;
567 }
568
569 struct _MM_RMAP_ENTRY*
570 MmGetRmapListHeadPage(PFN_TYPE Pfn)
571 {
572 return(MmPageArray[Pfn].RmapListHead);
573 }
574
575 VOID
576 MmMarkPageMapped(PFN_TYPE Pfn)
577 {
578 KIRQL oldIrql;
579
580 if (Pfn < MmPageArraySize)
581 {
582 KeAcquireSpinLock(&PageListLock, &oldIrql);
583 if (MmPageArray[Pfn].Flags.Type == MM_PHYSICAL_PAGE_FREE)
584 {
585 DbgPrint("Mapping non-used page\n");
586 KEBUGCHECK(0);
587 }
588 MmPageArray[Pfn].MapCount++;
589 KeReleaseSpinLock(&PageListLock, oldIrql);
590 }
591 }
592
593 VOID
594 MmMarkPageUnmapped(PFN_TYPE Pfn)
595 {
596 KIRQL oldIrql;
597
598 if (Pfn < MmPageArraySize)
599 {
600 KeAcquireSpinLock(&PageListLock, &oldIrql);
601 if (MmPageArray[Pfn].Flags.Type == MM_PHYSICAL_PAGE_FREE)
602 {
603 DbgPrint("Unmapping non-used page\n");
604 KEBUGCHECK(0);
605 }
606 if (MmPageArray[Pfn].MapCount == 0)
607 {
608 DbgPrint("Unmapping not mapped page\n");
609 KEBUGCHECK(0);
610 }
611 MmPageArray[Pfn].MapCount--;
612 KeReleaseSpinLock(&PageListLock, oldIrql);
613 }
614 }
615
616 ULONG
617 MmGetFlagsPage(PFN_TYPE Pfn)
618 {
619 KIRQL oldIrql;
620 ULONG Flags;
621
622 assert (Pfn < MmPageArraySize);
623 KeAcquireSpinLock(&PageListLock, &oldIrql);
624 Flags = MmPageArray[Pfn].AllFlags;
625 KeReleaseSpinLock(&PageListLock, oldIrql);
626
627 return(Flags);
628 }
629
630
631 VOID
632 MmSetSavedSwapEntryPage(PFN_TYPE Pfn, SWAPENTRY SavedSwapEntry)
633 {
634 KIRQL oldIrql;
635
636 assert (Pfn < MmPageArraySize);
637 KeAcquireSpinLock(&PageListLock, &oldIrql);
638 MmPageArray[Pfn].SavedSwapEntry = SavedSwapEntry;
639 KeReleaseSpinLock(&PageListLock, oldIrql);
640 }
641
642 SWAPENTRY
643 MmGetSavedSwapEntryPage(PFN_TYPE Pfn)
644 {
645 SWAPENTRY SavedSwapEntry;
646 KIRQL oldIrql;
647
648 assert (Pfn < MmPageArraySize);
649 KeAcquireSpinLock(&PageListLock, &oldIrql);
650 SavedSwapEntry = MmPageArray[Pfn].SavedSwapEntry;
651 KeReleaseSpinLock(&PageListLock, oldIrql);
652
653 return(SavedSwapEntry);
654 }
655
656 VOID
657 MmReferencePage(PFN_TYPE Pfn)
658 {
659 KIRQL oldIrql;
660
661 DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn << PAGE_SHIFT);
662
663 if (Pfn == 0 || Pfn >= MmPageArraySize)
664 {
665 KEBUGCHECK(0);
666 }
667
668 KeAcquireSpinLock(&PageListLock, &oldIrql);
669
670 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
671 {
672 DbgPrint("Referencing non-used page\n");
673 KEBUGCHECK(0);
674 }
675
676 MmPageArray[Pfn].ReferenceCount++;
677 KeReleaseSpinLock(&PageListLock, oldIrql);
678 }
679
680 ULONG
681 MmGetReferenceCountPage(PFN_TYPE Pfn)
682 {
683 KIRQL oldIrql;
684 ULONG RCount;
685
686 DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
687
688 if (Pfn == 0 || Pfn >= MmPageArraySize)
689 {
690 KEBUGCHECK(0);
691 }
692
693 KeAcquireSpinLock(&PageListLock, &oldIrql);
694
695 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
696 {
697 DbgPrint("Getting reference count for free page\n");
698 KEBUGCHECK(0);
699 }
700
701 RCount = MmPageArray[Pfn].ReferenceCount;
702
703 KeReleaseSpinLock(&PageListLock, oldIrql);
704 return(RCount);
705 }
706
707 BOOLEAN
708 MmIsUsablePage(PFN_TYPE Pfn)
709 {
710
711 DPRINT("MmIsUsablePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
712
713 if (Pfn == 0 || Pfn >= MmPageArraySize)
714 {
715 KEBUGCHECK(0);
716 }
717
718 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED &&
719 MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_BIOS)
720 {
721 return(FALSE);
722 }
723
724 return(TRUE);
725 }
726
727 VOID
728 MmDereferencePage(PFN_TYPE Pfn)
729 {
730 KIRQL oldIrql;
731
732 DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
733
734 if (Pfn == 0 || Pfn >= MmPageArraySize)
735 {
736 KEBUGCHECK(0);
737 }
738
739 KeAcquireSpinLock(&PageListLock, &oldIrql);
740
741 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
742 {
743 DbgPrint("Dereferencing free page\n");
744 KEBUGCHECK(0);
745 }
746 if (MmPageArray[Pfn].ReferenceCount == 0)
747 {
748 DbgPrint("Derefrencing page with reference count 0\n");
749 KEBUGCHECK(0);
750 }
751
752 MmPageArray[Pfn].ReferenceCount--;
753 if (MmPageArray[Pfn].ReferenceCount == 0)
754 {
755 MmStats.NrFreePages++;
756 MmStats.NrSystemPages--;
757 RemoveEntryList(&MmPageArray[Pfn].ListEntry);
758 if (MmPageArray[Pfn].RmapListHead != NULL)
759 {
760 DbgPrint("Freeing page with rmap entries.\n");
761 KEBUGCHECK(0);
762 }
763 if (MmPageArray[Pfn].MapCount != 0)
764 {
765 DbgPrint("Freeing mapped page (0x%x count %d)\n",
766 Pfn << PAGE_SHIFT, MmPageArray[Pfn].MapCount);
767 KEBUGCHECK(0);
768 }
769 if (MmPageArray[Pfn].LockCount > 0)
770 {
771 DbgPrint("Freeing locked page\n");
772 KEBUGCHECK(0);
773 }
774 if (MmPageArray[Pfn].SavedSwapEntry != 0)
775 {
776 DbgPrint("Freeing page with swap entry.\n");
777 KEBUGCHECK(0);
778 }
779 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
780 {
781 DbgPrint("Freeing page with flags %x\n",
782 MmPageArray[Pfn].Flags.Type);
783 KEBUGCHECK(0);
784 }
785 MmPageArray[Pfn].Flags.Type = MM_PHYSICAL_PAGE_FREE;
786 MmPageArray[Pfn].Flags.Consumer = MC_MAXIMUM;
787 InsertTailList(&FreeUnzeroedPageListHead,
788 &MmPageArray[Pfn].ListEntry);
789 UnzeroedPageCount++;
790 if (UnzeroedPageCount > 8 && 0 == KeReadStateEvent(&ZeroPageThreadEvent))
791 {
792 KeSetEvent(&ZeroPageThreadEvent, IO_NO_INCREMENT, FALSE);
793 }
794 }
795 KeReleaseSpinLock(&PageListLock, oldIrql);
796 }
797
798 ULONG
799 MmGetLockCountPage(PFN_TYPE Pfn)
800 {
801 KIRQL oldIrql;
802 ULONG LockCount;
803
804 DPRINT("MmGetLockCountPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
805
806 if (Pfn == 0 || Pfn >= MmPageArraySize)
807 {
808 KEBUGCHECK(0);
809 }
810
811 KeAcquireSpinLock(&PageListLock, &oldIrql);
812
813 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
814 {
815 DbgPrint("Getting lock count for free page\n");
816 KEBUGCHECK(0);
817 }
818
819 LockCount = MmPageArray[Pfn].LockCount;
820 KeReleaseSpinLock(&PageListLock, oldIrql);
821
822 return(LockCount);
823 }
824
825 VOID
826 MmLockPage(PFN_TYPE Pfn)
827 {
828 KIRQL oldIrql;
829
830 DPRINT("MmLockPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
831
832 if (Pfn == 0 || Pfn >= MmPageArraySize)
833 {
834 KEBUGCHECK(0);
835 }
836
837 KeAcquireSpinLock(&PageListLock, &oldIrql);
838
839 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
840 {
841 DbgPrint("Locking free page\n");
842 KEBUGCHECK(0);
843 }
844
845 MmPageArray[Pfn].LockCount++;
846 KeReleaseSpinLock(&PageListLock, oldIrql);
847 }
848
849 VOID
850 MmUnlockPage(PFN_TYPE Pfn)
851 {
852 KIRQL oldIrql;
853
854 DPRINT("MmUnlockPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
855
856 if (Pfn == 0 || Pfn >= MmPageArraySize)
857 {
858 KEBUGCHECK(0);
859 }
860
861 KeAcquireSpinLock(&PageListLock, &oldIrql);
862
863 if (MmPageArray[Pfn].Flags.Type != MM_PHYSICAL_PAGE_USED)
864 {
865 DbgPrint("Unlocking free page\n");
866 KEBUGCHECK(0);
867 }
868
869 MmPageArray[Pfn].LockCount--;
870 KeReleaseSpinLock(&PageListLock, oldIrql);
871 }
872
873 PFN_TYPE
874 MmAllocPage(ULONG Consumer, SWAPENTRY SavedSwapEntry)
875 {
876 PFN_TYPE PfnOffset;
877 PLIST_ENTRY ListEntry;
878 PPHYSICAL_PAGE PageDescriptor;
879 KIRQL oldIrql;
880 BOOLEAN NeedClear = FALSE;
881
882 DPRINT("MmAllocPage()\n");
883
884 KeAcquireSpinLock(&PageListLock, &oldIrql);
885 if (IsListEmpty(&FreeZeroedPageListHead))
886 {
887 if (IsListEmpty(&FreeUnzeroedPageListHead))
888 {
889 DPRINT1("MmAllocPage(): Out of memory\n");
890 KeReleaseSpinLock(&PageListLock, oldIrql);
891 return 0;
892 }
893 ListEntry = RemoveTailList(&FreeUnzeroedPageListHead);
894 UnzeroedPageCount--;
895
896 PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
897
898 NeedClear = TRUE;
899 }
900 else
901 {
902 ListEntry = RemoveTailList(&FreeZeroedPageListHead);
903
904 PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
905 }
906
907 if (PageDescriptor->Flags.Type != MM_PHYSICAL_PAGE_FREE)
908 {
909 DbgPrint("Got non-free page from freelist\n");
910 KEBUGCHECK(0);
911 }
912 if (PageDescriptor->MapCount != 0)
913 {
914 DbgPrint("Got mapped page from freelist\n");
915 KEBUGCHECK(0);
916 }
917 if (PageDescriptor->ReferenceCount != 0)
918 {
919 DPRINT1("%d\n", PageDescriptor->ReferenceCount);
920 KEBUGCHECK(0);
921 }
922 PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_USED;
923 PageDescriptor->Flags.Consumer = Consumer;
924 PageDescriptor->ReferenceCount = 1;
925 PageDescriptor->LockCount = 0;
926 PageDescriptor->MapCount = 0;
927 PageDescriptor->SavedSwapEntry = SavedSwapEntry;
928 InsertTailList(&UsedPageListHeads[Consumer], ListEntry);
929
930 MmStats.NrSystemPages++;
931 MmStats.NrFreePages--;
932
933 KeReleaseSpinLock(&PageListLock, oldIrql);
934
935 PfnOffset = PageDescriptor - MmPageArray;
936 if (NeedClear)
937 {
938 MiZeroPage(PfnOffset);
939 }
940 if (PageDescriptor->MapCount != 0)
941 {
942 DbgPrint("Returning mapped page.\n");
943 KEBUGCHECK(0);
944 }
945 return PfnOffset;
946 }
947
948
949 NTSTATUS STDCALL
950 MmZeroPageThreadMain(PVOID Ignored)
951 {
952 NTSTATUS Status;
953 KIRQL oldIrql;
954 PLIST_ENTRY ListEntry;
955 PPHYSICAL_PAGE PageDescriptor;
956 PFN_TYPE Pfn;
957 static PVOID Address = NULL;
958 ULONG Count;
959
960 while(1)
961 {
962 Status = KeWaitForSingleObject(&ZeroPageThreadEvent,
963 0,
964 KernelMode,
965 FALSE,
966 NULL);
967 if (!NT_SUCCESS(Status))
968 {
969 DbgPrint("ZeroPageThread: Wait failed\n");
970 KEBUGCHECK(0);
971 return(STATUS_UNSUCCESSFUL);
972 }
973
974 Count = 0;
975 KeAcquireSpinLock(&PageListLock, &oldIrql);
976 while (!IsListEmpty(&FreeUnzeroedPageListHead))
977 {
978 ListEntry = RemoveTailList(&FreeUnzeroedPageListHead);
979 UnzeroedPageCount--;
980 PageDescriptor = CONTAINING_RECORD(ListEntry, PHYSICAL_PAGE, ListEntry);
981 /* We set the page to used, because MmCreateVirtualMapping failed with unused pages */
982 PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_USED;
983 KeReleaseSpinLock(&PageListLock, oldIrql);
984 Count++;
985 Pfn = PageDescriptor - MmPageArray;
986 if (Address == NULL)
987 {
988 Address = ExAllocatePageWithPhysPage(Pfn);
989 }
990 else
991 {
992 Status = MmCreateVirtualMapping(NULL,
993 Address,
994 PAGE_READWRITE | PAGE_SYSTEM,
995 &Pfn,
996 1);
997 if (!NT_SUCCESS(Status))
998 {
999 DbgPrint("Unable to create virtual mapping\n");
1000 KEBUGCHECK(0);
1001 }
1002 }
1003 memset(Address, 0, PAGE_SIZE);
1004 MmDeleteVirtualMapping(NULL, (PVOID)Address, FALSE, NULL, NULL);
1005 KeAcquireSpinLock(&PageListLock, &oldIrql);
1006 if (PageDescriptor->MapCount != 0)
1007 {
1008 DbgPrint("Mapped page on freelist.\n");
1009 KEBUGCHECK(0);
1010 }
1011 PageDescriptor->Flags.Zero = 1;
1012 PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_FREE;
1013 InsertHeadList(&FreeZeroedPageListHead, ListEntry);
1014 }
1015 DPRINT("Zeroed %d pages.\n", Count);
1016 KeResetEvent(&ZeroPageThreadEvent);
1017 KeReleaseSpinLock(&PageListLock, oldIrql);
1018 }
1019 }
1020
1021 NTSTATUS INIT_FUNCTION
1022 MmInitZeroPageThread(VOID)
1023 {
1024 KPRIORITY Priority;
1025 NTSTATUS Status;
1026
1027 Status = PsCreateSystemThread(&ZeroPageThreadHandle,
1028 THREAD_ALL_ACCESS,
1029 NULL,
1030 NULL,
1031 &ZeroPageThreadId,
1032 (PKSTART_ROUTINE) MmZeroPageThreadMain,
1033 NULL);
1034 if (!NT_SUCCESS(Status))
1035 {
1036 return(Status);
1037 }
1038
1039 Priority = 1;
1040 NtSetInformationThread(ZeroPageThreadHandle,
1041 ThreadPriority,
1042 &Priority,
1043 sizeof(Priority));
1044
1045 return(STATUS_SUCCESS);
1046 }
1047
1048 /* EOF */
A free Windows-compatible Operating System