9c2f5bdbe18c5a246462b16de9800a80adf72a6a
[reactos.git] / ntoskrnl / mm / ARM3 / pfnlist.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pfnlist.c
5 * PURPOSE: ARM Memory Manager PFN List Manipulation
6 * PROGRAMMERS: ReactOS Portable Systems Group
7 */
8
9 /* INCLUDES *******************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 #line 15 "ARMĀ³::PFNLIST"
16 #define MODULE_INVOLVED_IN_ARM3
17 #include "../ARM3/miarm.h"
18
19 #if DBG
20 #define ASSERT_LIST_INVARIANT(x) \
21 do { \
22 ASSERT(((x)->Total == 0 && \
23 (x)->Flink == LIST_HEAD && \
24 (x)->Blink == LIST_HEAD) || \
25 ((x)->Total != 0 && \
26 (x)->Flink != LIST_HEAD && \
27 (x)->Blink != LIST_HEAD)); \
28 } while (0)
29 #else
30 #define ASSERT_LIST_INVARIANT(x)
31 #endif
32
33 /* GLOBALS ********************************************************************/
34
35 BOOLEAN MmDynamicPfn;
36 BOOLEAN MmMirroring;
37 ULONG MmSystemPageColor;
38
39 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
40 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
41 MMPFNLIST MmStandbyPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
42 MMPFNLIST MmModifiedPageListHead = {0, ModifiedPageList, LIST_HEAD, LIST_HEAD};
43 MMPFNLIST MmModifiedNoWritePageListHead = {0, ModifiedNoWritePageList, LIST_HEAD, LIST_HEAD};
44 MMPFNLIST MmBadPageListHead = {0, BadPageList, LIST_HEAD, LIST_HEAD};
45 MMPFNLIST MmRomPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
46
47 PMMPFNLIST MmPageLocationList[] =
48 {
49 &MmZeroedPageListHead,
50 &MmFreePageListHead,
51 &MmStandbyPageListHead,
52 &MmModifiedPageListHead,
53 &MmModifiedNoWritePageListHead,
54 &MmBadPageListHead,
55 NULL,
56 NULL
57 };
58
59 ULONG MI_PFN_CURRENT_USAGE;
60 CHAR MI_PFN_CURRENT_PROCESS_NAME[16] = "None yet";
61
62 /* FUNCTIONS ******************************************************************/
63
64 VOID
65 NTAPI
66 MiZeroPhysicalPage(IN PFN_NUMBER PageFrameIndex)
67 {
68 KIRQL OldIrql;
69 PVOID VirtualAddress;
70 PEPROCESS Process = PsGetCurrentProcess();
71
72 /* Map in hyperspace, then wipe it using XMMI or MEMSET */
73 VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
74 ASSERT(VirtualAddress);
75 KeZeroPages(VirtualAddress, PAGE_SIZE);
76 MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
77 }
78
79 VOID
80 NTAPI
81 MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
82 {
83 PFN_NUMBER OldFlink, OldBlink;
84 PMMPFNLIST ListHead;
85 MMLISTS ListName;
86 ULONG Color;
87 PMMCOLOR_TABLES ColorTable;
88 PMMPFN Pfn1;
89
90 /* Make sure the PFN lock is held */
91 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
92
93 /* Make sure the PFN entry isn't in-use */
94 ASSERT(Entry->u3.e1.WriteInProgress == 0);
95 ASSERT(Entry->u3.e1.ReadInProgress == 0);
96
97 /* Find the list for this entry, make sure it's the free or zero list */
98 ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
99 ListName = ListHead->ListName;
100 ASSERT(ListHead != NULL);
101 ASSERT(ListName <= FreePageList);
102 ASSERT_LIST_INVARIANT(ListHead);
103
104 /* Remove one count */
105 ASSERT(ListHead->Total != 0);
106 ListHead->Total--;
107
108 /* Get the forward and back pointers */
109 OldFlink = Entry->u1.Flink;
110 OldBlink = Entry->u2.Blink;
111
112 /* Check if the next entry is the list head */
113 if (OldFlink != LIST_HEAD)
114 {
115 /* It is not, so set the backlink of the actual entry, to our backlink */
116 MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
117 }
118 else
119 {
120 /* Set the list head's backlink instead */
121 ListHead->Blink = OldBlink;
122 }
123
124 /* Check if the back entry is the list head */
125 if (OldBlink != LIST_HEAD)
126 {
127 /* It is not, so set the backlink of the actual entry, to our backlink */
128 MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
129 }
130 else
131 {
132 /* Set the list head's backlink instead */
133 ListHead->Flink = OldFlink;
134 }
135
136 /* Get the page color */
137 OldBlink = MiGetPfnEntryIndex(Entry);
138 Color = OldBlink & MmSecondaryColorMask;
139
140 /* Get the first page on the color list */
141 ColorTable = &MmFreePagesByColor[ListName][Color];
142
143 /* Check if this was was actually the head */
144 OldFlink = ColorTable->Flink;
145 if (OldFlink == OldBlink)
146 {
147 /* Make the table point to the next page this page was linking to */
148 ColorTable->Flink = Entry->OriginalPte.u.Long;
149 if (ColorTable->Flink != LIST_HEAD)
150 {
151 /* And make the previous link point to the head now */
152 MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
153 }
154 else
155 {
156 /* And if that page was the head, loop the list back around */
157 ColorTable->Blink = (PVOID)LIST_HEAD;
158 }
159 }
160 else
161 {
162 /* This page shouldn't be pointing back to the head */
163 ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
164
165 /* Make the back link point to whoever the next page is */
166 Pfn1 = MI_PFN_ELEMENT(Entry->u4.PteFrame);
167 Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
168
169 /* Check if this page was pointing to the head */
170 if (Entry->OriginalPte.u.Long != LIST_HEAD)
171 {
172 /* Make the back link point to the head */
173 Pfn1 = MI_PFN_ELEMENT(Entry->OriginalPte.u.Long);
174 Pfn1->u4.PteFrame = Entry->u4.PteFrame;
175 }
176 else
177 {
178 /* Then the table is directly back pointing to this page now */
179 ColorTable->Blink = Pfn1;
180 }
181 }
182
183 /* One less colored page */
184 ASSERT(ColorTable->Count >= 1);
185 ColorTable->Count--;
186
187 /* ReactOS Hack */
188 Entry->OriginalPte.u.Long = 0;
189
190 /* We are not on a list anymore */
191 Entry->u1.Flink = Entry->u2.Blink = 0;
192 ASSERT_LIST_INVARIANT(ListHead);
193
194 /* See if we hit any thresholds */
195 if (MmAvailablePages == MmHighMemoryThreshold)
196 {
197 /* Clear the high memory event */
198 KeClearEvent(MiHighMemoryEvent);
199 }
200 else if (MmAvailablePages == MmLowMemoryThreshold)
201 {
202 /* Signal the low memory event */
203 KeSetEvent(MiLowMemoryEvent, 0, FALSE);
204 }
205
206 /* One less page */
207 if (--MmAvailablePages < MmMinimumFreePages)
208 {
209 /* FIXME: Should wake up the MPW and working set manager, if we had one */
210 }
211
212 #if MI_TRACE_PFNS
213 ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
214 Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
215 memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
216 // MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
217 // memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
218 #endif
219 }
220
221 PFN_NUMBER
222 NTAPI
223 MiRemovePageByColor(IN PFN_NUMBER PageIndex,
224 IN ULONG Color)
225 {
226 PMMPFN Pfn1;
227 PMMPFNLIST ListHead;
228 MMLISTS ListName;
229 PFN_NUMBER OldFlink, OldBlink;
230 ULONG OldColor, OldCache;
231 PMMCOLOR_TABLES ColorTable;
232
233 /* Make sure PFN lock is held */
234 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
235 ASSERT(Color < MmSecondaryColors);
236
237 /* Get the PFN entry */
238 Pfn1 = MI_PFN_ELEMENT(PageIndex);
239 ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
240 ASSERT(Pfn1->u3.e1.Rom == 0);
241
242 /* Capture data for later */
243 OldColor = Pfn1->u3.e1.PageColor;
244 OldCache = Pfn1->u3.e1.CacheAttribute;
245
246 /* Could be either on free or zero list */
247 ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
248 ASSERT_LIST_INVARIANT(ListHead);
249 ListName = ListHead->ListName;
250 ASSERT(ListName <= FreePageList);
251
252 /* Remove a page */
253 ListHead->Total--;
254
255 /* Get the forward and back pointers */
256 OldFlink = Pfn1->u1.Flink;
257 OldBlink = Pfn1->u2.Blink;
258
259 /* Check if the next entry is the list head */
260 if (OldFlink != LIST_HEAD)
261 {
262 /* It is not, so set the backlink of the actual entry, to our backlink */
263 MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
264 }
265 else
266 {
267 /* Set the list head's backlink instead */
268 ListHead->Blink = OldBlink;
269 }
270
271 /* Check if the back entry is the list head */
272 if (OldBlink != LIST_HEAD)
273 {
274 /* It is not, so set the backlink of the actual entry, to our backlink */
275 MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
276 }
277 else
278 {
279 /* Set the list head's backlink instead */
280 ListHead->Flink = OldFlink;
281 }
282
283 /* We are not on a list anymore */
284 ASSERT_LIST_INVARIANT(ListHead);
285 Pfn1->u1.Flink = Pfn1->u2.Blink = 0;
286
287 /* Zero flags but restore color and cache */
288 Pfn1->u3.e2.ShortFlags = 0;
289 Pfn1->u3.e1.PageColor = OldColor;
290 Pfn1->u3.e1.CacheAttribute = OldCache;
291
292 /* Get the first page on the color list */
293 ASSERT(Color < MmSecondaryColors);
294 ColorTable = &MmFreePagesByColor[ListName][Color];
295 ASSERT(ColorTable->Count >= 1);
296
297 /* Set the forward link to whoever we were pointing to */
298 ColorTable->Flink = Pfn1->OriginalPte.u.Long;
299
300 /* Get the first page on the color list */
301 if (ColorTable->Flink == LIST_HEAD)
302 {
303 /* This is the beginning of the list, so set the sentinel value */
304 ColorTable->Blink = (PVOID)LIST_HEAD;
305 }
306 else
307 {
308 /* The list is empty, so we are the first page */
309 MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
310 }
311
312 /* One less page */
313 ColorTable->Count--;
314
315 /* ReactOS Hack */
316 Pfn1->OriginalPte.u.Long = 0;
317
318 /* See if we hit any thresholds */
319 if (MmAvailablePages == MmHighMemoryThreshold)
320 {
321 /* Clear the high memory event */
322 KeClearEvent(MiHighMemoryEvent);
323 }
324 else if (MmAvailablePages == MmLowMemoryThreshold)
325 {
326 /* Signal the low memory event */
327 KeSetEvent(MiLowMemoryEvent, 0, FALSE);
328 }
329
330 /* One less page */
331 if (--MmAvailablePages < MmMinimumFreePages)
332 {
333 /* FIXME: Should wake up the MPW and working set manager, if we had one */
334 }
335
336 #if MI_TRACE_PFNS
337 //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
338 Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE;
339 memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
340 //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
341 //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
342 #endif
343
344 /* Return the page */
345 return PageIndex;
346 }
347
348 PFN_NUMBER
349 NTAPI
350 MiRemoveAnyPage(IN ULONG Color)
351 {
352 PFN_NUMBER PageIndex;
353 PMMPFN Pfn1;
354
355 /* Make sure PFN lock is held and we have pages */
356 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
357 ASSERT(MmAvailablePages != 0);
358 ASSERT(Color < MmSecondaryColors);
359
360 /* Check the colored free list */
361 PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
362 if (PageIndex == LIST_HEAD)
363 {
364 /* Check the colored zero list */
365 PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
366 if (PageIndex == LIST_HEAD)
367 {
368 /* Check the free list */
369 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
370 PageIndex = MmFreePageListHead.Flink;
371 Color = PageIndex & MmSecondaryColorMask;
372 if (PageIndex == LIST_HEAD)
373 {
374 /* Check the zero list */
375 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
376 PageIndex = MmZeroedPageListHead.Flink;
377 Color = PageIndex & MmSecondaryColorMask;
378 ASSERT(PageIndex != LIST_HEAD);
379 if (PageIndex == LIST_HEAD)
380 {
381 /* FIXME: Should check the standby list */
382 ASSERT(MmZeroedPageListHead.Total == 0);
383 }
384 }
385 }
386 }
387
388 /* Remove the page from its list */
389 PageIndex = MiRemovePageByColor(PageIndex, Color);
390
391 /* Sanity checks */
392 Pfn1 = MI_PFN_ELEMENT(PageIndex);
393 ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
394 (Pfn1->u3.e1.PageLocation == ZeroedPageList));
395 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
396 ASSERT(Pfn1->u2.ShareCount == 0);
397 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
398 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
399
400 /* Return the page */
401 return PageIndex;
402 }
403
404 PFN_NUMBER
405 NTAPI
406 MiRemoveZeroPage(IN ULONG Color)
407 {
408 PFN_NUMBER PageIndex;
409 PMMPFN Pfn1;
410 BOOLEAN Zero;
411
412 /* Make sure PFN lock is held and we have pages */
413 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
414 ASSERT(MmAvailablePages != 0);
415 ASSERT(Color < MmSecondaryColors);
416
417 /* Check the colored zero list */
418 PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
419 if (PageIndex == LIST_HEAD)
420 {
421 /* Check the zero list */
422 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
423 PageIndex = MmZeroedPageListHead.Flink;
424 Color = PageIndex & MmSecondaryColorMask;
425 if (PageIndex == LIST_HEAD)
426 {
427 /* This means there's no zero pages, we have to look for free ones */
428 ASSERT(MmZeroedPageListHead.Total == 0);
429 Zero = TRUE;
430
431 /* Check the colored free list */
432 PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
433 if (PageIndex == LIST_HEAD)
434 {
435 /* Check the free list */
436 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
437 PageIndex = MmFreePageListHead.Flink;
438 Color = PageIndex & MmSecondaryColorMask;
439 ASSERT(PageIndex != LIST_HEAD);
440 if (PageIndex == LIST_HEAD)
441 {
442 /* FIXME: Should check the standby list */
443 ASSERT(MmZeroedPageListHead.Total == 0);
444 }
445 }
446 }
447 }
448
449 /* Sanity checks */
450 Pfn1 = MI_PFN_ELEMENT(PageIndex);
451 ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
452 (Pfn1->u3.e1.PageLocation == ZeroedPageList));
453
454 /* Remove the page from its list */
455 PageIndex = MiRemovePageByColor(PageIndex, Color);
456 ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
457
458 /* Zero it, if needed */
459 if (Zero) MiZeroPhysicalPage(PageIndex);
460
461 /* Sanity checks */
462 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
463 ASSERT(Pfn1->u2.ShareCount == 0);
464 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
465 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
466
467 /* Return the page */
468 return PageIndex;
469 }
470
471 VOID
472 NTAPI
473 MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
474 {
475 PMMPFNLIST ListHead;
476 PFN_NUMBER LastPage;
477 PMMPFN Pfn1;
478 ULONG Color;
479 PMMPFN Blink;
480 PMMCOLOR_TABLES ColorTable;
481
482 /* Make sure the page index is valid */
483 ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
484 ASSERT((PageFrameIndex != 0) &&
485 (PageFrameIndex <= MmHighestPhysicalPage) &&
486 (PageFrameIndex >= MmLowestPhysicalPage));
487
488 /* Get the PFN entry */
489 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
490
491 /* Sanity checks that a right kind of page is being inserted here */
492 ASSERT(Pfn1->u4.MustBeCached == 0);
493 ASSERT(Pfn1->u3.e1.Rom != 1);
494 ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
495 ASSERT(Pfn1->u4.VerifierAllocation == 0);
496 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
497
498 /* Get the free page list and increment its count */
499 ListHead = &MmFreePageListHead;
500 ASSERT_LIST_INVARIANT(ListHead);
501 ListHead->Total++;
502
503 /* Get the last page on the list */
504 LastPage = ListHead->Blink;
505 if (LastPage != LIST_HEAD)
506 {
507 /* Link us with the previous page, so we're at the end now */
508 MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
509 }
510 else
511 {
512 /* The list is empty, so we are the first page */
513 ListHead->Flink = PageFrameIndex;
514 }
515
516 /* Now make the list head point back to us (since we go at the end) */
517 ListHead->Blink = PageFrameIndex;
518 ASSERT_LIST_INVARIANT(ListHead);
519
520 /* And initialize our own list pointers */
521 Pfn1->u1.Flink = LIST_HEAD;
522 Pfn1->u2.Blink = LastPage;
523
524 /* Set the list name and default priority */
525 Pfn1->u3.e1.PageLocation = FreePageList;
526 Pfn1->u4.Priority = 3;
527
528 /* Clear some status fields */
529 Pfn1->u4.InPageError = 0;
530 Pfn1->u4.AweAllocation = 0;
531
532 /* Increase available pages */
533 MmAvailablePages++;
534
535 /* Check if we've reached the configured low memory threshold */
536 if (MmAvailablePages == MmLowMemoryThreshold)
537 {
538 /* Clear the event, because now we're ABOVE the threshold */
539 KeClearEvent(MiLowMemoryEvent);
540 }
541 else if (MmAvailablePages == MmHighMemoryThreshold)
542 {
543 /* Otherwise check if we reached the high threshold and signal the event */
544 KeSetEvent(MiHighMemoryEvent, 0, FALSE);
545 }
546
547 /* Get the page color */
548 Color = PageFrameIndex & MmSecondaryColorMask;
549
550 /* Get the first page on the color list */
551 ColorTable = &MmFreePagesByColor[FreePageList][Color];
552 if (ColorTable->Flink == LIST_HEAD)
553 {
554 /* The list is empty, so we are the first page */
555 Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
556 ColorTable->Flink = PageFrameIndex;
557 }
558 else
559 {
560 /* Get the previous page */
561 Blink = (PMMPFN)ColorTable->Blink;
562
563 /* Make it link to us, and link back to it */
564 Blink->OriginalPte.u.Long = PageFrameIndex;
565 Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
566 }
567
568 /* Now initialize our own list pointers */
569 ColorTable->Blink = Pfn1;
570
571 /* This page is now the last */
572 Pfn1->OriginalPte.u.Long = LIST_HEAD;
573
574 /* And increase the count in the colored list */
575 ColorTable->Count++;
576
577 /* Notify zero page thread if enough pages are on the free list now */
578 if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
579 {
580 /* Set the event */
581 MmZeroingPageThreadActive = TRUE;
582 KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
583 }
584
585 #if MI_TRACE_PFNS
586 Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
587 RtlZeroMemory(Pfn1->ProcessName, 16);
588 #endif
589 }
590
591 /* Note: This function is hardcoded only for the zeroed page list, for now */
592 VOID
593 NTAPI
594 MiInsertPageInList(IN PMMPFNLIST ListHead,
595 IN PFN_NUMBER PageFrameIndex)
596 {
597 PFN_NUMBER Flink;
598 PMMPFN Pfn1, Pfn2;
599 MMLISTS ListName;
600 PMMCOLOR_TABLES ColorHead;
601 ULONG Color;
602
603 /* For free pages, use MiInsertPageInFreeList */
604 ASSERT(ListHead != &MmFreePageListHead);
605
606 /* Make sure the lock is held */
607 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
608
609 /* Make sure the PFN is valid */
610 ASSERT((PageFrameIndex) &&
611 (PageFrameIndex <= MmHighestPhysicalPage) &&
612 (PageFrameIndex >= MmLowestPhysicalPage));
613
614 /* Page should be unused */
615 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
616 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
617 ASSERT(Pfn1->u3.e1.Rom != 1);
618
619 /* Only used for zero pages in ReactOS */
620 ListName = ListHead->ListName;
621 ASSERT(ListName == ZeroedPageList);
622 ListHead->Total++;
623
624 /* Don't handle bad pages yet yet */
625 ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
626
627 /* Make the head of the list point to this page now */
628 Flink = ListHead->Flink;
629 ListHead->Flink = PageFrameIndex;
630
631 /* Make the page point to the previous head, and back to the list */
632 Pfn1->u1.Flink = Flink;
633 Pfn1->u2.Blink = LIST_HEAD;
634
635 /* Was the list empty? */
636 if (Flink != LIST_HEAD)
637 {
638 /* It wasn't, so update the backlink of the previous head page */
639 Pfn2 = MI_PFN_ELEMENT(Flink);
640 Pfn2->u2.Blink = PageFrameIndex;
641 }
642 else
643 {
644 /* It was empty, so have it loop back around to this new page */
645 ListHead->Blink = PageFrameIndex;
646 }
647
648 /* Move the page onto its new location */
649 Pfn1->u3.e1.PageLocation = ListName;
650
651 /* One more page on the system */
652 MmAvailablePages++;
653
654 /* Check if we've reached the configured low memory threshold */
655 if (MmAvailablePages == MmLowMemoryThreshold)
656 {
657 /* Clear the event, because now we're ABOVE the threshold */
658 KeClearEvent(MiLowMemoryEvent);
659 }
660 else if (MmAvailablePages == MmHighMemoryThreshold)
661 {
662 /* Otherwise check if we reached the high threshold and signal the event */
663 KeSetEvent(MiHighMemoryEvent, 0, FALSE);
664 }
665
666 /* Sanity checks */
667 ASSERT(ListName == ZeroedPageList);
668 ASSERT(Pfn1->u4.InPageError == 0);
669
670 /* Get the page color */
671 Color = PageFrameIndex & MmSecondaryColorMask;
672
673 /* Get the list for this color */
674 ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
675
676 /* Get the old head */
677 Flink = ColorHead->Flink;
678
679 /* Make this page point back to the list, and point forwards to the old head */
680 Pfn1->OriginalPte.u.Long = Flink;
681 Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
682
683 /* Set the new head */
684 ColorHead->Flink = PageFrameIndex;
685
686 /* Was the head empty? */
687 if (Flink != LIST_HEAD)
688 {
689 /* No, so make the old head point to this page */
690 Pfn2 = MI_PFN_ELEMENT(Flink);
691 Pfn2->u4.PteFrame = PageFrameIndex;
692 }
693 else
694 {
695 /* Yes, make it loop back to this page */
696 ColorHead->Blink = (PVOID)Pfn1;
697 }
698
699 /* One more paged on the colored list */
700 ColorHead->Count++;
701
702 #if MI_TRACE_PFNS
703 //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
704 Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
705 MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
706 RtlZeroMemory(Pfn1->ProcessName, 16);
707 #endif
708 }
709
710 VOID
711 NTAPI
712 MiInitializePfn(IN PFN_NUMBER PageFrameIndex,
713 IN PMMPTE PointerPte,
714 IN BOOLEAN Modified)
715 {
716 PMMPFN Pfn1;
717 NTSTATUS Status;
718 PMMPTE PointerPtePte;
719 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
720
721 /* Setup the PTE */
722 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
723 Pfn1->PteAddress = PointerPte;
724
725 /* Check if this PFN is part of a valid address space */
726 if (PointerPte->u.Hard.Valid == 1)
727 {
728 /* Only valid from MmCreateProcessAddressSpace path */
729 ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
730
731 /* Make this a demand zero PTE */
732 MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
733 }
734 else
735 {
736 /* Copy the PTE data */
737 Pfn1->OriginalPte = *PointerPte;
738 ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
739 (Pfn1->OriginalPte.u.Soft.Transition == 1)));
740 }
741
742 /* Otherwise this is a fresh page -- set it up */
743 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
744 Pfn1->u3.e2.ReferenceCount = 1;
745 Pfn1->u2.ShareCount = 1;
746 Pfn1->u3.e1.PageLocation = ActiveAndValid;
747 ASSERT(Pfn1->u3.e1.Rom == 0);
748 Pfn1->u3.e1.Modified = Modified;
749
750 /* Get the page table for the PTE */
751 PointerPtePte = MiAddressToPte(PointerPte);
752 if (PointerPtePte->u.Hard.Valid == 0)
753 {
754 /* Make sure the PDE gets paged in properly */
755 Status = MiCheckPdeForPagedPool(PointerPte);
756 if (!NT_SUCCESS(Status))
757 {
758 /* Crash */
759 KeBugCheckEx(MEMORY_MANAGEMENT,
760 0x61940,
761 (ULONG_PTR)PointerPte,
762 (ULONG_PTR)PointerPtePte->u.Long,
763 (ULONG_PTR)MiPteToAddress(PointerPte));
764 }
765 }
766
767 /* Get the PFN for the page table */
768 PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
769 ASSERT(PageFrameIndex != 0);
770 Pfn1->u4.PteFrame = PageFrameIndex;
771
772 /* Increase its share count so we don't get rid of it */
773 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
774 Pfn1->u2.ShareCount++;
775 }
776
777 PFN_NUMBER
778 NTAPI
779 MiAllocatePfn(IN PMMPTE PointerPte,
780 IN ULONG Protection)
781 {
782 KIRQL OldIrql;
783 PFN_NUMBER PageFrameIndex;
784 MMPTE TempPte;
785
786 /* Sanity check that we aren't passed a valid PTE */
787 ASSERT(PointerPte->u.Hard.Valid == 0);
788
789 /* Make an empty software PTE */
790 MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
791
792 /* Lock the PFN database */
793 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
794
795 /* Check if we're running low on pages */
796 if (MmAvailablePages < 128)
797 {
798 DPRINT1("Warning, running low on memory: %d pages left\n", MmAvailablePages);
799 //MiEnsureAvailablePageOrWait(NULL, OldIrql);
800 }
801
802 /* Grab a page */
803 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
804 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
805 PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
806
807 /* Write the software PTE */
808 MI_WRITE_INVALID_PTE(PointerPte, TempPte);
809 PointerPte->u.Soft.Protection |= Protection;
810
811 /* Initialize its PFN entry */
812 MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
813
814 /* Release the PFN lock and return the page */
815 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
816 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
817 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
818 return PageFrameIndex;
819 }
820
821 VOID
822 NTAPI
823 MiDecrementShareCount(IN PMMPFN Pfn1,
824 IN PFN_NUMBER PageFrameIndex)
825 {
826 ASSERT(PageFrameIndex > 0);
827 ASSERT(MI_PFN_ELEMENT(PageFrameIndex) != NULL);
828 ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
829 ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
830
831 /* Page must be in-use */
832 if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
833 (Pfn1->u3.e1.PageLocation != StandbyPageList))
834 {
835 /* Otherwise we have PFN corruption */
836 KeBugCheckEx(PFN_LIST_CORRUPT,
837 0x99,
838 PageFrameIndex,
839 Pfn1->u3.e1.PageLocation,
840 0);
841 }
842
843 /* Check if the share count is now 0 */
844 ASSERT(Pfn1->u2.ShareCount < 0xF000000);
845 if (!--Pfn1->u2.ShareCount)
846 {
847 /* ReactOS does not handle these */
848 ASSERT(Pfn1->u3.e1.PrototypePte == 0);
849
850 /* Put the page in transition */
851 Pfn1->u3.e1.PageLocation = TransitionPage;
852
853 /* PFN lock must be held */
854 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
855
856 /* Page should at least have one reference */
857 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
858 if (Pfn1->u3.e2.ReferenceCount == 1)
859 {
860 /* In ReactOS, this path should always be hit with a deleted PFN */
861 ASSERT(MI_IS_PFN_DELETED(Pfn1) == TRUE);
862
863 /* Clear the last reference */
864 Pfn1->u3.e2.ReferenceCount = 0;
865 ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
866
867 /* Mark the page temporarily as valid, we're going to make it free soon */
868 Pfn1->u3.e1.PageLocation = ActiveAndValid;
869
870 /* Bring it back into the free list */
871 MiInsertPageInFreeList(PageFrameIndex);
872 }
873 else
874 {
875 /* Otherwise, just drop the reference count */
876 InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
877 }
878 }
879 }
880
881 VOID
882 NTAPI
883 MiDecrementReferenceCount(IN PMMPFN Pfn1,
884 IN PFN_NUMBER PageFrameIndex)
885 {
886 /* PFN lock must be held */
887 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
888
889 /* Sanity checks on the page */
890 ASSERT(PageFrameIndex < MmHighestPhysicalPage);
891 ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
892 ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
893
894 /* Dereference the page, bail out if it's still alive */
895 InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
896 if (Pfn1->u3.e2.ReferenceCount) return;
897
898 /* Nobody should still have reference to this page */
899 if (Pfn1->u2.ShareCount != 0)
900 {
901 /* Otherwise something's really wrong */
902 KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
903 }
904
905 /* And it should be lying on some page list */
906 ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
907
908 /* Did someone set the delete flag? */
909 if (MI_IS_PFN_DELETED(Pfn1))
910 {
911 /* Insert it into the free list, there's nothing left to do */
912 MiInsertPageInFreeList(PageFrameIndex);
913 return;
914 }
915
916 /* We don't have a modified list yet */
917 ASSERT(Pfn1->u3.e1.Modified == 0);
918 ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
919
920 /* FIXME: Normally it would go on the standby list, but we're pushing it on the free list */
921 MiInsertPageInFreeList(PageFrameIndex);
922 }
923
924 VOID
925 NTAPI
926 MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex,
927 IN PMMPTE PointerPte,
928 IN PFN_NUMBER PteFrame)
929 {
930 PMMPFN Pfn1;
931
932 /* Setup the PTE */
933 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
934 Pfn1->PteAddress = PointerPte;
935
936 /* Make this a software PTE */
937 MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
938
939 /* Setup the page */
940 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
941 Pfn1->u3.e2.ReferenceCount = 1;
942 Pfn1->u2.ShareCount = 1;
943 Pfn1->u3.e1.PageLocation = ActiveAndValid;
944 Pfn1->u3.e1.Modified = TRUE;
945 Pfn1->u4.InPageError = FALSE;
946
947 /* Did we get a PFN for the page table */
948 if (PteFrame)
949 {
950 /* Store it */
951 Pfn1->u4.PteFrame = PteFrame;
952
953 /* Increase its share count so we don't get rid of it */
954 Pfn1 = MI_PFN_ELEMENT(PteFrame);
955 Pfn1->u2.ShareCount++;
956 }
957 }
958
959 /* EOF */